How to clean nodejs response after every user request? - node.js

Every time a user enter to our URL he get the response of the last user + his new response.
I'm new with nodejs and I think I don't understand the way this server works :)
Anyway, here you can see the app:
https://s1-nodejs.herokuapp.com (You will need to reload the page to see the duplicate response every time you reload).
And this is the code I deployed to Heroku:
const start = Date.now();
// using the http module
var http = require('http');
var https = require('https');
var request = require('request');
var domain;
var apiKey = 'example';
var domainResultJson = [];
var promises = [];
// look for PORT environment variable,
// else look for CLI argument,
// else use hard coded value for port 8080
port = process.env.PORT || process.argv[2] || 8080;
// create a simple server
let server = http.createServer(function (req, res) {
for (var i = 0 ; i < 10 ; i++ ) {
var domain = 'example-'+i+'.com';
promises.push(CheckIfDomainAvailable(https,domain,apiKey,domainResultJson));
}
Promise.all(promises).then(function(values) {
//console.log(values);
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
for (var i in values) {
val = values[i];
res.write(val);
}
res.end();
}).catch(
console.error
);
function CheckIfDomainAvailable(https,domain,apiKey,domainResultJson) {
return new Promise(function (success, failure) {
request('https://example.com/api/v1?'
+ 'apiKey=' + apiKey + '&d=' + domain + '', function (error, response, body) {
if (!error && response.statusCode == 200) {
domainResultJson.push('1');
success(body);
} else {
failure(error);
}
});
});
}
});
// listen on the port
server.listen(port, function () {
console.log('app up on port: ' + port);
});

Try to declare let promises = []; inside function (req, res) { ... }, like this:
// ...
let server = http.createServer(function (req, res) {
let promises = []; // <= Define promises container here.
for (var i = 0 ; i < 10 ; i++ ) {
var domain = 'example-'+i+'.com';
promises.push(CheckIfDomainAvailable(https,domain,apiKey,domainResultJson));
}
// ...

Related

Heroku Nodejs Webhosting

Im trying to host a election website.
My Current Method explained:
There is server.js file with serves the public file. If its a POST request(sent by frontend.js to send data to server) edits the poll.json file based on the data sent.
I have a public file having index.html, frontend.js, poll.json where poll.json stores all the data.
My current code work all well in my localhost.
But when running it in Heroku I get a error in the POST request line saying 'POST http://localhost:53390/ net::ERR_CONNECTION_REFUSED1
*********.herokuapp.com/:1 Uncaught (in promise) TypeError: Failed to fetch'
My server.js code:
const createServer = require('http').createServer;
const express = require('express');
const app = express();
let pollData = require(__dirname+'/public/poll.json');
const fs = require('fs');
var all_usn_no=[];
const listening_port=process.env.PORT || 4000;
function get_roll_and_usn(pollData){
for(var i=0;i<=pollData.students.length-1;i++){
//all_roll_no.push(pollData.students[i][0]);
all_usn_no.push(pollData.students[i][1]);
}
}
function roll_to_row(in_usn){
get_roll_and_usn(pollData)
return all_usn_no.indexOf(in_usn);
}
function write_vote(votes){
var checking_row=roll_to_row(votes[1]);
pollData.students[checking_row]=votes;
fs.writeFile(__dirname+'/public/poll.json', JSON.stringify(pollData), (err) => {
if (err) throw err;
console.log('Data written to file');
});
}
write_vote([listening_port,0]);
app.use(express.static(__dirname+'/public'));
app.get('/', (req, res) => {
res.sendFile(__dirname);// + '/index.html');
});
app.post('/', (req, res) => {
let body = '';
req.on('data', data => body += data)
req.on('end', () => {
res.writeHead(200, {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'POST, GET',
});
body=body.split(",");
console.log(body.toString());
write_vote(body);
res.end(`{ "response": "${body ? body : 'No body sent' }" }`);
})
});
app.listen(listening_port, () => {
console.log('Example app listening at http://localhost:'+listening_port)
});
my frontend.js program:
var roll='';
var client_ip='';
var usn='';
var date='';
const all_roll_no=[]
const all_usn_no=[]
var port='';
function check(roll,usn){
var data='';
const url='poll.json';
const Http = new XMLHttpRequest();
Http.open("GET", url);
Http.send();
Http.onload = () =>{
data=JSON.parse(Http.response);
get_roll_and_usn(data);
check_validity(roll,usn,data);
}
}
function get_roll_and_usn(pollData){
for(var i=0;i<=pollData.students.length-1;i++){
//all_roll_no.push(pollData.students[i][0]);
all_usn_no.push(pollData.students[i][1]);
}
}
function usn_to_row(in_usn){
return all_usn_no.indexOf(in_usn);
}
function check_validity(checking_roll,checking_usn,data){
var checking_row=usn_to_row(checking_usn);
port=data.students[0][0];
//if(all_roll_no.indexOf(checking_roll)>=0 && checking_usn==all_usn_no[all_roll_no.indexOf(checking_roll)] && data.students[checking_row][2]==0){
if(all_usn_no.indexOf(checking_usn)>=0 && data.students[checking_row][2]==0){
//console.log("valid");
document.getElementById("page_2").style.display = "none";
document.getElementById("page_3").style.display = "block";
fetch('https://api.ipify.org/?format=json').then(results=> results.json()).then(data => client_ip=data.ip);
}
else{
alert("You cannot vote/ You have already voted")
//console.log('invalid');
}
}
document.getElementById("startbutton").onclick = function(){
roll = document.getElementById("roll_no").value;
usn = document.getElementById("usn_no").value;
date = Date();
check(roll,usn);
}
document.getElementById("next").onclick = function() {
document.getElementById("page_1").style.display = "none";
document.getElementById("page_2").style.display = "block";
}
document.getElementById("finish").onclick = function() {
var splb=[document.getElementById("splb1").checked,document.getElementById("splb2").checked,document.getElementById("splb3").checked,document.getElementById("splb4").checked,document.getElementById("splb5").checked,document.getElementById("splb6").checked,document.getElementById("splb7").checked];
var splg=[document.getElementById("splg1").checked,document.getElementById("splg2").checked,document.getElementById("splg3").checked,document.getElementById("splg4").checked,document.getElementById("splg5").checked,document.getElementById("splg6").checked,document.getElementById("splg7").checked];
var asplb=[document.getElementById("asplb1").checked,document.getElementById("asplb2").checked,document.getElementById("asplb3").checked,document.getElementById("asplb4").checked,document.getElementById("asplb5").checked,document.getElementById("asplb6").checked,document.getElementById("asplb7").checked];
var asplg=[document.getElementById("asplg1").checked,document.getElementById("asplg2").checked,document.getElementById("asplg3").checked,document.getElementById("asplg4").checked,document.getElementById("asplg5").checked,document.getElementById("asplg6").checked,document.getElementById("asplg7").checked];
var csb=[document.getElementById("csb1").checked,document.getElementById("csb2").checked,document.getElementById("csb3").checked,document.getElementById("csb4").checked,document.getElementById("csb5").checked,document.getElementById("csb6").checked,document.getElementById("csb7").checked];
var csg=[document.getElementById("csg1").checked,document.getElementById("csg2").checked,document.getElementById("csg3").checked,document.getElementById("csg4").checked,document.getElementById("csg5").checked,document.getElementById("csg6").checked,document.getElementById("csg7").checked];
var acsb=[document.getElementById("acsb1").checked,document.getElementById("acsb2").checked,document.getElementById("acsb3").checked,document.getElementById("acsb4").checked,document.getElementById("acsb5").checked,document.getElementById("acsb6").checked,document.getElementById("acsb7").checked];
var acsg=[document.getElementById("acsg1").checked,document.getElementById("acsg2").checked,document.getElementById("acsg3").checked,document.getElementById("acsg4").checked,document.getElementById("acsg5").checked,document.getElementById("acsg6").checked,document.getElementById("acsg7").checked];
var scb=[document.getElementById("scb1").checked,document.getElementById("scb2").checked,document.getElementById("scb3").checked,document.getElementById("scb4").checked,document.getElementById("scb5").checked,document.getElementById("scb6").checked,document.getElementById("scb7").checked];
var scg=[document.getElementById("scg1").checked,document.getElementById("scg2").checked,document.getElementById("scg3").checked,document.getElementById("scg4").checked,document.getElementById("scg5").checked,document.getElementById("scg6").checked,document.getElementById("scg7").checked];
var ascb=[document.getElementById("ascb1").checked,document.getElementById("ascb2").checked,document.getElementById("ascb3").checked,document.getElementById("ascb4").checked,document.getElementById("ascb5").checked,document.getElementById("ascb6").checked,document.getElementById("ascb7").checked];
var ascg=[document.getElementById("ascg1").checked,document.getElementById("ascg2").checked,document.getElementById("ascg3").checked,document.getElementById("ascg4").checked,document.getElementById("ascg5").checked,document.getElementById("ascg6").checked,document.getElementById("ascg7").checked];
var vote=[String(splb.indexOf(true)),String(splg.indexOf(true)),String(asplb.indexOf(true)),String(asplg.indexOf(true)),String(csb.indexOf(true)),String(csg.indexOf(true)),String(acsb.indexOf(true)),String(acsg.indexOf(true)),String(scb.indexOf(true)),String(scg.indexOf(true)),String(ascb.indexOf(true)),String(ascg.indexOf(true))]
var update=[String(roll),String(usn),"1",String(client_ip),String(date)].concat(vote);
if (update.indexOf("-1")<0){
alert("Pls vote for all posts")
}
else{
document.getElementById("page_1").style.display = "none";
document.getElementById("page_2").style.display = "none";
document.getElementById("page_3").style.display = "none";
fetch('http://localhost:'+port,{method:'POST',body:update}).then(results=> results.json()).then(console.log);
//console.log(update);
alert("Your vote has been registered")
}
}
You can just ignore the function as they are just to process the data and do necessary funtions.
Main problem: PORT request from frontend.js to server.js sending data to edit the poll.json file return error.
Thanks in advance.
Try using const PORT = process.env.PORT || 3333;
with
app.listen(PORT, () => {
console.log(`Server is starting on ${PORT}`);
})

Get database data from different url pages - what is the best technology?

I want to create a mobile app that call various urls and get for each url a different response based on SQL Server data.
I try to use Nodejs for this job but when I get the data, I cannot retrieve a single response for that page.
Someone suggested to use Socketweb.io instead of routers... what is the best option for your experience?
This is the code:
var express = require('express');
var app = express();
var dbResp;
const sql = require("msnodesqlv8");
var buf = '';
var router = express.Router();
const connectionString = <my connection string>;
router.get('/page1', function (req, res) {
var query = "SELECT query 1";
sql.query(connectionString, query, (err, rows) => {
try {
grows = rows;
for (i = 0; i < rows.length; i++) {
buf += "get the data...";
}
}
catch {
if (err) {
console.log(err);
}
}
});
res.send(buf);
});
router.get('/page2', function (req, res) {
var query = "SELECT query 2";
sql.query(connectionString, query, (err, rows) => {
try {
grows = rows;
for (i = 0; i < rows.length; i++) {
buf += "get the data...";
}
}
catch {
if (err) {
console.log(err);
}
}
});
res.send(buf);
});
app.use('/page1', router);
app.use('/page2', router);
app.listen(8080, function () {
console.log('Example app listening on port 8080!');
});

Chunk/Stream API data using Node.js

We have requirement where we need to write a node application which can read URL of image from database (approx more than million). Use image-size npm package to retrieve image meta data like height, width. Here should be an API which can list out result.
I am able to console log data but when i convert it to API, i need to chunk data so it can start appearing on browser and i'm unable to do that and need help. Here is my code
var express = require('express');
var url = require('url');
var http = require('http');
var sizeOf = require('image-size');
const sql = require('mssql');
var app = express();
var port = process.env.PORT || 3000;
const hostname = 'localhost';
var config1 = {
user: '*********',
password: '*********',
server: '*********',
database: '*******',
port: 1433,
debug: true,
options: {
encrypt: false // Use this if you're on Windows Azure
}
};
app.get('/', function(req, res){
//res.writeHead(200, { 'Content-Type': 'application/json' });
var finalResult = [];
sql.close();
sql.connect(config1, function (err) {
if (err) console.log(err);
const request = new sql.Request()
var myQuery = `select imagename from media`;
request.stream = true;
request.query(myQuery);
request.on('row', row => {
//console.log('Image : ' + row.ImageUrl);
if (row.ImageUrl != ''){
if (row.ImageUrl.indexOf('http') < 0)
row.ImageUrl = "http:" + row.ImageUrl;
var options = url.parse(row.ImageUrl);
http.get(options, function (response) {
if (response.statusCode == 200)
{
var chunks = [];
response.on('data', function (chunk) {
chunks.push(chunk);
}).on('end', function() {
var buffer = Buffer.concat(chunks);
//console.log(options.href);
//console.log(sizeOf(buffer).height);
var result = {};
result.MaskUrl = row.MaskUrl;
result.ImageUrl = options.href;
result.Height = sizeOf(buffer).height;
result.Width = sizeOf(buffer).width;
result.statusCode = 200;
finalResult.push(result);
//console.log(result);
console.log(finalResult);
res.write(result, function(){
res.end();
});
});
}
else
{
var result = {};
result.MaskUrl = row.MaskUrl;
result.ImageUrl = options.href;
result.Height = 0;
result.Width = 0;
result.statusCode = response.statusCode;
finalResult.push(result);
console.log(result);
res.write(result, function(){
res.end();
});
}
});
}
})
request.on('error', err => {
console.log ('Error for ' + row.ImageUrl );
})
request.on('done', err => {
console.log('Last Time' + finalResult.length);
})
// request.query(myQuery,(err,result) =>{
// console.log(result);
// });
});
console.log('Last Time' + finalResult.length);
res.send(finalResult);
});
app.listen(port, hostname, function(){
console.log('ImageSize running on PORT: ' + port);
});
I tried res.write, res.end without any success.
The probable reason for your problem is that here:
res.write(result, function(){
res.end();
});
You end and close the request just after the first image is read.
I would rewrite the code a little and use some functional framework, like scramjet, to stream the data straight from the DB. As Nicholas pointed out it's not super easy to run your code so I'm writing blindly - but if you fix any of my obvious error this should just work:
First:
npm install scramjet JSONStream node-fetch
Next, try this code:
var express = require('express');
var sizeOf = require('image-size');
const sql = require('mssql');
var app = express();
var port = process.env.PORT || 3000;
const hostname = 'localhost';
const {DataStream} = require('scramjet');
const fetch = require('node-fetch');
const JSONStream = require('JSONStream');
var config1 = {
user: '*********',
password: '*********',
server: '*********',
database: '*******',
port: 1433,
debug: true,
options: {
encrypt: false // Use this if you're on Windows Azure
}
};
app.get('/', function(req, res, next){
// you should consider not doing these two lines on each request,
// but I don't want to mess you code...
sql.close();
sql.connect(config1, function (err) {
if (err) next(err);
res.writeHead(200, { 'Content-Type': 'application/json' });
const request = new sql.Request();
var myQuery = `select imagename from media`;
request.stream = true;
request.query(myQuery);
const stream = new DataStream();
request.on('row', row => stream.write(row));
stream.filter(
row => row.ImageUrl !== ''
)
.map(
async row => {
if (row.ImageUrl.indexOf('http') !== 0) // url must start with http.
row.ImageUrl = "http:" + row.ImageUrl;
const response = await fetch(row.ImageUrl);
let size = {width:0, height:0};
if (response.status === 200) {
const buffer = await response.buffer();
size = sizeOf(buffer);
}
return {
MaskUrl: row.MaskUrl,
ImageUrl: row.ImageUrl,
Height: size.height,
Width: size.width,
statusCode: response.status
};
}
)
.pipe(
JSONStream.stringify()
).pipe(
res
);
request.on('error', () => {
res.writeHead(500, { 'Content-Type': 'application/json' });
stream.end("{error:true}");
});
request.on('done', () => stream.end());
});
});
app.listen(port, hostname, function(){
console.log('ImageSize running on PORT: ' + port);
});

Node.js http-proxy inside cluster worker

I build a sample code in node js
var cluster = require("cluster"),
http = require("http"),
express = require('express'),
port = parseInt(process.argv[2]||8001),
servers = ['http://127.0.0.1:800821', 'http://127.0.0.1:800831'];;
if (cluster.isMaster) {
console.log('Master ' + process.pid + ' has started.');
var numWorkers = require('os').cpus().length;
console.log('Master cluster setting up ' + numWorkers + ' workers...');
for(var i = 0; i < 1; i++) {
cluster.fork();
}
cluster.on('online', function(worker) {
console.log('Worker ' + worker.process.pid + ' is online');
});
cluster.on('exit', function(worker, code, signal) {
console.log('Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal);
console.log('Starting a new worker');
cluster.fork();
});
} else {
var httpProxy = require('http-proxy');
var proxy = httpProxy.createProxyServer();
var count = 0;
// Workers share the TCP connection in this server
var app = express();
app.get('/', function (req, res) {
console.log('Cluster => ' + process.pid);
loadBalanceProxy(req,res);
}).listen(port);
var currentServer = 1;
function loadBalanceProxy(req, res){
var cur = currentServer%servers.length;
currentServer++;
var target = servers[cur];
console.log("Proxy => " + target);
proxy.web(req, res, {
target: target
});
}
}
http.createServer(function (req, res) {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.write('request successfully proxied to: ' + req.url + '\n' + JSON.stringify(req.headers, true, 2));
res.end();
}).listen(800831);
http.createServer(function (req, res) {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.write('request successfully proxied to: ' + req.url + '\n' + JSON.stringify(req.headers, true, 2));
res.end();
}).listen(800821);
In this sample, I want to create proxy server inside cluster worker and this give me error bind EADDRINUSE null:800831
I want to know can I create http-proxy inside cluster worker. If I can't then there's solution for load balance between machines ?
I create a proxy reverse exemple proxy reverse inside cluster and i create 3 servers in another files, it works for me (dont forget to run the servers separately for example on windows you launch servers in 4 cmd, 1 cmd for proxy and other (3 cmd) for servers )
// var https = require('https'); scure
var http = require('http');
var proxy = require('http-proxy');
var cluster = require('cluster');
var fs = require('fs');
var request = require('request');
// const numCPUs = require('os').cpus().length;
var numCPUs = 4;
if (cluster.isMaster) {
for (var i = 0; i < numCPUs; i++) {
cluster.fork();
}
cluster.on('fork', (worker) => {
console.log('worker ' + worker.process.pid);
});
cluster.on('exit', function(worker, code, signal) {
console.log('Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal);
console.log('Starting a new worker');
cluster.fork();
});
} else {
startProxyReverse();
}
function startProxyReverse() {
http.globalAgent.maxSockets = 10240;
// Define the servers to load balance.
var servers = [{
host: '127.0.0.1',
port: 8001
}, {
host: '127.0.0.1',
port: 8003
}, {
host: '127.0.0.1',
port: 8002
}];
var failoverTimer = [];
// load the SSL cert
// var ca = [
// fs.readFileSync('./certs/PositiveSSLCA2.crt'),
// fs.readFileSync('./certs/AddTrustExternalCARoot.crt')
// ];
// var opts = {
// ca : ca,
// key : fs.readFileSync('./certs/example_wild.key'),
// cert : fs.readFileSync('./certs/STAR_example_com.crt')
// };
// Create a proxy object for each target.
var proxies = servers.map(function(target) {
return new proxy.createProxyServer({
target: target
// ws : true,
// xfwd : true,
// ssl : opts,
// down : false
});
});
/**
* Select a random server to proxy to. If a 'server' cookie is set, use that
* as the sticky session so the user stays on the same server (good for ws fallbacks).
* #param {Object} req HTTP request data
* #param {Object} res HTTP response
* #return {Number} Index of the proxy to use.
*/
var selectServer = function(req, res) {
var index = -1;
var i = 0;
// Check if there are any cookies.
if (req.headers && req.headers.cookie && req.headers.cookie.length > 1) {
var cookies = req.headers.cookie.split('; ');
for (i = 0; i < cookies.length; i++) {
if (cookies[i].indexOf('server=') === 0) {
var value = cookies[i].substring(7, cookies[i].length);
if (value && value !== '') {
index = value;
break;
}
}
}
}
// Select a random server if they don't have a sticky session.
if (index < 0 || !proxies[index]) {
index = Math.floor(Math.random() * proxies.length);
}
// If the selected server is down, select one that isn't down.
if (proxies[index].options.down) {
index = -1;
var tries = 0;
while (tries < 5 && index < 0) {
var randIndex = Math.floor(Math.random() * proxies.length);
if (!proxies[randIndex].options.down) {
index = randIndex;
}
tries++;
}
}
index = index >= 0 ? index : 0;
// Store the server index as a sticky session.
if (res) {
res.setHeader('Set-Cookie', 'server=' + index + '; path=/');
}
return index;
};
/**
* Fired when there is an error with a request.
* Sets up a 10-second interval to ping the host until it is back online.
* There is a 10-second buffer before requests start getting blocked to this host.
* #param {Number} index Index in the proxies array.
*/
var startFailoverTimer = function(index) {
if (failoverTimer[index]) {
return;
}
failoverTimer[index] = setTimeout(function() {
// Check if the server is up or not
request({
url: 'http://' + proxies[index].options.target.host + ':' + proxies[index].options.target.port,
method: 'HEAD',
timeout: 10000
}, function(err, res, body) {
failoverTimer[index] = null;
if (res && res.statusCode === 200) {
proxies[index].options.down = false;
console.log('Server #' + index + ' is back up.');
} else {
proxies[index].options.down = true;
startFailoverTimer(index);
console.log('Server #' + index + ' is still down.');
}
});
}, 10000);
};
// Select the next server and send the http request.
var serverCallback = function(req, res) {
console.log('Process ' + process.pid + ' is listening to all incoming requests');
var proxyIndex = selectServer(req, res);
console.log(proxyIndex);
var proxy = proxies[proxyIndex];
proxy.web(req, res);
proxy.on('error', function(err) {
startFailoverTimer(proxyIndex);
});
};
console.log('create server');
// var server = http.createServer(opts, serverCallback); scure server
var server = http.createServer(serverCallback);
// http.createServer(serverCallback).listen(8000);
// Get the next server and send the upgrade request.
// server.on('upgrade', function (req, socket, head) {
// var proxyIndex = selectServer(req);
// var proxy = proxies[proxyIndex];
// proxy.ws(req, socket, head);
// proxy.on('error', function (err, req, socket) {
// socket.end();
// startFailoverTimer(proxyIndex);
// });
// });
server.listen(8000);
// server.listen(443); scure port
// var proxi = proxy.createServer();
// http.createServer(function (req, res) {
// var target = {
// target : servers.shift()
// };
// console.log('Process ' + process.pid + ' is listening to all incoming requests');
// console.log('balancing request to: ', target);
// proxi.web(req, res, target);
// servers.push(target.target);
// }).listen(8000);
}
Ports should in 1-65535 range.

Socket.io emit doesnt work

Basically,
I first initiate socket.io like this:
var io = require('socket.io')(1337);
Then, after using http to get a POST request and check some info, I try this:
var updateArray = {timer:"start"};
jsonUpdate = JSON.stringify(updateArray);
io.emit('update', jsonUpdate);
But it doesn't send the sockets, and I really can't understand the socket.io documentation sadly, so I'd be happy if someone can help me out.
Server code:
var http = require('http');
var fs = require('fs');
var io = require('socket.io')(1337);
var initialBomb = 0;
function now() {
return Math.floor(new Date() / 1000);
}
http.createServer(function (req, res) {
var body = "";
req.on('data', function (chunk) {
if (req.method == 'POST') {
body += chunk;
}
});
req.on('end', function () {
parsedBody = JSON.parse(body);
if (parsedBody.round["bomb"] == "planted") {
var rightNow = now();
var initialCheck = initialBomb + 41;
if (rightNow > initialCheck) {
initialBomb = now();
var updateArray = {timer:"start"};
jsonUpdate = JSON.stringify(updateArray);
io.emit('update', jsonUpdate);
console.log(jsonUpdate);
}
}
});
}).listen(3000);
Client Code:
<script>
var socket = io('87.98.219.48:1337');
socket.on('update', function(payload) {
var data = JSON.parse(payload);
console.log(payload);
if (data['timer'] == 'start') {
initTick = timerNow();
setTimeout(tick, delay);
}
});
</script>

Resources