Socket io server freezing(it seems) after certain amount of time? - node.js

My code is too long to post in here, but basically I use socket.io server to pull data from database and refresh in the client every 1 second.
Like this:
function updateTimer(){
//db->query
io.sockets.emit('updated data', data);
}
setInterval(updateTimer, 1000);
After a certain amount of time, the server just stops emitting data. I use a chat on the website as well and it stops too. But in the server console nothing is shown, no errors or any outputs, just "listening on port 3000..." stays on the screen all the time.
I thought it could be the loop maybe but I think there's no other way of refreshing data every 1 second, am I right?
If someone can help me and needs the full code please open a discussion and I'll paste it somewhere.
EDIT for the function code:
function checkRoulleteTime() {
try {
pool.getConnection(function(err, connection) {
connection.query('SELECT * FROM `roullete` WHERE status=\'active\'', function(err, rows) {
if (err) {
console.log(err);
return;
}
if (rows.length == 0) return;
var time = rows[0].time - (Math.floor(Date.now() / 1000));
if (time <= 1) {
connection.query('UPDATE `roullete` SET `status`=\'closed\' WHERE `id`=\'' + rows[0].id + '\'', function(error, fields) {
if (error) throw error;
});
setTimeout(roll, 1000);
setTimeout(function(){
io.sockets.emit('add hist');
}, 10500);
setTimeout(updatePoints, 12000);
setTimeout(newRound, 12500);
}
var contagem = Object.keys(clients).length;
io.sockets.emit('login count', contagem);
io.sockets.emit('roullete time', time);
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'black\'',function(error2, rows2){
if (error2) throw error2;
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'green\'',function(error4, rows4){
if (error4) throw error4;
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'red\'',function(error3, rows3){
if (error3) throw error3;
var onBlack = rows2[0].points;
var onRed = rows3[0].points;
var onGreen = rows4[0].points;
io.sockets.emit('calculations', {"time": time, "black" : onBlack, "red" : onRed, "green" : onGreen});
});
});
});
});
connection.release();
});
} catch(e) {
console.log('error here:'+e);
}
setTimeout(checkRoulleteTime, 1000);
}
setTimeout(checkRoulleteTime, 1000);
Alright, that's my code.

My guess is that you have some sort of programming error in your database query that, after a little while exhausts some resources so your database query starts failing every time, throwing an exception or just returning an error and thus you don't ever send any data because of the error. Because the code is inside the setInterval() callback, any exception is not logged for you.
We could probably help you further if you included your actual database code, but you can start to debug it yourself by putting an exception handler around it like this:
function updateTimer(){
try {
//db->query
io.sockets.emit('updated data', data);
} catch(e) {
console.log(e);
}
}
setInterval(updateTimer, 1000);
And, if your DB query is async (which I'm assuming it is), you will need to have explicit error handling and an exception at each callback level (since exceptions don't propagate up async callbacks).
If your database may get slow, then it might be safer to change your recurring code to work like this:
function updateTimer(){
try {
//db->query
io.sockets.emit('updated data', data);
} catch(e) {
console.log(e);
}
// schedule next updateTimer() call when this one has finished
setTimeout(updateTimer, 1000);
}
// schedule first updateTimer() call
setTimeout(updateTimer, 1000);
You have LOTS of places in your code where you are leaking a pooled connection and lots of places where you are not logging an error. My guess is that you are running out of pooled connections, getting an error over and over that you don't log.
Here's a version of your code that attempts to clean things up so all errors are logged and no pooled connections are leaked. Personally, I would write this code using promises which makes robust error handling and reporting a ton easier. But, here's a modified version of your code:
function checkRoulleteTime() {
try {
pool.getConnection(function (err, connection) {
if (err) {
console.log("Failed on pool.getConnection()", err);
return;
}
connection.query('SELECT * FROM `roullete` WHERE status=\'active\'', function (err, rows) {
if (err) {
connection.release();
console.log(err);
return;
}
if (rows.length == 0) {
connection.release();
return;
}
var time = rows[0].time - (Math.floor(Date.now() / 1000));
if (time <= 1) {
connection.query('UPDATE `roullete` SET `status`=\'closed\' WHERE `id`=\'' + rows[0].id + '\'', function (error, fields) {
if (error) {
console.log(error);
connection.release();
return;
}
});
setTimeout(roll, 1000);
setTimeout(function () {
io.sockets.emit('add hist');
}, 10500);
setTimeout(updatePoints, 12000);
setTimeout(newRound, 12500);
}
var contagem = Object.keys(clients).length;
io.sockets.emit('login count', contagem);
io.sockets.emit('roullete time', time);
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'black\'', function (error2, rows2) {
if (error2) {
console.log(error2);
connection.release();
return;
}
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'green\', function (error4, rows4) {
if (error4) {
console.log(error4);
connection.release();
return;
}
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'red\'', function (error3, rows3) {
connection.release();
if (error3) {
console.log(error3);
return;
}
var onBlack = rows2[0].points;
var onRed = rows3[0].points;
var onGreen = rows4[0].points;
io.sockets.emit('calculations', {
"time": time,
"black": onBlack,
"red": onRed,
"green": onGreen
});
});
});
});
});
});
} catch (e) {
console.log('error here:' + e);
}
setTimeout(checkRoulleteTime, 1000);
}
setTimeout(checkRoulleteTime, 1000);
And to give you an idea how much simpler it can be to do error handling and chaining of sequential or dependent async functions, here's your function rewritten using promises. I have no idea if this runs without error since I have no way of testing it, but it should give you an idea how much cleaner programming with promises can be:
var Promise = require('bluebird');
pool = Promise.promisifyAll(pool);
function logErr(err) {
console.log(err);
}
function checkRoulleteTime() {
pool.getConnectionAsync().then(function(connection) {
var query = Promise.promisify(connection.query, {context: connection});
return query('SELECT * FROM `roullete` WHERE status=\'active\'').then(function(rows) {
if (rows.length !== 0) {
var time = rows[0].time - (Math.floor(Date.now() / 1000));
if (time <= 1) {
query('UPDATE `roullete` SET `status`=\'closed\' WHERE `id`=\'' + rows[0].id + '\'').catch(logErr);
setTimeout(roll, 1000);
setTimeout(function () {
io.sockets.emit('add hist');
}, 10500);
setTimeout(updatePoints, 12000);
setTimeout(newRound, 12500);
}
var contagem = Object.keys(clients).length;
io.sockets.emit('login count', contagem);
io.sockets.emit('roullete time', time);
function doQuery(color) {
return query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'' + color + '\'');
}
return Promise.all([doQuery('black'), doQuery('green'), doQuery('red')]).then(function(results) {
io.sockets.emit('calculations', {
"time": time,
"black": results[0][0].points,
"green": results[1][0].points,
"red": results[2][0].points
});
});
}
}).catch(logErr).finally(function() {
connection.release();
setTimeout(checkRoulleteTime, 1000);
});
}, function(err) {
console.log("Err getting connection: ", err);
});
}

Related

node js await does not seem to wait

I am making a request to a mySQL Database and want to work with the returned results. I have the async function
async function doQuery(query) {
try {
console.log('0. connection start');
var connection = MySQL.createConnection({
host: "xxxxxxxx",
user: "yyyyyyyy",
password: "zzzzzzzz"
});
console.log('1. Connection set up. Connecting...');
var queryResults;
await connection.connect(function(err) {
if (err) console.log(err);
console.log("2. Connected!");
connection.query(query, function(err, result) {
if (err) console.log(err);
queryResults = JSON.parse(JSON.stringify(result));
console.log("3. " + util.inspect(result, {depth: 4}));
console.log("3.5 " + util.inspect(queryResults, {depth: 4}));
connection.end();
});
})
console.log ("results before return:" + JSON.stringify(queryResults));
return queryResults;
}
catch (err){
console.log("async err: " + err);
}
}
The function is then called from another function:
function handle async(handlerInput) {
console.log("ExecuteDBSearchHandler");
var query = buildQuery(handlerInput);
try{
var resultsPromise = doQuery(query)
.then(results => {
console.log("4. final results: " + results);
const count = results.length;
var speechText = 'Abfrage Erfolgreich, es gab ' + count + " Ergebnisse.";
return handlerInput.responseBuilder
.speak(speechText)
.withSimpleCard('Abfrage Erfolgreich', speechText)
.withShouldEndSession(false)
.getResponse();
})
}
catch (err){
console.log("error:" + err);
}
}
This code is to be used in a nodejs Alexa skill. When the function runs, I would expect the outputs in the log to be ordered ascending. The output I consistently get is 0, 1, 4, 2, 3, 3.5. The .then part is executed before the actual connection is established and the database queried.
Instead of the .then, I tried a simple await before, making the line
var results = await doQuery(query);
This lead to the exact same result. I am at a loss as to why await is infact not waiting. As a sidenote: While inside the block
connection.query(query, function (err, result) {
if (err) console.log(err);
queryResults = JSON.parse(JSON.stringify(result));
console.log("3. " + util.inspect(result, {depth: 4}));
console.log("3.5 " + util.inspect(queryResults, {depth: 4}));
connection.end();
});
queryResults shows results (in the debug log step 3.5). In the line
console.log ("results before return:" + JSON.stringify(queryResults));
it returns undefined. I tried so many things to get the results to the function scope of doQuery, but I can't seem to achieve that. Does the await fail due to this? And how can I fix it? I have read upwards of a dozen articles and docs about async/await and promises, yet I can't figure this out.
You need to wrap your callback function in Promise to make use of await keyword
For example:
async function dbQuery(query) {
try {
const connection = MySQL.createConnection({
host: "xxxxxxxx",
user: "yyyyyyyy",
password: "zzzzzzzz"
});
return new Promise(function(resolve, reject) {
connection.connect(function(err) {
if (err) reject(err);
connection.query(query, function (err, result) {
if (err) reject(err);
connection.end();
resolve(JSON.parse(JSON.stringify(result)));
});
});
});
} catch {
console.error("async err: " + err);
}
}
The one can also use util.promisify to wrap callback in promise

Passing arguments through predefined function in Node.js

I'm struggling with passing data concept in Node.js.
Let's take SQL tedious as example. Here is code from examples:
//acquire a connection
pool.acquire(function poolAcquire(err, connection) {
if (err) { console.error(err); return; }
//use the connection as normal
var request = new Request('select 1;select 2;select 3', function requestCallback (err, rowCount) {
if (err) { console.error(err); return;}
console.log('rowCount: ' + rowCount);
//release the connection back to the pool when finished
connection.release();
});
request.on('row', function onRequestRow(columns) {
console.log('value: ' + columns[0].value);
});
connection.execSql(request);
});
pool.acguire takes function as the argument and this function has particular signature (err,connection)
My question is - how do I pass SQL statement inside this function?
I cannot change signature because different function signature is not called.
Also I cannot use global scope because variable may be changed outside.
In other words I need to find way to bypass wrappers calls and still pass some data.
Something like
var mySQLs = ['select 1;select 2;select 3','select 4;select 5;'];
async.forEach(mySQLs,WorkWithOneItem, AllIsDone);
function WorkWithOneItem(item, callback){
pool.acquire(?????(item));
callback(); // tell async that the iterator has completed
}
function AllIsDone (err) {
console.log('All done');
}
By wrapping it in another function:
function aquire(sql, callback) {
pool.acquire(function poolAcquire(err, connection) {
if (err) { console.error(err); return callback(); }
//use the connection as normal
var request = new Request(sql, function requestCallback (err, rowCount) {
if (err) { console.error(err); return;}
console.log('rowCount: ' + rowCount);
//release the connection back to the pool when finished
connection.release();
callback();
});
request.on('row', function onRequestRow(columns) {
console.log('value: ' + columns[0].value);
});
connection.execSql(request);
});
}
function WorkWithOneItem(item, callback){
acquire(item, () => {
callback(); // tell async that the iterator has completed
});
}
Do you need the results out as well?
var mySQLs = ['select 1;select 2;select 3','select 4;select 5;'];
async.forEach(mySQLs, WorkWithOneItem, AllIsDone);
function WorkWithOneItem(sql, callback){
pool.acquire(function poolAcquire(err, connection) {
if (err) return callback(err);
//use the connection as normal
var request = new Request(sql, function requestCallback (err, rowCount) {
if (err) return callback(err);
console.log('rowCount: ' + rowCount);
//release the connection back to the pool when finished
connection.release();
});
var rows = [];
var count = 0;
request.on('row', function onRequestRow(columns) {
console.log('value: ' + columns[0].value);
rows.push(columns[0].value); // or whatever you actually want out of these.
});
request.on('done', function onRequestDone() {
callback(null, rows);
});
connection.execSql(request);
});
callback(); // tell async that the iterator has completed
}
function AllIsDone (err) {
console.log('All done');
// you probably want async.map, so you can get the result back
// as the second argument for a function like this
}

Convert to using promises in DB wrapper module function with Bluebird

I anticipate a callback hell is beginning to form in my code so I decided to start using promises. But I can't wrap my head around implementing it. For example I have a function:
DB.prototype = {
findUser: function (username) {
this._pool.getConnection(function (err, connection) {
if (err) { return callback(true, false); }
connection.query('SELECT password_hash, password_salt FROM users WHERE email = ? AND admin = 1', username,
function (err, rows) {
connection.release();
if (rows.length === 1) { callback(false, rows[0]); }
else { callback(false, false); }
});
connection.on('error', function () { callback(true, false); });
});
}
};
How would I adapt this to using promises instead of callbacks? And how would I use this adapted db.findUser() ?
EDIT:
I got something working. It looks like this:
DB.prototype = {
getConnection: function() {
return this._pool.getConnectionAsync();
}
}
And the usage:
Promise.using(db.getConnection(), function(connection) {
return connection.queryAsync("SELECT password_hash, password_salt FROM users WHERE email = ? AND admin = 1", "exampleUser")
.then(function(rows) {
connection.release();
console.log("is there a row?", rows.length === 1, rows);
// do something with results
});
}).catch(function(err) {
// This is only run if an error is thrown
console.log("error is", err);
});
Is this a good implementation or could something be improved?

NodeJS Express Becomes Unresponsive and Getting Timeouts

I have a feeling I have some kind of leak somewhere, but I'm not sure how to identify or troubleshoot. I'm using express and the request module for nodejs. Under load, https calls made by request to the Facebook Graph API start experiencing long delays or time out altogether. At first I thought it was a throttling issue on the receiving side (Facebook), but if I make a simple C# console application that calls the same URLs several hundred times, none of the response times are greater than 150ms. However, the same code in node varies from 50ms to up to 10s. If I set the timeout property when using request, I start getting ESOCKETTIMEDOUT errors. If I set pool.maxsize on the request options to 100, then I get ETIMEDOUT errors instead.
How can I figure out where my hang up is occurring?
Here's a sample of my use of the request module. I also tried adding this to my app:
require('http').globalAgent.maxSockets = Infinity;
require('https').globalAgent.maxSockets = Infinity;
var executeGetUrl = function getUrl(url, cacheKey, parseJson, accessToken, callback) {
if (accessToken) {
url = url + '?access_token=' + accessToken;
}
try {
request({url: url, timeout: 10000, 'pool.maxSockets' : 100}, function (err, res, body) {
if (err) {
logger.error('Made facebook api call to ' + url + ' with error ' + err);
callback(err, null);
return;
}
if (parseJson) {
try {
body = JSON.parse(body);
} catch (err) {
callback(new Error('Could not parse ' + res.body + ': ' + err), null);
return;
}
if (body.error) {
err = new Error('Error calling ' + url + '. ' + body.error.message);
err.code = body.error.code;
callback(err, null);
}
else {
callback(null, body || {});
}
} else {
if (res.statusCode != 200) {
callback(new Error(res.body));
} else {
callback(null, res.body || {});
}
}
});
} catch (err) {
callback(null, err);
}
};
I'm memoizing the results for 60 seconds, but that doesn't seem to have any relation to the problem
var getUrl = memoize(executeGetUrl, {async: true, maxAge: 60000, length: 2});
And using the function in a promise chain
var deferred = q.defer();
getUrl(photosUrl, null, false, null, function (err, body) {
if (err) {
deferred.reject(err);
return;
}
deferred.resolve(body);
});
return deferred.promise;

Node.js Async | insert into postgresql database results from api

I am quite newbie with node.js. What i am trying to achieve is the following:
Connect to my postgresql database and get info of a place (id, coordinates).
call a weather api and get the info of that spot using the coordinates obtained in the previous step.
Insert the returned json in the database. I get 8 hourly objects, with the weather info every 3 hours (0,3,6,9,12,15,18,21). I need to iterate through this objects and the store them in 8 records in the database.
I wrote the following code:
app.get('/getapi', function(req, res){
var json_bbdd;
//------------ BBDD CONNECTION----------------
var pg = require('pg');
var conString = "postgres://postgres:postgres2#localhost/places";
var client = new pg.Client(conString);
client.connect(function(err) {
if(err) {
console.log('could not connect to postgres');
}
client.query('SELECT * from places where id=3276', function(err, result) {
if(err) {
console.log('error running query');
}
json_bbdd=result.rows[0];
var coords = JSON.parse(json_bbdd.json).coordinates;
var id = json_bbdd.id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: '',
callback: 'MarineWeatherCallback'
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" + input.query + "&format=" + input.format + "&fx=" + input.fx + "&key=" + _PremiumApiKey + "&tide=yes";
$.ajax({
type: 'GET',
url: url,
async: false,
contentType: "application/json",
dataType: 'jsonp',
success: function (json) {
var date= json.data.weather[0].date;
for (var i=0; i < 8; i++){
var hourly = json.data.weather[0].hourly[i];
var time= hourly.time;
client.query('INSERT into parte (id, date, time) VALUES($1, $2, $3)', [id, date, time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' ' + time);
}
});
} // FOR
},
error: function (e) {
console.log(e.message);
}
});
client.end();
});
});
});
The steps 1 and 2 are performed perfectly. The third step, on the other hand, does nothing and it doesn't even throw an error.
I read in this post: node-postgres will not insert data, but doesn't throw errors either that using async module could help but i have no idea how to rewrite the code. I need some help.
Regards,
Aitor
I didn't test your snippet, I can only help you with things which looks bad to my eyes.
It is better not to use jQuery on node server. There is excellent library called request to do remote http requests.
You should better handle database errors because in your example your code will continue after DB error.
You are calling client.end() too early and at the time when you try to insert data to the database a connection is already closed. You have to move client.end() at the end of success and error functions and wait to all callbacks are done.
I think it is also better to use connection pool instead of Client.
You can possibly use JSON type in PostgreSQL to avoid serializing/deserializing JSON data in your code.
Here is revised example(untested). I didn't replace jQuery here, some minor tweaking included.
var pg = require('pg');
var conString = "postgres://postgres:postgres2#localhost/places";
app.get('/getapi', function(req, res, next){
var json_bbdd;
//------------ BBDD CONNECTION----------------
pg.connect(conString, function(err, client, done) {
if(err) {
// example how can you handle errors
console.error('could not connect to postgres');
return next(new Error('Database error'));
}
client.query('SELECT * from places where id=3276', function(err, result) {
if(err) {
console.error('error running query');
done();
return next(new Error('Database error'));
}
json_bbdd = result.rows[0];
var coords = JSON.parse(json_bbdd.json).coordinates;
var id = json_bbdd.id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: '',
callback: 'MarineWeatherCallback'
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" +
input.query + "&format=" + input.format +
"&fx=" + input.fx + "&key=" +
_PremiumApiKey + "&tide=yes";
$.ajax({
type: 'GET',
url: url,
async: false,
contentType: "application/json",
dataType: 'jsonp',
success: function (json) {
var date = json.data.weather[0].date;
var callbacks = 0;
for (var i=0; i < 8; i++) {
var hourly = json.data.weather[0].hourly[i];
var time= hourly.time;
client.query(
'INSERT into parte (id, date, time) VALUES($1, $2, $3)',
[id, date, time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' ' + time);
}
callbacks++;
if (callbacks === 8) {
console.log('All callbacks done!');
done(); // done(); is rough equivalent of client.end();
}
});
} // FOR
},
error: function (e) {
console.error(e.message);
done(); // done(); is rough equivalent of client.end();
return next(new Error('Http error'));
}
});
});
});
});
Ok, now cam up with another problem...i was doubting of creating a new post but i think that maybe could have relation with the previous post.
The aim is to read from the database instead of one place 3 places and do the same process than before for each one.
The code is as follows (with the changes proposed by ivoszz):
app.get('/getapi', function(req, res, next){
//------------ BBDD CONNECTION----------------
pg.connect(conString, function(err, client, done) {
if(err) {
// example how can you handle errors
console.error('could not connect to postgres',err);
return next(new Error('Database error'));
}
client.query('SELECT * from places where id>3274 and id<3278', function(err, result) {
if(err) {
console.error('error running query',err);
done();
return next(new Error('Database error'));
}
var first_callback = 0;
for (var y=0; y<result.rows.length; y++) {
var coords = JSON.parse(result.rows[y].json).coordinates;
var id = result.rows[y].id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: ''
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" + input.query + "&format=" + input.format + "&fx=" + input.fx + "&key=" + _PremiumApiKey;
request(url, function(err, resp, body) {
body = JSON.parse(body);
if (!err && resp.statusCode == 200) {
var date = body.data.weather[0].date;
var callbacks = 0;
for (var i=0; i < 8; i++) {
var hourly = body.data.weather[0].hourly[i];
client.query(
'INSERT into parte (id, date, time) VALUES($1, $2, $3)',
[id, date, hourly.time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' iteration ' + i);
}
callbacks++;
if (callbacks === 8) {
console.log('All callbacks done!from id '+id);
//done(); // done(); is rough equivalent of client.end();
//res.send("done");
}
});
} // FOR
}
else { // if the API http request throws an error
console.error(err);
done(); // done(); is rough equivalent of client.end();
return next(new Error('Http API error'));
}
}); // REQUEST API URL
first_callback++;
if (first_callback === result.rows.length-1) {
console.log('All global callbacks done!');
done(); // done(); is rough equivalent of client.end();
res.send("done");
}}
}); // SELECT from pg
}); // CONNECT to pg
}); // app.get
I don't know why it tries to insert the id=3277 three times instead of inserting id=3275, id=3276 and then id=3277... what it does instead is: it inserts the first 8 records ok the first time (id=3277), but then it throws an error saying that the records are already inserted (primary key=id,date,time) with id 3277...
It seems that first does the 3 iterations of the first FOR and then does the 3 iteration of the second FOR but with the info of the last iteration(place). I can't understand it very well...

Resources