I have a feeling I have some kind of leak somewhere, but I'm not sure how to identify or troubleshoot. I'm using express and the request module for nodejs. Under load, https calls made by request to the Facebook Graph API start experiencing long delays or time out altogether. At first I thought it was a throttling issue on the receiving side (Facebook), but if I make a simple C# console application that calls the same URLs several hundred times, none of the response times are greater than 150ms. However, the same code in node varies from 50ms to up to 10s. If I set the timeout property when using request, I start getting ESOCKETTIMEDOUT errors. If I set pool.maxsize on the request options to 100, then I get ETIMEDOUT errors instead.
How can I figure out where my hang up is occurring?
Here's a sample of my use of the request module. I also tried adding this to my app:
require('http').globalAgent.maxSockets = Infinity;
require('https').globalAgent.maxSockets = Infinity;
var executeGetUrl = function getUrl(url, cacheKey, parseJson, accessToken, callback) {
if (accessToken) {
url = url + '?access_token=' + accessToken;
}
try {
request({url: url, timeout: 10000, 'pool.maxSockets' : 100}, function (err, res, body) {
if (err) {
logger.error('Made facebook api call to ' + url + ' with error ' + err);
callback(err, null);
return;
}
if (parseJson) {
try {
body = JSON.parse(body);
} catch (err) {
callback(new Error('Could not parse ' + res.body + ': ' + err), null);
return;
}
if (body.error) {
err = new Error('Error calling ' + url + '. ' + body.error.message);
err.code = body.error.code;
callback(err, null);
}
else {
callback(null, body || {});
}
} else {
if (res.statusCode != 200) {
callback(new Error(res.body));
} else {
callback(null, res.body || {});
}
}
});
} catch (err) {
callback(null, err);
}
};
I'm memoizing the results for 60 seconds, but that doesn't seem to have any relation to the problem
var getUrl = memoize(executeGetUrl, {async: true, maxAge: 60000, length: 2});
And using the function in a promise chain
var deferred = q.defer();
getUrl(photosUrl, null, false, null, function (err, body) {
if (err) {
deferred.reject(err);
return;
}
deferred.resolve(body);
});
return deferred.promise;
Related
I have a function in a utils file that I want to call and assign the exported result to a variable.
Currently, the variable is defined then I try to assign the return result but I am getting undefined as the result when I console.log it.
Here is my utils/consul file
var consul = require("consul")({host: config.consul.host'});
var consulBase = [];
var options;
module.exports = {
consulQuery: function(service){
consul.catalog.service.nodes(service, function(err, results) {
if(err) {console.log(err); throw err;}
if(results.length <= 0) return {message: `Error could not find any service of ${service} registered with consul,`, errorCode: 500};
if(results.length > 0) consulBase = [];
results.forEach((result) => {
consulBase.push(result.ServiceAddress+ ':' +result.ServicePort);
});
var serviceURL = 'http://' + consulBase[Math.floor(Math.random()*consulBase.length)];
return options = {
baseUrl : serviceURL,
form: {'':''},
headers: {authorization: ''}
};
});
}
Then in another file, I am calling like this and then trying to assign the value to 'options' but am getting undefined.
var consulQuery = require("../utils/consul").consulQuery;
// Get options array right away
var options = consulQuery('auth');
// Get options array every 5 seconds
setInterval(() => {
options = consulQuery('auth');
console.log(options);
}, 5 * 1000);
OK you have a couple issues.
First, is conceptual about what you are trying to do. Second is what do you actually need to change in your code to make it work.
I will not talk about the first part because, and there are plenty of good resources to learn about async with examples better then I can do here.
For the actual problems with your code:
You are missing a callback for consulQuery()
It should be something like this (notice the cb i added):
module.exports = {
consulQuery: function (service, cb) {
consul.catalog.service.nodes(service, function (err, results) {
if (err) {
console.log(err);
cb(err, null)
throw err;
}
if (results.length <= 0) return {
message: `Error could not find any service of ${service} registered with consul,`,
errorCode: 500
};
if (results.length > 0) consulBase = [];
results.forEach((result) => {
consulBase.push(result.ServiceAddress + ':' + result.ServicePort);
});
var serviceURL = 'http://' + consulBase[Math.floor(Math.random() * consulBase.length)];
cb(null, {
baseUrl: serviceURL,
form: {'': ''},
headers: {authorization: ''}
});
});
}
}
Second, in the other file in which you invoke the function, you will have to now pass a callback function.
options = consulQuery('auth', (err, response) => {
if(err){
console.log(err)
}
console.log(response)
});
My code is too long to post in here, but basically I use socket.io server to pull data from database and refresh in the client every 1 second.
Like this:
function updateTimer(){
//db->query
io.sockets.emit('updated data', data);
}
setInterval(updateTimer, 1000);
After a certain amount of time, the server just stops emitting data. I use a chat on the website as well and it stops too. But in the server console nothing is shown, no errors or any outputs, just "listening on port 3000..." stays on the screen all the time.
I thought it could be the loop maybe but I think there's no other way of refreshing data every 1 second, am I right?
If someone can help me and needs the full code please open a discussion and I'll paste it somewhere.
EDIT for the function code:
function checkRoulleteTime() {
try {
pool.getConnection(function(err, connection) {
connection.query('SELECT * FROM `roullete` WHERE status=\'active\'', function(err, rows) {
if (err) {
console.log(err);
return;
}
if (rows.length == 0) return;
var time = rows[0].time - (Math.floor(Date.now() / 1000));
if (time <= 1) {
connection.query('UPDATE `roullete` SET `status`=\'closed\' WHERE `id`=\'' + rows[0].id + '\'', function(error, fields) {
if (error) throw error;
});
setTimeout(roll, 1000);
setTimeout(function(){
io.sockets.emit('add hist');
}, 10500);
setTimeout(updatePoints, 12000);
setTimeout(newRound, 12500);
}
var contagem = Object.keys(clients).length;
io.sockets.emit('login count', contagem);
io.sockets.emit('roullete time', time);
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'black\'',function(error2, rows2){
if (error2) throw error2;
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'green\'',function(error4, rows4){
if (error4) throw error4;
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'red\'',function(error3, rows3){
if (error3) throw error3;
var onBlack = rows2[0].points;
var onRed = rows3[0].points;
var onGreen = rows4[0].points;
io.sockets.emit('calculations', {"time": time, "black" : onBlack, "red" : onRed, "green" : onGreen});
});
});
});
});
connection.release();
});
} catch(e) {
console.log('error here:'+e);
}
setTimeout(checkRoulleteTime, 1000);
}
setTimeout(checkRoulleteTime, 1000);
Alright, that's my code.
My guess is that you have some sort of programming error in your database query that, after a little while exhausts some resources so your database query starts failing every time, throwing an exception or just returning an error and thus you don't ever send any data because of the error. Because the code is inside the setInterval() callback, any exception is not logged for you.
We could probably help you further if you included your actual database code, but you can start to debug it yourself by putting an exception handler around it like this:
function updateTimer(){
try {
//db->query
io.sockets.emit('updated data', data);
} catch(e) {
console.log(e);
}
}
setInterval(updateTimer, 1000);
And, if your DB query is async (which I'm assuming it is), you will need to have explicit error handling and an exception at each callback level (since exceptions don't propagate up async callbacks).
If your database may get slow, then it might be safer to change your recurring code to work like this:
function updateTimer(){
try {
//db->query
io.sockets.emit('updated data', data);
} catch(e) {
console.log(e);
}
// schedule next updateTimer() call when this one has finished
setTimeout(updateTimer, 1000);
}
// schedule first updateTimer() call
setTimeout(updateTimer, 1000);
You have LOTS of places in your code where you are leaking a pooled connection and lots of places where you are not logging an error. My guess is that you are running out of pooled connections, getting an error over and over that you don't log.
Here's a version of your code that attempts to clean things up so all errors are logged and no pooled connections are leaked. Personally, I would write this code using promises which makes robust error handling and reporting a ton easier. But, here's a modified version of your code:
function checkRoulleteTime() {
try {
pool.getConnection(function (err, connection) {
if (err) {
console.log("Failed on pool.getConnection()", err);
return;
}
connection.query('SELECT * FROM `roullete` WHERE status=\'active\'', function (err, rows) {
if (err) {
connection.release();
console.log(err);
return;
}
if (rows.length == 0) {
connection.release();
return;
}
var time = rows[0].time - (Math.floor(Date.now() / 1000));
if (time <= 1) {
connection.query('UPDATE `roullete` SET `status`=\'closed\' WHERE `id`=\'' + rows[0].id + '\'', function (error, fields) {
if (error) {
console.log(error);
connection.release();
return;
}
});
setTimeout(roll, 1000);
setTimeout(function () {
io.sockets.emit('add hist');
}, 10500);
setTimeout(updatePoints, 12000);
setTimeout(newRound, 12500);
}
var contagem = Object.keys(clients).length;
io.sockets.emit('login count', contagem);
io.sockets.emit('roullete time', time);
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'black\'', function (error2, rows2) {
if (error2) {
console.log(error2);
connection.release();
return;
}
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'green\', function (error4, rows4) {
if (error4) {
console.log(error4);
connection.release();
return;
}
connection.query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'red\'', function (error3, rows3) {
connection.release();
if (error3) {
console.log(error3);
return;
}
var onBlack = rows2[0].points;
var onRed = rows3[0].points;
var onGreen = rows4[0].points;
io.sockets.emit('calculations', {
"time": time,
"black": onBlack,
"red": onRed,
"green": onGreen
});
});
});
});
});
});
} catch (e) {
console.log('error here:' + e);
}
setTimeout(checkRoulleteTime, 1000);
}
setTimeout(checkRoulleteTime, 1000);
And to give you an idea how much simpler it can be to do error handling and chaining of sequential or dependent async functions, here's your function rewritten using promises. I have no idea if this runs without error since I have no way of testing it, but it should give you an idea how much cleaner programming with promises can be:
var Promise = require('bluebird');
pool = Promise.promisifyAll(pool);
function logErr(err) {
console.log(err);
}
function checkRoulleteTime() {
pool.getConnectionAsync().then(function(connection) {
var query = Promise.promisify(connection.query, {context: connection});
return query('SELECT * FROM `roullete` WHERE status=\'active\'').then(function(rows) {
if (rows.length !== 0) {
var time = rows[0].time - (Math.floor(Date.now() / 1000));
if (time <= 1) {
query('UPDATE `roullete` SET `status`=\'closed\' WHERE `id`=\'' + rows[0].id + '\'').catch(logErr);
setTimeout(roll, 1000);
setTimeout(function () {
io.sockets.emit('add hist');
}, 10500);
setTimeout(updatePoints, 12000);
setTimeout(newRound, 12500);
}
var contagem = Object.keys(clients).length;
io.sockets.emit('login count', contagem);
io.sockets.emit('roullete time', time);
function doQuery(color) {
return query('SELECT SUM(points) as points FROM `roullete_bets` WHERE `round`=\'' + rows[0].id + '\' AND `color`=\'' + color + '\'');
}
return Promise.all([doQuery('black'), doQuery('green'), doQuery('red')]).then(function(results) {
io.sockets.emit('calculations', {
"time": time,
"black": results[0][0].points,
"green": results[1][0].points,
"red": results[2][0].points
});
});
}
}).catch(logErr).finally(function() {
connection.release();
setTimeout(checkRoulleteTime, 1000);
});
}, function(err) {
console.log("Err getting connection: ", err);
});
}
OK I have a NodeJS app and I'm trying to download lots of images from a web server (about 500 for now but the number will increase). The problem I get is a "Unhandled stream error in pipe Error: EMFILE" because it seems that too much files get opened at the same time.
So I'm trying to use async.queue to process files by batches of 20. But I still get the error.
SomeModel.find({}, function(err, photos){
if (err) {
console.log(err);
}
else {
photos.forEach(function(photo){
var url = photo.PhotoURL;
var image = url.replace('http://someurl.com/media.ashx?id=', '').replace('&otherstuffattheend', '.jpg');
photo.target = image;
var q = async.queue(function (task) {
request
.get(task.PhotoURL)
.on('response', function(response) {
console.log(task.PhotoURL + ' : ' + response.statusCode, response.headers['content-type']);
console.log(task.target);
})
.on('error', function(err) {
console.log(err);
})
.pipe(fs.createWriteStream(task.target));
}, 20);
q.push(photo, function(err) {
if (err) {
console.log(err);
}
});
q.drain = function() {
console.log('Done.')
}
});
}
});
What am I doing wrong ? Many thanks for your time and help.
The problem is that you're creating a new queue for each photo and each queue receives just one photo. Instead, only create a queue once (outside of the forEach()) and push the photo objects to it. You're also missing the callback in your task handler. For example:
var q = async.queue(function(task, cb) {
request
.get(task.PhotoURL)
.on('response', function(response) {
console.log(task.PhotoURL + ' : ' + response.statusCode, response.headers['content-type']);
console.log(task.target);
// the call to `cb` could instead be made on the file stream's `finish` event
// if you want to wait until it all gets flushed to disk before consuming the
// next task in the queue
cb();
})
.on('error', function(err) {
console.log(err);
cb(err);
})
.pipe(fs.createWriteStream(task.target));
}, 20);
q.drain = function() {
console.log('Done.')
};
photos.forEach(function(photo) {
var url = photo.PhotoURL;
var image = url.replace('http://someurl.com/media.ashx?id=', '').replace('&otherstuffattheend', '.jpg');
photo.target = image;
q.push(photo, function(err) {
if (err) {
console.log(err);
}
});
});
Having a problem with making a lot of requests with method 'HEAD'.
I've made async.queue set to 20, and timeout to 3000ms.
Anyway when i run:
I see 10-15 success, than some timouts with some more success, an hangs... nothing happening further.
If i remove timeout i have about 10 success and hang...
And i dont get the error message neither.
The Code of request:
function getHeader(link)
{
var correctUrl = url.parse(link);
var options = {method: 'HEAD', host: correctUrl.hostname, port: 80, path: correctUrl.pathname};
var req = http.request(options, function(res) {
if(res.statusCode == 404 || res.statusCode == 500) return;
var x = {
loc : link
};
if(typeof(res.headers['last-modified']) != "undefined")
{
x.lastmod = dateConverter(res.headers['last-modified']);
console.log("Added lastmodify: " + x.lastmod);
}
console.log(res.headers);
parser.allObjects.push(x);
});
req.setTimeout(3000, function() {
console.log("Timeout reached. Link:" + link);
req.abort();
});
req.on('error', function (e) {
console.log('problem with request: ' + e.message);
});
req.end();
}
And the queue is here:
var queue = async.queue(function (href, callback) {
getHeader(href,function(err){
if(err) return callback(err);
return callback();
});
}, parser.serverMight); // this set to 20 at the mom (decreased from 50)
queue.drain = function() {
formXml(null, parser.allObjects);
};
queue.push(toRequest, function(err) {
if(err) console.log(err);
});
Any help is highly appreciated, thanks.
Heh, found myself. Maybe this may help someone.
So the mistake was very simple:
I didn't callback from the getHeader function, i just used return. That's why the queue couldn't start the next round.
Httpreq takes less space, so i i'll let it stay.
Here is how the correct code look:
function getHeader(link, callback)
{
httpreq.get(link, function(err, res) {
if(err) return callback(err);
if(res.statusCode == 404 || res.statusCode == 500)
{
parser.allHrefs.remove(parser.allHrefs.indexOf(link));
console.log("Faced status code 404 || 500. url deleted: " + link);
return callback(null);
}
//collect header-info
var x = { loc : link };
if(typeof(res.headers['last-modified']) != "undefined")
x.lastmod = dateConverter(res.headers['last-modified']);
console.log("Success adding header:" + x.loc);
parser.allObjects.push(x);
return callback(null);
});
}
p.s.: somewhy the 'httpreq' (requesting full request body) is making this faster, than 'http' (requesting HEAD)...
I am quite newbie with node.js. What i am trying to achieve is the following:
Connect to my postgresql database and get info of a place (id, coordinates).
call a weather api and get the info of that spot using the coordinates obtained in the previous step.
Insert the returned json in the database. I get 8 hourly objects, with the weather info every 3 hours (0,3,6,9,12,15,18,21). I need to iterate through this objects and the store them in 8 records in the database.
I wrote the following code:
app.get('/getapi', function(req, res){
var json_bbdd;
//------------ BBDD CONNECTION----------------
var pg = require('pg');
var conString = "postgres://postgres:postgres2#localhost/places";
var client = new pg.Client(conString);
client.connect(function(err) {
if(err) {
console.log('could not connect to postgres');
}
client.query('SELECT * from places where id=3276', function(err, result) {
if(err) {
console.log('error running query');
}
json_bbdd=result.rows[0];
var coords = JSON.parse(json_bbdd.json).coordinates;
var id = json_bbdd.id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: '',
callback: 'MarineWeatherCallback'
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" + input.query + "&format=" + input.format + "&fx=" + input.fx + "&key=" + _PremiumApiKey + "&tide=yes";
$.ajax({
type: 'GET',
url: url,
async: false,
contentType: "application/json",
dataType: 'jsonp',
success: function (json) {
var date= json.data.weather[0].date;
for (var i=0; i < 8; i++){
var hourly = json.data.weather[0].hourly[i];
var time= hourly.time;
client.query('INSERT into parte (id, date, time) VALUES($1, $2, $3)', [id, date, time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' ' + time);
}
});
} // FOR
},
error: function (e) {
console.log(e.message);
}
});
client.end();
});
});
});
The steps 1 and 2 are performed perfectly. The third step, on the other hand, does nothing and it doesn't even throw an error.
I read in this post: node-postgres will not insert data, but doesn't throw errors either that using async module could help but i have no idea how to rewrite the code. I need some help.
Regards,
Aitor
I didn't test your snippet, I can only help you with things which looks bad to my eyes.
It is better not to use jQuery on node server. There is excellent library called request to do remote http requests.
You should better handle database errors because in your example your code will continue after DB error.
You are calling client.end() too early and at the time when you try to insert data to the database a connection is already closed. You have to move client.end() at the end of success and error functions and wait to all callbacks are done.
I think it is also better to use connection pool instead of Client.
You can possibly use JSON type in PostgreSQL to avoid serializing/deserializing JSON data in your code.
Here is revised example(untested). I didn't replace jQuery here, some minor tweaking included.
var pg = require('pg');
var conString = "postgres://postgres:postgres2#localhost/places";
app.get('/getapi', function(req, res, next){
var json_bbdd;
//------------ BBDD CONNECTION----------------
pg.connect(conString, function(err, client, done) {
if(err) {
// example how can you handle errors
console.error('could not connect to postgres');
return next(new Error('Database error'));
}
client.query('SELECT * from places where id=3276', function(err, result) {
if(err) {
console.error('error running query');
done();
return next(new Error('Database error'));
}
json_bbdd = result.rows[0];
var coords = JSON.parse(json_bbdd.json).coordinates;
var id = json_bbdd.id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: '',
callback: 'MarineWeatherCallback'
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" +
input.query + "&format=" + input.format +
"&fx=" + input.fx + "&key=" +
_PremiumApiKey + "&tide=yes";
$.ajax({
type: 'GET',
url: url,
async: false,
contentType: "application/json",
dataType: 'jsonp',
success: function (json) {
var date = json.data.weather[0].date;
var callbacks = 0;
for (var i=0; i < 8; i++) {
var hourly = json.data.weather[0].hourly[i];
var time= hourly.time;
client.query(
'INSERT into parte (id, date, time) VALUES($1, $2, $3)',
[id, date, time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' ' + time);
}
callbacks++;
if (callbacks === 8) {
console.log('All callbacks done!');
done(); // done(); is rough equivalent of client.end();
}
});
} // FOR
},
error: function (e) {
console.error(e.message);
done(); // done(); is rough equivalent of client.end();
return next(new Error('Http error'));
}
});
});
});
});
Ok, now cam up with another problem...i was doubting of creating a new post but i think that maybe could have relation with the previous post.
The aim is to read from the database instead of one place 3 places and do the same process than before for each one.
The code is as follows (with the changes proposed by ivoszz):
app.get('/getapi', function(req, res, next){
//------------ BBDD CONNECTION----------------
pg.connect(conString, function(err, client, done) {
if(err) {
// example how can you handle errors
console.error('could not connect to postgres',err);
return next(new Error('Database error'));
}
client.query('SELECT * from places where id>3274 and id<3278', function(err, result) {
if(err) {
console.error('error running query',err);
done();
return next(new Error('Database error'));
}
var first_callback = 0;
for (var y=0; y<result.rows.length; y++) {
var coords = JSON.parse(result.rows[y].json).coordinates;
var id = result.rows[y].id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: ''
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" + input.query + "&format=" + input.format + "&fx=" + input.fx + "&key=" + _PremiumApiKey;
request(url, function(err, resp, body) {
body = JSON.parse(body);
if (!err && resp.statusCode == 200) {
var date = body.data.weather[0].date;
var callbacks = 0;
for (var i=0; i < 8; i++) {
var hourly = body.data.weather[0].hourly[i];
client.query(
'INSERT into parte (id, date, time) VALUES($1, $2, $3)',
[id, date, hourly.time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' iteration ' + i);
}
callbacks++;
if (callbacks === 8) {
console.log('All callbacks done!from id '+id);
//done(); // done(); is rough equivalent of client.end();
//res.send("done");
}
});
} // FOR
}
else { // if the API http request throws an error
console.error(err);
done(); // done(); is rough equivalent of client.end();
return next(new Error('Http API error'));
}
}); // REQUEST API URL
first_callback++;
if (first_callback === result.rows.length-1) {
console.log('All global callbacks done!');
done(); // done(); is rough equivalent of client.end();
res.send("done");
}}
}); // SELECT from pg
}); // CONNECT to pg
}); // app.get
I don't know why it tries to insert the id=3277 three times instead of inserting id=3275, id=3276 and then id=3277... what it does instead is: it inserts the first 8 records ok the first time (id=3277), but then it throws an error saying that the records are already inserted (primary key=id,date,time) with id 3277...
It seems that first does the 3 iterations of the first FOR and then does the 3 iteration of the second FOR but with the info of the last iteration(place). I can't understand it very well...