Event-driven asynchronous callbacks in Node js - node.js

Is there a way to block the asynchronous callback property of node.js?
Please Advice...
For example,
var express = require('express');
var app = express();
var MongoClient = require('mongodb').MongoClient,
format = require('util').format;
var cors = require('cors');
app.get('/gantt', cors(), function (request, response) {
MongoClient.connect('mongodb://127.0.0.1:27017/test', function (err, db) {
if (err) throw err;
var collection = db.collection('ganttdata');
collection.find({}, {
"_id": 0
}).toArray(function (err, results) {
var jsonString = JSON.stringify(results);
response.setHeader('Content-Type', 'text/plain');
response.send('{\"data\":' + jsonString + '}');
});
});
});
app.listen(3000);
console.log('Listening on port 3000...');
Inspite the Node.js prints the console statement first,i want app.get() to be executed.
My scenario is same as that of the above one.
This is my scenario
var ganttresult = new Array();
app.get('/get', cors(), function (request, response) {
console.log('hello');
connection.query("SELECT distinct id FROM ganttdata", function (err, rows) {
if (err) {
console.log('error in fetching ' + err);
} else {
var all_id = rows;
for (var j = 0; j < all_id.length; j++) {
console.log('hello1');
connection.query("SELECT id,tailName FROM ganttdata where id= '" + all_id[j].id + "'", function (err, rows) {
if (err) {
console.log('error in fetching ' + err);
} else {
var jsonString1 = rows;
var set_id = jsonString1[0].id;
connection.query("SELECT item_id,name,start,end FROM ganttdata where id= '" + set_id + "'", function (err, rows) {
if (err) {
console.log('error in fetching ' + err);
} else {
var jsonString2 = rows;
var gantt1 = new Object();
gantt1.id = jsonString1[0].id;
gantt1.tailName = jsonString1[0].tailName;
var series = new Array();
for (var i = 0; i < jsonString2.length; i++) {
var gantt2 = new Object();
gantt2.item = jsonString2[i];
series.push(gantt2);
gantt1.series = series;
}
//console.log(gantt1);
console.log('hi');
ganttresult.push(gantt1);
console.log(ganttresult);
}
});
}
});
}
var result = JSON.stringify(ganttresult);
console.log(result);
response.send('{\"data\":' + result + '}');
response.end();
}
});
});
When I run this code,
I get an empty resultset and when I re-run I get the result.
I guess it is due to asynchronous callback nature of node js.
Please advice...
Thanks
I have tried async.waterfall method as given below
app.get('/get',cors(), function(request,response) {
async.waterfall([
function(result) {
connection.query("SELECT id FROM Gantt",function(err, rows) {
if (err) {
console.log('error in fetching ' + err);
}
else{
var all_id=rows;
for(var j=0;j<all_id.length;j++){
connection.query("SELECT id,tailName FROM Gantt where id= '"+all_id[j].id+"'",function(err, rows) {
if (err) {
console.log('error in fetching ' + err);
}
else{
var jsonString1=rows;
var set_id=jsonString1[0].id;
connection.query("SELECT item_id,name,start,end FROM GanttFlight where id= '"+set_id+"'",function(err, rows) {
if (err) {
console.log('error in fetching ' + err);
}
else{
var jsonString2=rows;
var gantt1=new Object();
gantt1.id=jsonString1[0].id;
gantt1.name=jsonString1[0].tailName;
var series = new Array();
series=[];
for(var i=0;i<jsonString2.length;i++){
var gantt2=new Object();
gantt2.item=jsonString2[i];
series.push(gantt2);
gantt1.series=series;
}
ganttresult.push(gantt1);
}
});
}
});
}
var result= JSON.stringify(ganttresult);
console.log(ganttresult);
response.send(ganttresult);
ganttresult=[];
//response.send('{\"data\":'+result+'}');
response.end();
}
});
}
], function(err, status) {
console.log(status);
});
});
app.listen(3000);
console.log('Listening on port 3000...');
i am getting empty result first and when refresh the browser,i get the required result
Please Advice

Related

MongoDB Fetch check if data exists

I am trying to find the best way to write this code. I fetch locations from a remote resource and need to check if there are any new locations present in the data, if there are I should add them to my database, if they are not new I just want to update them.
const http = require('http');
const timeout = 5000; //5 seconds
const MongoClient = require('mongodb').MongoClient;
// Database Name
const dbName = 'weatherApp';
const url = 'mongodb://localhost:27017';
// Connect using MongoClient
MongoClient.connect(url, function(err, client) {
if(err){
console.log(err);
return;
}
const locationsCollection = client.db(dbName).collection('locations');
(function fetchTemperatureLoop(){
console.log('Started http request..');
http.get('remote url..', function(resp){
var data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
if(data.isJson()){
var locations = JSON.parse(data).toArray();
(function locationsLoop(){
var location = locations.pop();
locationsCollection.findOne({location: location.location}, function(err, result){
if(err){
console.log(err);
return;
}
if(result){
//Exists
var measurements = result.measurements;
measurements.push({timestamp: +new Date, temperature: location.temperature})
locationsCollection.update({location: location.location}, {$set: {measurements: measurements}}, function(err){
if(err){
console.log(err);
return;
}
console.log('Added new temperature for location: ' + location.location);
continueLocationsLoop();
});
}else{
//Doesnt exist
location.measurements = [];
location.measurements.push({timestamp: +new Date, temperature: location.temperature});
locationsCollection.insert(location, function(err){
if(err){
console.log(err);
return;
}
console.log('Created new location: ' + location.location);
continueLocationsLoop();
});
}
});
function continueLocationsLoop(){
if(locations.length){
locationsLoop()
}else{
setTimeout(fetchTemperatureLoop, timeout);
}
}
})();
}
});
}).on("error", (err) => {
console.log("Error: " + err.message);
console.log("Continue anyways..");
setTimeout(fetchTemperatureLoop, timeout);
});
})();
});
String.prototype.isJson = function(){
try{
JSON.parse(this);
}catch(e){
return false;
}
return true;
}
Object.prototype.toArray = function(){
var arr = [];
for(var key in this){
if(this.hasOwnProperty(key)){
arr.push(this[key]);
}
}
return arr;
}
I really want to avoid using so many closures but I dont want to repeat myself either. Any help rewriting this code in an optimal way is much appriciated.
My main problem was illiterating through the locations and doing the calls to the database.

async.eachSeries runs only once with async.waterfall inside for each iteration

I am new to async library. I have used async.eachSeries and async.waterfall for each iteration. I see, the async.waterfall runs only once.
Here is my code :
var fs = require('fs'),
async = require('async'),
Client = require('node-rest-client').Client;
// REST API Call and output in jsonOutput.results
console.log(jsonOutput.results.length); // jsonOutput.results has 124 records.
async.eachSeries(jsonOutput.results, function(account, callback) {
var dataObject = {};
dataObject.updatetime = new Date();
var setAccountInfoURL = ""; // Data Update REST API Request
async.waterfall([
function setAccountInfo(updateCallback) {
// client.get(setAccountInfoURL, function (data, response) {
// var jsonOutput = JSON.parse(data.toString('utf8'));
updateCallback(null, "output", account)
// });
},
function saveAccountInfo(jsonOutput, account, updateCallback) {
var debuglog = JSON.stringify(account) + "\n" + jsonOutput;
fs.appendFile("debuginfo.json", debuglog + "\n", function (err) {
if(err) {
console.log(err);
}
console.log("JSON saved to " + "debuginfo.json");
updateCallback(null);
});
}
],function asyncComplete(err) {
if (err) {
console.warn('Error setting account info.', err);
}
console.log('async completed');
});
}, function(err){
if (err) {
console.log('error in loop');
}
console.log('loop completed');
});
Output:
124
JSON saved to debuginfo.json
async completed
Any help is really appreciated.
I found my mistake. I missed calling the callback after each iteration just after async is completed.
var fs = require('fs'),
async = require('async'),
Client = require('node-rest-client').Client;
// REST API Call and output in jsonOutput.results
console.log(jsonOutput.results.length); // jsonOutput.results has 124 records.
async.eachSeries(jsonOutput.results, function(account, callback) {
var dataObject = {};
dataObject.updatetime = new Date();
var setAccountInfoURL = ""; // Data Update REST API Request
async.waterfall([
function setAccountInfo(updateCallback) {
// client.get(setAccountInfoURL, function (data, response) {
// var jsonOutput = JSON.parse(data.toString('utf8'));
updateCallback(null, "output", account)
// });
},
function saveAccountInfo(jsonOutput, account, updateCallback) {
var debuglog = JSON.stringify(account) + "\n" + jsonOutput;
fs.appendFile("debuginfo.json", debuglog + "\n", function (err) {
if(err) {
console.log(err);
}
console.log("JSON saved to " + "debuginfo.json");
updateCallback(null);
});
}
],function asyncComplete(err) {
if (err) {
console.warn('Error setting account info.', err);
}
console.log('async completed');
callback(null); // this is the change.
});
}, function(err){
if (err) {
console.log('error in loop');
}
console.log('loop completed');
});

"object is not a function" error during MongoDB document insertion from a CSV stream using async.queue

I'm trying MongoDB document insertion from a CSV stream using async.queue.
But I face this following error. I've tried all the remedies given in similar SO posts.
Exact error message is:
C:\Users\admin\node_modules\mongodb\lib\mongo_client.js:406
throw err
TypeError:object is not a function
at C:\Users\admin\Desktop\mynodefile.js:13:2
at C:\Users\admin\node_modules\mongodb\lib\mongo_client.js:403:11
at process._tickCallback(node.js:355:11)
node.js code I used:
var csv = require('csv');
var async = require('async');
var fs = require('fs');
var MongoClient = require('mongodb').MongoClient;
MongoClient.connect('mongodb://localhost:27017', function(err, db) {
if (err) throw err;
var collection = db.collection('myCSVs');
var queue = async.queue(collection.insert.bind(collection), 5);
csv()
.from.path('./input.csv', { columns: true })
.transform(function (data, index, cb) {
queue.push(data, function (err, res) {
if (err) return cb(err);
cb(null, res[0]);
});
})
.on('error', function (err) {
console.log('ERROR: ' + err.message);
})
.on('end', function () {
queue.drain = function() {
collection.count(function(err, count) {
console.log('Number of documents:', count);
db.close();
});
};
});
});
You haven't mentioned the database name in your MongoClient.connect function call. You can do so like this:
MongoClient.connect('mongodb://localhost:27017/database_name',function(err, db) {
Then you can do:
var collection = db.collection('myCSVs');
If myCSVs is a collection inside database_name
Or you can also do:
MongoClient.connect('mongodb://localhost:27017',function(err, mongoclient) {
var db = mongoclient.db('database_name');
var collection = db.collection('myCSVs');
});
You have to change
var queue = async.queue(collection.insert.bind(collection), 5);
Into:
var q = async.queue(function (task, callback) {
console.log('hello ' + task.name);
callback();
}, 2);
IN this line :
queue.push(data, function (err, res) {
if (err) return cb(err);
cb(null, res[0]);
});
you are calling push with data and with a callback, but its not implemented in your
var queue = async.queue(collection.insert.bind(collection), 5);

wrap couchbase access function

below is my couchbase nodejs code
kdatabase.js
var couchbase = require('couchbase');
var db = new couchbase.Connection({
host: "http://127.0.0.1:8091",
bucket: "default",
},
function(err) {
if (err) throw err;
db.get('id1', function(err, result) {
if (err) throw err;
console.log(result.value);
process.exit(0);
});
});
it works
but I hope to wrap it to object that can be easily to operate
module.exports = function(app) {
return new KDatabase(app);
};
var KDatabase = function(app) {
this.app = app;
};
//couchbase
KDatabase.prototype.query = function(userName) {
var couchbase = require('couchbase');
var db = new couchbase.Connection({
host: "http://127.0.0.1:8091",
bucket: "default",
},
function(err) {
if (err) throw err;
console.log(userName + '!!!!--');
db.get(userName, function(err, result) {
if (err) throw err;
var o = result.value;
console.log(o['password'] + '***--');
return o['password'];
});
});
};
then I call
var db = require('kdatabase.js')();
var s = db.query(msg.username, function(err) {
if (err) {
console.log('aaa');
}
console.log('bbb');
return;
});
the lines
console.log(userName + '!!!!--');
console.log(o['password'] + '***--');
display correctly
but
console.log('aaa');
console.log('bbb');
are never executed
Your query method does not take a callback argument, so you never call it.
KDatabase.prototype.query = function(userName, cb) {
/* snip */
console.log(o['password'] + '***--');
cb(err, result);

Node.js Async | insert into postgresql database results from api

I am quite newbie with node.js. What i am trying to achieve is the following:
Connect to my postgresql database and get info of a place (id, coordinates).
call a weather api and get the info of that spot using the coordinates obtained in the previous step.
Insert the returned json in the database. I get 8 hourly objects, with the weather info every 3 hours (0,3,6,9,12,15,18,21). I need to iterate through this objects and the store them in 8 records in the database.
I wrote the following code:
app.get('/getapi', function(req, res){
var json_bbdd;
//------------ BBDD CONNECTION----------------
var pg = require('pg');
var conString = "postgres://postgres:postgres2#localhost/places";
var client = new pg.Client(conString);
client.connect(function(err) {
if(err) {
console.log('could not connect to postgres');
}
client.query('SELECT * from places where id=3276', function(err, result) {
if(err) {
console.log('error running query');
}
json_bbdd=result.rows[0];
var coords = JSON.parse(json_bbdd.json).coordinates;
var id = json_bbdd.id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: '',
callback: 'MarineWeatherCallback'
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" + input.query + "&format=" + input.format + "&fx=" + input.fx + "&key=" + _PremiumApiKey + "&tide=yes";
$.ajax({
type: 'GET',
url: url,
async: false,
contentType: "application/json",
dataType: 'jsonp',
success: function (json) {
var date= json.data.weather[0].date;
for (var i=0; i < 8; i++){
var hourly = json.data.weather[0].hourly[i];
var time= hourly.time;
client.query('INSERT into parte (id, date, time) VALUES($1, $2, $3)', [id, date, time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' ' + time);
}
});
} // FOR
},
error: function (e) {
console.log(e.message);
}
});
client.end();
});
});
});
The steps 1 and 2 are performed perfectly. The third step, on the other hand, does nothing and it doesn't even throw an error.
I read in this post: node-postgres will not insert data, but doesn't throw errors either that using async module could help but i have no idea how to rewrite the code. I need some help.
Regards,
Aitor
I didn't test your snippet, I can only help you with things which looks bad to my eyes.
It is better not to use jQuery on node server. There is excellent library called request to do remote http requests.
You should better handle database errors because in your example your code will continue after DB error.
You are calling client.end() too early and at the time when you try to insert data to the database a connection is already closed. You have to move client.end() at the end of success and error functions and wait to all callbacks are done.
I think it is also better to use connection pool instead of Client.
You can possibly use JSON type in PostgreSQL to avoid serializing/deserializing JSON data in your code.
Here is revised example(untested). I didn't replace jQuery here, some minor tweaking included.
var pg = require('pg');
var conString = "postgres://postgres:postgres2#localhost/places";
app.get('/getapi', function(req, res, next){
var json_bbdd;
//------------ BBDD CONNECTION----------------
pg.connect(conString, function(err, client, done) {
if(err) {
// example how can you handle errors
console.error('could not connect to postgres');
return next(new Error('Database error'));
}
client.query('SELECT * from places where id=3276', function(err, result) {
if(err) {
console.error('error running query');
done();
return next(new Error('Database error'));
}
json_bbdd = result.rows[0];
var coords = JSON.parse(json_bbdd.json).coordinates;
var id = json_bbdd.id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: '',
callback: 'MarineWeatherCallback'
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" +
input.query + "&format=" + input.format +
"&fx=" + input.fx + "&key=" +
_PremiumApiKey + "&tide=yes";
$.ajax({
type: 'GET',
url: url,
async: false,
contentType: "application/json",
dataType: 'jsonp',
success: function (json) {
var date = json.data.weather[0].date;
var callbacks = 0;
for (var i=0; i < 8; i++) {
var hourly = json.data.weather[0].hourly[i];
var time= hourly.time;
client.query(
'INSERT into parte (id, date, time) VALUES($1, $2, $3)',
[id, date, time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' ' + time);
}
callbacks++;
if (callbacks === 8) {
console.log('All callbacks done!');
done(); // done(); is rough equivalent of client.end();
}
});
} // FOR
},
error: function (e) {
console.error(e.message);
done(); // done(); is rough equivalent of client.end();
return next(new Error('Http error'));
}
});
});
});
});
Ok, now cam up with another problem...i was doubting of creating a new post but i think that maybe could have relation with the previous post.
The aim is to read from the database instead of one place 3 places and do the same process than before for each one.
The code is as follows (with the changes proposed by ivoszz):
app.get('/getapi', function(req, res, next){
//------------ BBDD CONNECTION----------------
pg.connect(conString, function(err, client, done) {
if(err) {
// example how can you handle errors
console.error('could not connect to postgres',err);
return next(new Error('Database error'));
}
client.query('SELECT * from places where id>3274 and id<3278', function(err, result) {
if(err) {
console.error('error running query',err);
done();
return next(new Error('Database error'));
}
var first_callback = 0;
for (var y=0; y<result.rows.length; y++) {
var coords = JSON.parse(result.rows[y].json).coordinates;
var id = result.rows[y].id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: ''
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" + input.query + "&format=" + input.format + "&fx=" + input.fx + "&key=" + _PremiumApiKey;
request(url, function(err, resp, body) {
body = JSON.parse(body);
if (!err && resp.statusCode == 200) {
var date = body.data.weather[0].date;
var callbacks = 0;
for (var i=0; i < 8; i++) {
var hourly = body.data.weather[0].hourly[i];
client.query(
'INSERT into parte (id, date, time) VALUES($1, $2, $3)',
[id, date, hourly.time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' iteration ' + i);
}
callbacks++;
if (callbacks === 8) {
console.log('All callbacks done!from id '+id);
//done(); // done(); is rough equivalent of client.end();
//res.send("done");
}
});
} // FOR
}
else { // if the API http request throws an error
console.error(err);
done(); // done(); is rough equivalent of client.end();
return next(new Error('Http API error'));
}
}); // REQUEST API URL
first_callback++;
if (first_callback === result.rows.length-1) {
console.log('All global callbacks done!');
done(); // done(); is rough equivalent of client.end();
res.send("done");
}}
}); // SELECT from pg
}); // CONNECT to pg
}); // app.get
I don't know why it tries to insert the id=3277 three times instead of inserting id=3275, id=3276 and then id=3277... what it does instead is: it inserts the first 8 records ok the first time (id=3277), but then it throws an error saying that the records are already inserted (primary key=id,date,time) with id 3277...
It seems that first does the 3 iterations of the first FOR and then does the 3 iteration of the second FOR but with the info of the last iteration(place). I can't understand it very well...

Resources