Blocking behaviour in node-mysql - node.js

I'm playing with node.js and node-mysql and I'd like to understand the following:
connection.connect(function(err) { if (err) throw err; });
console.log(" beginn insert "+ Date() );
for (var i=0;i<200;i++){
connection.query('INSERT INTO animals SET ?', {name: "tiger!"});
}
connection.end();
console.log(" end insert " + Date() );
The output is always ' beginn insert ' immediately followed by ' end insert ' and then only the database starts working. How is it possible that these node-mysql statements seem to be non-blocking?
Thanks, Felix

Node itself is non-blocking for IO so queries such as these do not block. Instead you need to pass a callback function when the query completes:
var queries = 0;
connection.query("INSERT INTO animals SET ?", {name: "tiger!"},
function (err) {
// handle error?
queries++;
if (queries == 200) {
// Last query has finished running
}
});

Use Async; something like the following:
console.log(" beginning insert "+ Date() );
// create your queries as an array of objects
queries = [];
for (var i=0;i<200;i++){
queries.push({name: "tiger!"});
};
async.map(queries, myQuery, function(err, data){
// Here all of your queries are done
connection.end();
console.log(" end insert " + Date() );
});
function myQuery(name, callback){
connection.query('INSERT INTO animals SET ?', [name], function(err, result) {
if(err){
console.error(err);
callback(err, null);
}else{
callback(null, result);
}
});
}

Related

Node JS res.send not a function

i'm trying create first node js api.
i wroted this code but when i start server.js it's not working.
here code :
var executeQuery = function(res, query){
sql.connect(dbConfig, function (err) {
if (err) {
console.log("Error while connecting database :- " + err);
res.send(err);
}
else {
// create Request object
var request = new sql.Request();
// query to the database
request.query(query, function (err, res) {
if (err) {
console.log("Error while querying database :- " + err);
res.send(err);
}
else {
res.send(res);
}
});
}
});
}
when i start it error say :
C:\node\atlasErpApi\server.js:47 res.send(res);
TypeError: res.send is not a function at C:\node\atlasErpApi\server.js:47:43
You are declaring a new variable res as argument to your request.query callback which shadows the res the comes as argument to your executeQuery function.
This should be most obvious at the line res.send(res) which, if you think about it, doesn't make sense at all.
Use different variable names for the two. For example, change the inner one to sqlRes:
var executeQuery = function(res, query) {
sql.connect(dbConfig, function(err) {
if (err) {
console.log("Error while connecting database :- " + err);
res.send(err);
} else {
// create Request object
var request = new sql.Request();
// query to the database
request.query(query, function(err, sqlRes) { // <<< CHANGED to sqlRes
if (err) {
console.log("Error while querying database :- " + err);
res.send(err);
} else {
res.send(sqlRes); // This makes more sense now :-)
}
});
}
});
}
Note: Technically this also happens for err - you have an err from request.query's callback shadowing the err from sql.connect's callback. But this usually doesn't matter because you won't need to use the outer err in the inner callback.

prepared statements node-postgresql error with null result

i am new in node.js and postgresql. am allready connected with postgres db and execute some test code. after am going to use prepared statements.am create a sample login script. if user exist it return username else return message "invalid username or password". if username and password is correct,it return username. but no data will present , then could not return message. my openion is that , the control will crashed after executing cliend.end() function.
this is my code
UserLogin.get = function(userName, callBack) {
pg.connect(pgConString, function(err, client, done) {
if (err) {
callBack("DB connection failed. " + err, null);
return;
}
var selectQuery="SELECT * from " + TABLE + " WHERE userName=($1)";
var query=client.query({
text:selectQuery,
values:[userName],
name:"selectQuery"});
query.on("error", function (error) {
callBack("DB fetch failed. Error Message: " + err, null);});
query.on('row', function(row) {
callBack(null, row);});
query.on("end", function (result) {
client.end();
return;
});
});
}
if row is empty, not return to callback. if row_result is not empty, its working fine.. How...???? any idea...???
finally i got the answer. thanks for u r supports
UserLogin.get = function(userName, callBack) {
pg.connect(pgConString, function(err, client, done) {
if (err) {
callBack("DB connection failed. " + err, null);
return;
}
var selectQuery="SELECT * from " + TABLE + " WHERE userName=($1)";
var query=client.query({
text:selectQuery,
values:[loginId],
name:"selectQuery"});
query.on("error", function (error) {
callBack("DB fetch failed. Error Message: " + err, null);return;});
query.on('row', function(row,result) {
result.addRow(row);
});
query.on('end', function (result) {
callBack(null,result.rows);
client.end();
return;
});
});
}
in my old code, callback will be called everytime the query fetches 1 row from the database. am just changed the logic
Your code
query.on('row', function(row) {
callBack(null, row);
});
means that the callback will be called everytime the query fetches 1 row from the database. In the case when the query has 0 results, the callback will never be called.

Node.js | get unique result from multiple querys

I wrote the following module which connects to postgresql or SQLServer depending a the Type var value:
exports.GetQueryResult = function (type, Name,con,callback) {
var sql='';
if (Type ='PG') {
sql=sql + ' SELECT …………..';
pg.connect(con, function(err, client, done) {
if(err) {
console.log("Error :" + err);
return callback(err);
}
client.query(sql,[Name], function(err, Result) {
if(err) {
console.log("Error: " +err);
return callback(err);
}
return callback(null,Result);
done();
});
});
}
else
{
sql=sql + ' SELECT …..';
sql.open(con, function (err, conn,done) {
if (err) {
console.log("Error :" + err);
return callback(err);
}
conn.queryRaw(sql,Name, function (err, Result) {
if (err) {
console.log("Error ejecutando la consulta. Error: " +err);
return callback(err);
}
callback(null,Result);
done;
});
});
}
};
I call this function from:
var MultiBD = require('./MultiBD.js');
var LayerType=['PG','SQL','PG'];
var con=’’;
for (var i=1; i<=Layers.length; i++) {
if (Layers[i-1]!=undefined){
con=MultiBD.conexion(LayerType [i-1],server,BD);
MultiBD.GetQueryResult(LayerType[i-1], Name[i-1],con,
function (err,Result){
console.log('Result : ' + Result.rows.length);
}
);
}
}
The results are:
Result : 111
Result : 2888
Result : 5
I get three query results. The first one returns 111 rows, the second one 2888 and the third one 5.
What i need is to get only one unique result with all the 3004 rows (111+2888+5).
Regards,
You'll need some flow control to wait until all 3 methods have completed their DB calls before printing their results. There are packages for this, either of the callback variety or Promises.
Here is an example using the async package (which is of the callback variety of flow-control):
var async = require('async');
var totalResults = 0;
async.each( Layers, function eachMethod( layer, eachCb ){
var con = MultiBD.conexion(layer,server,BD);
MultiBD.GetQueryResult(LayerType[i-1], Name[i-1],con, function (err,Result) {
if( err ) return eachCb( err );
totalResults += Result.rows.length;
eachCb( null );
} );
}, function finalEach( eachErr ){
if( eachErr ) console.log( "There was an error. " + eachErr );
else console.log( "Result: " + totalResults );
});
In this call, a method (the eachMethod) is called on every item in the array. The results are stored in a shared variable. When all items have completed, the finalEach method is called. Here errors have bubbled up (which you can choose to check or not) and all calls have completed, so we can simply print the value of the shared variable.

Node.js Async | insert into postgresql database results from api

I am quite newbie with node.js. What i am trying to achieve is the following:
Connect to my postgresql database and get info of a place (id, coordinates).
call a weather api and get the info of that spot using the coordinates obtained in the previous step.
Insert the returned json in the database. I get 8 hourly objects, with the weather info every 3 hours (0,3,6,9,12,15,18,21). I need to iterate through this objects and the store them in 8 records in the database.
I wrote the following code:
app.get('/getapi', function(req, res){
var json_bbdd;
//------------ BBDD CONNECTION----------------
var pg = require('pg');
var conString = "postgres://postgres:postgres2#localhost/places";
var client = new pg.Client(conString);
client.connect(function(err) {
if(err) {
console.log('could not connect to postgres');
}
client.query('SELECT * from places where id=3276', function(err, result) {
if(err) {
console.log('error running query');
}
json_bbdd=result.rows[0];
var coords = JSON.parse(json_bbdd.json).coordinates;
var id = json_bbdd.id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: '',
callback: 'MarineWeatherCallback'
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" + input.query + "&format=" + input.format + "&fx=" + input.fx + "&key=" + _PremiumApiKey + "&tide=yes";
$.ajax({
type: 'GET',
url: url,
async: false,
contentType: "application/json",
dataType: 'jsonp',
success: function (json) {
var date= json.data.weather[0].date;
for (var i=0; i < 8; i++){
var hourly = json.data.weather[0].hourly[i];
var time= hourly.time;
client.query('INSERT into parte (id, date, time) VALUES($1, $2, $3)', [id, date, time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' ' + time);
}
});
} // FOR
},
error: function (e) {
console.log(e.message);
}
});
client.end();
});
});
});
The steps 1 and 2 are performed perfectly. The third step, on the other hand, does nothing and it doesn't even throw an error.
I read in this post: node-postgres will not insert data, but doesn't throw errors either that using async module could help but i have no idea how to rewrite the code. I need some help.
Regards,
Aitor
I didn't test your snippet, I can only help you with things which looks bad to my eyes.
It is better not to use jQuery on node server. There is excellent library called request to do remote http requests.
You should better handle database errors because in your example your code will continue after DB error.
You are calling client.end() too early and at the time when you try to insert data to the database a connection is already closed. You have to move client.end() at the end of success and error functions and wait to all callbacks are done.
I think it is also better to use connection pool instead of Client.
You can possibly use JSON type in PostgreSQL to avoid serializing/deserializing JSON data in your code.
Here is revised example(untested). I didn't replace jQuery here, some minor tweaking included.
var pg = require('pg');
var conString = "postgres://postgres:postgres2#localhost/places";
app.get('/getapi', function(req, res, next){
var json_bbdd;
//------------ BBDD CONNECTION----------------
pg.connect(conString, function(err, client, done) {
if(err) {
// example how can you handle errors
console.error('could not connect to postgres');
return next(new Error('Database error'));
}
client.query('SELECT * from places where id=3276', function(err, result) {
if(err) {
console.error('error running query');
done();
return next(new Error('Database error'));
}
json_bbdd = result.rows[0];
var coords = JSON.parse(json_bbdd.json).coordinates;
var id = json_bbdd.id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: '',
callback: 'MarineWeatherCallback'
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" +
input.query + "&format=" + input.format +
"&fx=" + input.fx + "&key=" +
_PremiumApiKey + "&tide=yes";
$.ajax({
type: 'GET',
url: url,
async: false,
contentType: "application/json",
dataType: 'jsonp',
success: function (json) {
var date = json.data.weather[0].date;
var callbacks = 0;
for (var i=0; i < 8; i++) {
var hourly = json.data.weather[0].hourly[i];
var time= hourly.time;
client.query(
'INSERT into parte (id, date, time) VALUES($1, $2, $3)',
[id, date, time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' ' + time);
}
callbacks++;
if (callbacks === 8) {
console.log('All callbacks done!');
done(); // done(); is rough equivalent of client.end();
}
});
} // FOR
},
error: function (e) {
console.error(e.message);
done(); // done(); is rough equivalent of client.end();
return next(new Error('Http error'));
}
});
});
});
});
Ok, now cam up with another problem...i was doubting of creating a new post but i think that maybe could have relation with the previous post.
The aim is to read from the database instead of one place 3 places and do the same process than before for each one.
The code is as follows (with the changes proposed by ivoszz):
app.get('/getapi', function(req, res, next){
//------------ BBDD CONNECTION----------------
pg.connect(conString, function(err, client, done) {
if(err) {
// example how can you handle errors
console.error('could not connect to postgres',err);
return next(new Error('Database error'));
}
client.query('SELECT * from places where id>3274 and id<3278', function(err, result) {
if(err) {
console.error('error running query',err);
done();
return next(new Error('Database error'));
}
var first_callback = 0;
for (var y=0; y<result.rows.length; y++) {
var coords = JSON.parse(result.rows[y].json).coordinates;
var id = result.rows[y].id;
var input = {
query: coords[1] + ',' + coords[0] ,
format: 'JSON',
fx: ''
};
var url = _PremiumApiBaseURL + "marine.ashx?q=" + input.query + "&format=" + input.format + "&fx=" + input.fx + "&key=" + _PremiumApiKey;
request(url, function(err, resp, body) {
body = JSON.parse(body);
if (!err && resp.statusCode == 200) {
var date = body.data.weather[0].date;
var callbacks = 0;
for (var i=0; i < 8; i++) {
var hourly = body.data.weather[0].hourly[i];
client.query(
'INSERT into parte (id, date, time) VALUES($1, $2, $3)',
[id, date, hourly.time],
function(err, result) {
if (err) {
console.log(err);
} else {
console.log('row inserted: ' + id + ' iteration ' + i);
}
callbacks++;
if (callbacks === 8) {
console.log('All callbacks done!from id '+id);
//done(); // done(); is rough equivalent of client.end();
//res.send("done");
}
});
} // FOR
}
else { // if the API http request throws an error
console.error(err);
done(); // done(); is rough equivalent of client.end();
return next(new Error('Http API error'));
}
}); // REQUEST API URL
first_callback++;
if (first_callback === result.rows.length-1) {
console.log('All global callbacks done!');
done(); // done(); is rough equivalent of client.end();
res.send("done");
}}
}); // SELECT from pg
}); // CONNECT to pg
}); // app.get
I don't know why it tries to insert the id=3277 three times instead of inserting id=3275, id=3276 and then id=3277... what it does instead is: it inserts the first 8 records ok the first time (id=3277), but then it throws an error saying that the records are already inserted (primary key=id,date,time) with id 3277...
It seems that first does the 3 iterations of the first FOR and then does the 3 iteration of the second FOR but with the info of the last iteration(place). I can't understand it very well...

cannot print all elements in collection although count is positive

I can get positive count of elements in collection ...
var collection;
collection = db.collection("allCodes");
collection.count(function(err, count) {
if (err) {
throw err;
}
console.log("There are " + count + " records.");
});
... with output:
Connected to Database ok
There are 354 records.
... but can not get elements in this collection:
collection.find().each(function(err, doc) {
if (err) {
throw err;
}
console.log("each doc");
console.log(doc);
});
... it prints nothing. I am new in mongodb. So what I do wrong? I want to print all elements in allCodes collection.
Update: all code that inserts data then gets count and then tries to fetch data itself, but nothing comes out.
var MongoClient, collection;
MongoClient = require("mongodb").MongoClient;
var objectToInsert = [{
'a': 1
}, {
'a': 2
}, {
'b': 3
}]
MongoClient.connect("mongodb://127.0.0.1:27017/test", function(err, db) {
console.log("Connected to Database");
collection = db.collection("test2");
// clear collection -------------------------------
collection.remove(function(err, result) {
// insert ------------------------------------
collection.insert(objectToInsert, function(docs) {
// count - ok -----------------------------------
collection.count(function(err, count) {
console.log("Count: " + count);
// find - fail - no objects printed -----------------------
collection.find().toArray(function(err, docs) {
console.log("Printing docs from Array");
docs.forEach(function(doc) {
console.log("Doc from array");
console.log(doc);
});
});
db.close();
});
});
});
});
It has output:
Connected to Database
Count: 3
So why I have only count. Where is my data?
You're closing your connection to the database before the find gets a chance to complete.
Move the db.close() call inside the callback of toArray like this:
collection.find().toArray(function(err, docs) {
console.log("Printing docs from Array");
docs.forEach(function(doc) {
console.log("Doc from array");
console.log(doc);
});
db.close();
});
Disclaimer: I am not familiar with node.js at all.
From the example in the documentation, it seems that you are required to create a cursor object first and only afterwards iterate over the results. I'm not sure what considerations there are for chaining commands.
var cursor = collection.find();
// Execute the each command, triggers for each document
cursor.each( function( err, item ) {
console.log( "each doc" );
});

Resources