node-postgres : query not executing in sequence - node.js

i have kept insert & update code in 2 different files and based on condition
always insert should execute first and then update.but somehow update executes first then insert
test.js : simplified code
i am using these packages :pg , uuid
var pg = require('pg');
var uuid = require('node-uuid').v4;
var id = uuid().toString();
var conString = 'postgres://postgres:pass#127.0.0.1:5432/testdb';
// ------INSERT
pg.connect(conString, function(err, client, done) {
console.log('Executing Insert query');
client.query('insert into testdb (id,data,iscancelled) values ($1,$2,$3)',[id,'hello','no'], function(err, result) {
done();
if(err) { return console.error('error running query', err); }
console.log('finished executing Insert query');
});
});
// ------UPDATE
pg.connect(conString, function(err, client, done) {
console.log('Executing update query');
client.query("update testdb set iscancelled = 'yes' where id = $1",[id], function(err, result) {
done();
if(err) { return console.error('error running query', err); }
console.log('finished executing Update query');
});
});
output
tom#tom:~$node test.js
Executing Insert query
Executing update query
finished executing Update query //WHY UPDATE FINISHES FIRST
finished executing Insert query
Note :
this problem can be easily solved by using async.But my insert code and update code are in different files and depending on some situation update code might execute.so don't want to use async
Problem
Even though Insert query goes to execute first why does update finishes first in output
am i missing any thing ..?

Lets solve this question step by step
you "stated so don't want to use async" libraries
solution 1 :
if PostgreSQL make update faster, update will return result before insert. If you want start executing update query only after finishing insert then
you should set connection pool capacity to 1.
pg.defaults.poolSize = 1
but you should do this before any pg.connect()
The connect method retrieves a Client from the client pool, or if all pooled clients are busy and the pool is not full, the connect method will create a new client passing its first argument directly to the Client constructor. In either case, your supplied callback will only be called when the Client is ready to issue queries or an error is encountered. The callback will be called once and only once for each invocation of connect.
Conclusion : your queries will execute in sequence.BUT BUT BUT this solution is BAD for scaling app as there will be always only one connection serving all users .So till one connection is serving one user , other users will have to wait for response.
Solution 2 :
you also stated "i have kept insert & update code in 2 different files"
it looks like you need to designed your code in such a way that it you are able to use asynchronus libraries , that solves this problem

As I already mentioned, the only way to ensure that update function will be fired only after insert function is done, is to call it inside of insert function callback. That are the basics of asynchronous programming.
pg.connect(conString, function(err, client, done) {
console.log('Executing Insert query');
client.query('insert into testdb (id,data,iscancelled) values ($1,$2,$3)',[id,'hello','no'], function(err, result) {
done();
if(err) { return console.error('error running query', err); }
console.log('finished executing Insert query');
// ------UPDATE
pg.connect(conString, function(err, client, done) {
console.log('Executing update query');
client.query("update testdb set iscancelled = 'yes' where id = $1",[id], function(err, result) {
done();
if(err) { return console.error('error running query', err); }
console.log('finished executing Update query');
});
});
});

You are missing the asynchronous nature of the pg.connect and also client.query. The call to these return a callback which passes the control to next expression before the completion of execution and hence non-blocking nature of nodejs. If you want to assure the correct flow, either call the successive ones inside the callback success
var pg = require('pg');
var uuid = require('node-uuid').v4;
var id = uuid().toString();
// ------INSERT
return pg.connect;
// ------UPDATE
return pg.connect;
// your calling file
var insert = require('/path/to/insertfile');
var conString = 'postgres://postgres:pass#127.0.0.1:5432/testdb';
var update = require('/path/to/updatefile');
insert(conString, function (err, client, done) {
console.log('Executing Insert query');
client.query('insert into testdb (id,data,iscancelled) values ($1,$2,$3)',[id,'hello','no'], function (err, result) {
if (err) {
return console.error('error running query', err);
}
console.log('finished executing Insert query');
update(conString, function (error, client, done) {
console.log('Executing update query');
client.query("update testdb set iscancelled = 'yes' where id = $1",[id], function (err, result) {
if (err) {
return console.error('error running query', err);
}
console.log('finished executing Update query');
done();
});
});
done();
});
});
But this is very prone to callback hell. So consider making all async call return a promise. Take a look at bluebird. If you want an ORM that has built in promise based call, you can take a look at sequelize. It might be handy for you.
It has syntax as easy as:
var Model1 = require('/path/to/model1');
var Model2 = require('/path/to/model2');
var insertObj = {
"someKey": "value"
};
Model1.create(insertObj)
.then( function (createdObj1) {
return Model2.findOne({
where: {
"filter": "filterValue"
}
});
})
.then( function (documentToUpdate) {
return documentToUpdate.update({
"fieldToUpdate": "value"
});
})
.then( null, function (err) {
console.log(err);
});

Related

Connection reset while executing multiple queries using Postgres pg and node

Here is my code that performs multiple select queries one after other.
const {Pool} = require('pg');
const pool = new Pool(POSTGRES_CONFIG);
var aggregateDomainCount = [], domainCount={};
pool.connect((err, client, release) => {
if (err) {
return console.error('Connection error', err.stack)
}
console.log('Connection established');
client.query(firstQuery, function(err, results) {
// call `done()` to release the client back to the pool
if(err) {
return console.error('error running query', err);
}
domainCount = {"data1":results.rows[0].count};
aggregateDomainCount.push(domainCount);
});
......
client.query(lastQuery, function(err, results) {
// call `done()` to release the client back to the pool
if(err) {
return console.error('error running query', err);
}
domainCount = {"dataN":results.rows[0].count};
aggregateDomainCount.push(domainCount);
release();
response.json(aggregateDomainCount);
});
});
The issue is while executing above queries, console first prints('Connection established') and then after series of execution, again it tries to enter pool.connect before completing all queries and prints('Connection established') and finally results in unknown error to user interface. If i reduce number of queries by some, then I get expected results but which is again inconsistent.
So far, I have tried
nesting client.query one inside other
nesting pool.connect by grouping set of queries
setting different timeout parameters
using await client.query()
But all the above resulted in same issue. Could you please point me in right direction.

Postgres connection pooling error troubleshooting

I have an AWS Lambda function written in nodejs that is doing a set of recursive postgres database calls that result in the following error every time on the 81st call:
remaining connection slots are reserved for non-replication superuser
connections
I'm assuming I am leaking something at the postgres levels but I believe I am adhering to the recommended calls for performing a single pooled query as defined at https://node-postgres.com/features/pooling. I've simplified my code as shown below so that I'm only executing the same query every time and the result is still the same. The function testHarness is what initiates the logic within my lamba function. The intent here is execute a query against postgres, once it is completed to fire off the query again, repeating 500 times for this example. It always fails when the 81st call occurs. The DB_CONNECT environment variable contains the connection information including a "MAX" value of 3.
function testHarness(cb){
_test(0, cb);
}
function _test(pos, cb){
console.log(pos);
_testData(function (err, data){
if (err) return cb(err);
if (pos < 500){
_test(pos + 1, cb);
}
else{
return cb(null, 'done');
}
});
}
function _testData(cb){
const { Pool } = require('pg')
const pool = new Pool(JSON.parse(process.env.DB_CONNECT));
const sql = 'SELECT id, url, pub_date, description, title, duration FROM episodes WHERE feed_id = $1 ORDER BY pub_date DESC LIMIT 10';
pool.query(sql, ['28c65c8d-f96a-4499-a854-187eed7050bd'], (err, result) => {
if (err) throw err;
return cb(err, result);
})
}
So the problem is leaking Pool objects that you create in _testData function. After using a Pool you have to shut it down you and find the documentation here under "Shutdown" title, as it says:
pool.end()
But, the way you are using Pool does not make sense. It is better to put it in _testHarness function to be able to reuse the connection and save the connection overhead time to let your code run faster:
function testHarness(cb){
const { Pool } = require('pg')
const pool = new Pool(JSON.parse(process.env.DB_CONNECT));
_test(pool, 0, function(err, data){
pool.end();
cb(err, data);
});
}
function _test(pool, pos, cb){
console.log(pos);
_testData(pool, function (err, data){
if (err) return cb(err);
if (pos < 500){
_test(pos + 1, cb);
}
else{
return cb(null, 'done');
}
});
}
function _testData(pool, cb){
const sql = 'SELECT id, url, pub_date, description, title, duration FROM episodes WHERE feed_id = $1 ORDER BY pub_date DESC LIMIT 10';
pool.query(sql, ['28c65c8d-f96a-4499-a854-187eed7050bd'], (err, result) => {
if (err) throw err;
return cb(err, result);
})
}
I am not AWS user, but I guess it should be like any other postgres database service, you might need to change it a bit to fit AWS service.
Also, don't you have the ability to use async/await pattern? It is lot easier to comprehend.

AWS Lambda query with async waterfall in node.js 8.10

I send two query sequentially
Query the data from A tables, and then accoring to the result, query the data from B table.
So, I query the data like that,
var async = require('async');
var mysql = require('mysql');
var config = require('./config.json');
var connection = mysql.createConnection({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
exports.handler = (event, context, callback) => {
// TODO implement
var tasks = [
function (callback) {
connection.query("SELECT email FROM Visitor WHERE id =?;", [1], function (err, row) {
if (err) return callback(err);
if (row.length == 0) return callback('No Result Error');
callback(null, row[0]);
})
},
function (data, callback) {
connection.query("SELECT id,signtime FROM Board WHERE email =?;", data.email, function (err, row) {
if (err) return callback(err);
if (row.length == 0) {
return callback('No Result Error');
}else {
callback(null, row[0])
}
})
}
];
async.waterfall(tasks, function (err, result) {
if (err)
console.log('err');
else
***return result;***
console.log('done');
connection.end();
});
};
I log the data with console.log(), it take the data in command line.
But in lambda, put the function into exports.handler, it response null.
If I change the 'return result' to callback(result), it occurs error.
I think it maybe too simple to solve this problem
If you know about that, please help me
In the first case, response is null because you didn't use neither Promise, nor callback to let the Lambda sandbox know that the job is done. In the second case, you used the callback, but you passed the result as the first argument to it. Lambda programming model for Node.js follows a principle called "error first callback". Long story short, if any error occurred during execution, you should go with callback(error), and if everything is ok and you need to return some result from lambda, you should go with callback(null, result). So basically on your line before console.log('done'); use callback(null, result) and it will work for you.

Sqlite transactions in nodejs

Accordingly with nodejs sqlite library (https://github.com/mapbox/node-sqlite3) you can serialize a sql transaction with db.serialize function. All executions in this function will be executed in the specified order.
But with a for loop how can i get the information about the transaction will be success or fail?
function transaction(callback) {
db.serialize(function() {
db.run("BEGIN");
for(...) {
db.run("INSERT", function(err) {
//this code is not serialized with the parent serialize function (this is my problem)
//if one run throw error i must resolve the callback with the error code
if (err)
callback("error");
});
}
db.run("END");
callback("ok");
}
}
Before start transaction you can define some vars e.g. var results = [];. On complete each query inside callback you can collect error or null as results.push(err) and check results.length. When results.length equals to length of for then all queries executed and now check results.every((i) => i === null) value. If true then do commit else rollback.
Much better to control flow is use async module.
'use strict'
var async = require('async');
...
db.run('begin', function(err){
if (err)
return console.log(err);
let run = (sql, callback) => db.run(sql, callback);
async.eachSeries(sqls, run, function(err){
db.run((err) ? 'rollback' : 'commit', ...)
})
})
If you add begin to sqls then code will be more simple.

res.send() is running before nested function completes with value

The 'GetUsers' function is running and it returns the correct values as expected. However, res.send(users) is running before the nested function has completed with the value so it is undefined. I have tried doing the res.send(users) from inside the nested function but it doesn't recognize the function. I have also tried using return users but can't get it to work. Any help would be appreciated!
app.get('/GetUsers', function (req, res){
var url = 'mongodb://localhost:27017/users';
var users = "";
MongoClient.connect(url, function (err, db) {
if (err) {
console.log('Unable to connect to the mongoDB server. Error:', err);
} else {
console.log('Connection established to', url);
// Get the documents collection
var collection = db.collection('users');
// Get some users
collection.find().toArray(function (err, res) {
if (err) {
console.log(err);
} else {
console.log('Got Information from db.');
}
//Close connection
db.close();
console.log('db closed.');
users = res;
console.log(users);
});
}
});
res.send(users);
});
Just move res.send(users) inside your callback before/after you call console.log(users).
You reused the variable res in the toArray call, so please rename it there: ...toArray(function (err, res)... to ...toArray(function (err, result)... and also in users = res; to users = result;.
this is async call it will send the response and will not wait for the database connection function.
write res.send in call back function
write your res.send function where you are closing your data base connection.
here is detail How do I return the response from an asynchronous call?
Found the answer!!!
I needed to use res.send(users) in my callback after I had closed the database as pointed out by the users suggestions above. HOWEVER, that alone wasn't working as res.send(users) was erroring as it wasn't recognized as a function. I had to use res.send({'matched':users}) and then access my response objects on the client side like this: response.matched.objects etc.
Thanks for your help everyone!

Resources