node.js synchronous call gather results - node.js

I have a table where Persons are saved an their time (in seconds) where they were active.
I would like to write a function that gathers the total time in another table called gather.
For each row I am checking if an entry in the gather table exists. Depending on that result I make an insert or an update.
db.serialize(function() {
db.each("SELECT * from TEST", function(err, row) {
db.get("SELECT * from GATHER where name = " + row.name "", function(err, row) {
if(row === undefined || row === null){
var stmt = db.prepare("INSTER INTO gather (name, time) VALUE(?,?)");
stmt.run([name, seconds], function(error){
console.log('lastInsert ' + this.lastID);
});
stmt.finalize();
}else{
seconds += row.time;//increment time
var stmt = db.prepare("UPDATE gather SET time = ? WHERE name = ?");
stmt.run([seconds, row.name], function(error){
console.log('lastInsert ' + row.idProcessed);
});
stmt.finalize();
}
});
});
});
The problem that I ecounter is that sqlite runs asynchronously. Therefore multiple entries are created in my gather table although lines should be updated.
What would be the right way to run this function sychronously? Should I limit the lines and call the function every second or is there a smarted way?

You could use async. For example (but first you should read final notes at the end):
var async = require('async');
var data = {}
var yourFirstSelect() = function(callback){
//You do your first select
//...
db.get("SELECT * from TEST", function(err, row) {
if(row){
data.name = row.name;
data.otherInterestingAttribute = row.otherInterestingAttribute;
callback(err, data);
}else{
callback('Row is null');
}
})
//..
}
var yourSecondSelect() = function(data, callback){
//You do your second select
//...
db.get("SELECT * from GATHER where name = " + data.name "", function(err, row) { //Note that I'm using data instead of row
if(row){
data.seconds = row.seconds;
data.otherInterestingAttribute = row.otherInterestingAttribute;
callback(err, data);
}else{
callback('Row is null');
}
})
//..
}
var decide() = function(data, callback){
if (data.somethingExists) { //Do your validations
data.type = 'update';
callback(err, data);
} else {
data.type = 'insert';
callback(err, data);
}
}
var update() = function(data,callback){
if (data.type === 'update') {
//...
//Do your stuff in order to update
seconds += row.time;//increment time
var stmt = db.prepare("UPDATE gather SET time = ? WHERE name = ?");
stmt.run([seconds, row.name], function(error){
console.log('update ' + row.idProcessed);
});
stmt.finalize();
//...
} else {
callback(err,data);
}
}
var insert() = function(data,callback){
if (data.type === 'insert') {
//...
//Do your stuff in order to insert
var stmt = db.prepare("INSTER INTO gather (name, time) VALUE(?,?)");
stmt.run([data.name, data.seconds], function(error){
console.log('lastInsert ' + this.lastID);
callback(err,data);
});
stmt.finalize();
//...
} else {
callback(err,data);
}
}
var niceWorkflow = function(){
async.waterfall([
yourFirstSelect,
yourSecondSelect,
decide,
update,
insert
],
function (err, result) {
console.log('Done');
})
}
//and call your workflow
niceWorkflow();
Off course this is not a 100% working code, I wrote it in order you look another way to do what you are trying. Many variables, validations and more are just examples and I intentionally forgot the db.each to avoid being too extense and confusing you and trying to answer your final question Is there a smarter way?.

You can also use Q.all from Q library.
It will return a promise after all the promises are resolved. It one promise fails, then Q.all will be rejected.

Related

How can I execute a query based on the result of another query? (Express, NodeJS, mysql)

I am trying to generate a session ID when an API call is made to my 'generateSession' endpoint. I want to make sure that I don't have any duplicate session ID, so I query the database checking for a match on the generated ID. If there isn't a match, the ID is valid and I make a second query to add an active user with said session ID.
Although my first query executes, the 'id_is_valid' boolean never gets set to true so my program gets stuck in the while loop.
I am fairly new to JavaScript, but from some research, I'm pretty sure the problem is due to the asynchronous nature of the database call. However, I'm not sure where to go from there. Could anyone with more js knowledge offer me some direction?
Thanks!
var express = require('express');
var router = express.Router();
var myDB = require('../db-connection');
function generateSession() {
var session_id = '';
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for (var i = 0; i < 30; i++) session_id += possible.charAt(Math.floor(Math.random() * possible.length));
return session_id;
}
router.get('/generateSession', function(req, res){
var session_id = '';
var id_is_valid = false;
while (!id_is_valid){
session_id = generateSession();
myDB.query("SELECT * FROM activeUser WHERE session_id = ?", [session_id], function(error, results, field){
if(error) throw error;
else{
if (results.length === 0) is_is_valid = true;
}
});
}
myDB.query("INSERT INTO activeUser (is_registered, session_id) VALUES (0, ?)", [session_id], function(error, results, fields){
if (error) res.send('{"success": false}');
else res.send('{"success": true, "session_id": "' + session_id + '"}');
});
});
Although my first query executes, the id_is_valid boolean never gets set to true so my program gets stuck in the while loop.
This is because db call, by nature is asynchronous. If you run the following program you will know it.
'use strict';
let id_is_valid = false;
let count = 0;
while (!id_is_valid) {
count++;
console.log(`No of time setTimeout Invoked ${count}`);
setTimeout(function() { // simulating a db call that takes a second for execution
id_is_valid = true;
}, 1000);
}
console.log('This line wont be printed');
Outputs
No of time setTimeout Invoked 61415
No of time setTimeout Invoked 61416
No of time setTimeout Invoked 61417
^C //I killed it.
Like damitj07, I too suggest using npms like shortId for uniquely generating the sessionId. This will help you to eliminate a database call.
But if your business logic restricts and you need it in the current fashion you wrote. I think we can use async & await
'use strict';
let isFound = false;
let count = 0;
function doDbQuery() {
return new Promise((resolve, reject) => {
setTimeout(function () {
resolve(true);
}, 2000);
});
}
async function run() {
while (!isFound) {
count++;
console.log(`No of time setTimeout Invoked ${count}`);
isFound = await doDbQuery();
}
console.log('This line WILL BE printed');
}
run();
Output
No of time setTimeout Invoked 1
This line WILL BE printed
Making those changes to your code,
router.get('/generateSession', async function (req, res) {
var session_id = '';
var id_is_valid = false;
while (!id_is_valid) {
session_id = generateSession();
id_is_valid = await checkSessionIdInDb(session_id);
}
myDB.query('INSERT INTO activeUser (is_registered, session_id) VALUES (0, ?)', [session_id], function (error, results, fields) {
if (error) {
res.send('{"success": false}');
} else {
res.send('{"success": true, "session_id": "' + session_id + '"}');
}
});
});
function checkSessionIdInDb() {
return new Promise((resolve, reject) => {
myDB.query('SELECT * FROM activeUser WHERE session_id = ?', [session_id], function (error, results, field) {
if (error) {
return reject(error);
} else {
if (results.length === 0) {
resolve(true);
}
resolve(false);
}
});
});
From what I could make out of your code what you want to do is basically create a new user session for logged in user with a unique session Id, also making sure that the session ID does not already exist in the collection.
So the solution to this can be to the first query to check if the session ID is already present in Active_Users Collection, if not make a save call to save the user with generated session Id.
var express = require('express');
var router = express.Router();
var myDB = require('../db-connection');
function generateSession() {
var session_id = '';
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for (var i = 0; i < 30; i++) session_id += possible.charAt(Math.floor(Math.random() * possible.length));
return session_id;
}
router.get('/generateSession', function(req, res) {
var session_id = '';
myDB.query("SELECT * FROM activeUser WHERE session_id = ?", [session_id], function(error, results, field) {
if (error) {
throw error;
} else {
if (results.length === 0) {
//if session id is not present , insert new user
session_id = generateSession();
myDB.query("INSERT INTO activeUser (is_registered, session_id) VALUES (0, ?)", [session_id], function(error, results, fields) {
if (error) res.send('{"success": false}');
else res.send('{"success": true, "session_id": "' + session_id + '"}');
});
}
//else do nothing or inser nothing
}
});
});
But Ideally, if you are generating a truly random session id each time when you make a request to the /generate session, is it really necessary to check for duplicates in the collection.
Now if the logic to generate a random ID is not perfect, you can always use modules like shortId to do the work for you. This will avoid unnecessary database call and save on performance and your code will be much neater.

async.each table insertion trouble?

I wrote a code using async.each for inserting data into tables.
var categoryList = [{"categoryName": "biriyani","productName":"chicken biriyani"}, {"categoryName":"biriyani","productName":"mutton biriyani"}]
async.each(categoryList, function(item,callback)
{
var categoryName=item.categoryName;
var productName=item.productName;
var categoryCheckQuery = pgFormat("select * from shop where categoryName LIKE '%"+categoryName+"%'");
model.client.query(categoryCheckQuery,function (err,result) {
if(result.rowCount==0){
var insertCategoryQuery = pgFormat("insert into shop(categoryName)values(%L)",categoryName);
model.client.query(insertCategoryQuery,function (err,result) {
if (!err) {
console.log("success");
}
});
}
else{
//insert product into product table
}
});
Explanation:
1)Here first Json array containing categoryName->biriyani is entered into shop table
2)when async.each fetching next json array containing categoryName->biriyani,
categoryCheckQuery checks the shop table whether categoryname = 'biriyani' is already exists.
3)If exists it wont be saved
Problem:
Here for both the data result.rowCount ==0 and both the data which have categoryname = biriyani is entered into shop table.
There are couple problems in this code.
One is the use of async.each(), async.eachSeries() should be used instead of async.each() because the operation of the next item in categoryList depends on current item's operation. async.eachSeries() ensures that first item is done before moving on to next item.
Another is async.each()'s callback() should be called to signal it that you're done with it.
Here is the revised code:
var categoryList = [{"categoryName": "biriyani","productName":"chicken biriyani"}, {"categoryName":"biriyani","productName":"mutton biriyani"}]
// Use async.eachSerices() instead of async.each()
async.eachSeries(categoryList, function(item,callback) {
var categoryName = item.categoryName;
var productName = item.productName;
var categoryCheckQuery = pgFormat("select * from shop where categoryName LIKE '%" + categoryName + "%'");
model.client.query(categoryCheckQuery, function (err, result) {
if (result.rowCount == 0) {
var insertCategoryQuery = pgFormat("insert into shop(categoryName)values(%L)", categoryName);
model.client.query(insertCategoryQuery, function (err, result) {
if (!err) {
console.log("success");
}
// passing non-null if you want to stop async.eachSeries() in case of error
callback(null); // <<<<< need to call async.each()'s callback() here
});
}
else {
//insert product into product table
doInsert(params, function(err, result) {
callback(null); // <<<<< need to call async.each()'s callback() here
});
}
});
});
Also, it's probably good practice to check for error returned.. specifically model.client.query() in this case

nodejs get sqlite3 query result using promise or wait

This is my first personal project in Nodejs. I'm trying to get in live soon.
I have a Nodejs server that uses sqlite3. There are only 3000 rows with word, transform and a precalculated value each in a column of the table, which is already populated.
I need to just lookup the word in the DB to be sure it is valid.
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.get("SELECT * FROM tab WHERE w = ?", word, function(err, row) {
if(err) { console.log("Lookup:",word,", Error => ",err); return false; }
return true;
});
The problem is that the caller of this code has a lot of context and need the operation to wait. So, I tried this
function dbLookup(db, w) {
return function(cb) {
var rows = [];
db.exec('SELECT w FROM tab WHERE w = "'+w+'"')
.on('row', function(r) {
rows.push(r)
})
.on('result', function() {
cb(rows);
});
}
async.each([word], function(w) {
dbLookup(this.db, w);
}, function(err) {
if(err) {console.log("...ERROR..."); return false; }
else {console.log("...SUCCESS..."); return true; }
});
This doesn't solve the wait issue as the callback can fire at its own pace.
I read that promise using something like bluebird can solve my problem
but now I'm not able to get the value/result of the query out:
I've been pulling my hair for so long. Please help me either get the async working or get the result back from the promise approach.
var async = require('async');
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
function check(word, callback) {
db.get("SELECT count(1) cnt FROM tab WHERE w = ?", word, callback)
}
async.map(words, check, function(err, results) {
if (err)
return console.log('Query error')
var all_checked = results.filter(function(r) {
return r.cnt > 0
});
...
});
Or
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.all("SELECT distinct w FROM tab", function(err, rows) {
var all_checked = words.filter(function (w) {
return rows.indexOf(w) != -1;
})
...
})

How to send mongodb data async from inner functions to outside callback function from a for loop?

NEW POST:
Here is the sample of the working async code without a db.
The problem is, if i replace the vars (data1_nodb,...) with the db.collection.find();
function, all needed db vars received at the end and the for() loop ends not
correct. I hope that explains my problem a bit better. OA
var calc = new Array();
function mach1(callback){
error_buy = 0;
// some vars
for(var x_c99 = 0; x_c99 < array_temp_check0.length;x_c99++){
// some vars
calc[x_c99] = new Array();
calc[x_c99][0]= new Array();
calc[x_c99][0][0] = "dummy1";
calc[x_c99][0][1] = "dummy2";
calc[x_c99][0][2] = "dummy3";
calc[x_c99][0][3] = "dummy4";
calc[x_c99][0][4] = "dummy5";
function start_query(callback) {
data1_nodb = "data1";
data2_nodb = "data2";
data3_nodb = "data3";
data4_nodb = "data4";
calc[x_c99][0][0] = data1_nodb;
calc[x_c99][0][1] = data2_nodb;
calc[x_c99][0][2] = data3_nodb;
callback(data1_nodb,data2_nodb,etc..);
}
start_query(function() {
console.log("start_query OK!");
function start_query2(callback) {
data4_nodb = "data5";
data5_nodb = "data6";
data6_nodb = "data7";
calc[x_c99][0][3] = data4_nodb;
calc[x_c99][0][4] = data5_nodb;
callback(data5_nodb,data6_nodb,etc..);
}
start_query2(function() {
console.log("start_query2 OK!");
function start_query3(callback) {
for(...){
// do something
}
callback(vars...);
}
start_query3(function() {
console.log("start_query3 OK!");
});
});
});
}
callback(calc);
};
function mach2(callback){
mach1(function() {
console.log("mach1 OK!");
for(...){
// do something
}
});
callback(calc,error_buy);
};
mach2(function() {
console.log("mach2 OK 2!");
});
OLD POST:
i try to read data from the mongodb and send them back with a callback to the next
function, that needs the infos from the db to proceed.
Without the mongodb read functions it works perfect but now i dont know how
i can send the db vars out of the two inner functions to the first callback function.
Hope someone can help me...
Thanks
var error = 0; var var1 = "yessir";
function start_query(callback) {
var db_name = "db1";
db[db_name].find({name:var1},{data1:1, data2:1, data3:1, data4:1}, function(err, data_catch,callback) {
if( err || !data_catch ) {
console.log("Problem finding data_catch:" + err);
} else {
data_catch.forEach( function(data_catch_finder,callback) {
data1_db = data_catch_finder.data1;
data2_db = data_catch_finder.data2;
data3_db = data_catch_finder.data3;
data4_db = data_catch_finder.data4;
if(data1_db == "" || data2_db == "" || data3_db == "" || data4_db == ""){error = 1; console.log("Error: data_catch_finder");}
callback(data1_db, data2_db, data3_db, data4_db, error);
});
}
});
callback(data1, data2, data3, data4, error);
}
//########################################################################
start_query(function() {
function start_query2(callback) {
console.log("DATA1 from callback:" + data1_db);
console.log("DATA2 from callback:" + data2_db);
console.log("DATA3 from callback:" + data3_db);
console.log("DATA4 from callback:" + data4_db);
var var_no_db = "testing";
//do something else and callback
callback(var_no_db);
}
start_query2(function() {
console.log("Var from callback start_query2:" + var_no_db);
console.log("The end");
});
});
your callback signature are issuing callback as a parameter.
As far as I can understand your code, you need to keep reference of the first callback, the one you receive here: function start_query(callback).
In every callback function you made the mistake to bind the variable name callback to the parameter from the mongo driver (a simple undefined i think).
You can fix it removing every reference of callback from the signature of your inner functions.
a simple example:
function async (cb) {
// params: Error, data
cb(null, 'moo');
}
function foo(callback) {
async(function(err, data, callback){
console.log(callback); // undefined
});
console.log(callback); // Moo
}
foo('Moo');
Take a look at Eloquent Javascript to better understand the JS context switching;
edit
The only way to wait the results of an async function is recall the first callback inside the last nested callback.
function ugly_nested (callback) {
dbquery('...', function(err, data_01) {
if (!! err) return callback(err);
dbquery('...', function(err, data_02) {
if (!! err) return callback(err);
dbquery('...', function(err, data_03) {
if (!! err) return callback(err);
callback(null, data_01, data_02, data_03);
});
});
});
}
ugly_nested(function(err, data01, data02, data03) {
if (!! err) throw err;
manage_data(data01, data02, data03);
});
The FOR loop is synchronous, but, the database calls are asynchronous, so, the for loop will end before the database returns his results. If you really need that for loop you can try out one of the nice flow control libraries out there

nested loops asynchronously in Node.js, next loop must start only after one gets completed

Check below algorithm...
users = getAllUsers();
for(i=0;i<users.length;i++)
{
contacts = getContactsOfUser(users[i].userId);
contactslength = contacts.length;
for(j=o;j<contactsLength;j++)
{
phones = getPhonesOfContacts(contacts[j].contactId);
contacts[j].phones = phones;
}
users[i].contacts = contacts;
}
return users;
I want to develop such same logic using node.js.
I have tried using async with foreach and concat and foreachseries functions. But all fail in the second level.
While pointer is getting contacts of one user, a value of i increases and the process is getting started for next users.
It is not waiting for the process of getting contacts & phones to complete for one user. and only after that starting the next user. I want to achieve this.
Actually, I want to get the users to object with proper
Means all the sequences are getting ruined, can anyone give me general idea how can I achieve such a series process. I am open to change my algorithm also.
In node.js you need to use asynchronous way. Your code should look something like:
var processUsesrs = function(callback) {
getAllUsers(function(err, users) {
async.forEach(users, function(user, callback) {
getContactsOfUser(users.userId, function(err, contacts) {
async.forEach(contacts, function(contact, callback) {
getPhonesOfContacts(contacts.contactId, function(err, phones) {
contact.phones = phones;
callback();
});
}, function(err) {
// All contacts are processed
user.contacts = contacts;
callback();
});
});
}, function(err) {
// All users are processed
// Here the finished result
callback(undefined, users);
});
});
};
processUsers(function(err, users) {
// users here
});
You could try this method without using async:
function getAllUserContacts(users, callback){
var index = 0;
var results = [];
var getUserContacts = function(){
getContactsOfUser(users[index].userId, function(contacts){
var index2 = 0;
var getContactsPhones = function(){
getPhonesOfContacts(contacts[index2].contactId, function(phones){
contacts[index2].phones = phones;
if(index2 === (contacts.length - 1)){
users[index].contacts = contacts;
if(index === (users.length - 1)){
callback(users)
} else {
index++;
getUserContacts();
}
}else{
index2++;
getContactsPhones();
}
});
}
getContactsPhones();
});
}
getUserContacts();
}
//calling the function
getAllUsers(function(users){
getAllUsersWithTheirContacts(users, function(usersWithContacts){
console.log(usersWithContacts);
})
})
//Asynchronous nested loop
async.eachSeries(allContact,function(item, cb){
async.eachSeries(item,function(secondItem,secondCb){
console.log(secondItem);
return secondCb();
}
return cb();
},function(){
console.log('after all process message');
});

Resources