I wrote a code using async.each for inserting data into tables.
var categoryList = [{"categoryName": "biriyani","productName":"chicken biriyani"}, {"categoryName":"biriyani","productName":"mutton biriyani"}]
async.each(categoryList, function(item,callback)
{
var categoryName=item.categoryName;
var productName=item.productName;
var categoryCheckQuery = pgFormat("select * from shop where categoryName LIKE '%"+categoryName+"%'");
model.client.query(categoryCheckQuery,function (err,result) {
if(result.rowCount==0){
var insertCategoryQuery = pgFormat("insert into shop(categoryName)values(%L)",categoryName);
model.client.query(insertCategoryQuery,function (err,result) {
if (!err) {
console.log("success");
}
});
}
else{
//insert product into product table
}
});
Explanation:
1)Here first Json array containing categoryName->biriyani is entered into shop table
2)when async.each fetching next json array containing categoryName->biriyani,
categoryCheckQuery checks the shop table whether categoryname = 'biriyani' is already exists.
3)If exists it wont be saved
Problem:
Here for both the data result.rowCount ==0 and both the data which have categoryname = biriyani is entered into shop table.
There are couple problems in this code.
One is the use of async.each(), async.eachSeries() should be used instead of async.each() because the operation of the next item in categoryList depends on current item's operation. async.eachSeries() ensures that first item is done before moving on to next item.
Another is async.each()'s callback() should be called to signal it that you're done with it.
Here is the revised code:
var categoryList = [{"categoryName": "biriyani","productName":"chicken biriyani"}, {"categoryName":"biriyani","productName":"mutton biriyani"}]
// Use async.eachSerices() instead of async.each()
async.eachSeries(categoryList, function(item,callback) {
var categoryName = item.categoryName;
var productName = item.productName;
var categoryCheckQuery = pgFormat("select * from shop where categoryName LIKE '%" + categoryName + "%'");
model.client.query(categoryCheckQuery, function (err, result) {
if (result.rowCount == 0) {
var insertCategoryQuery = pgFormat("insert into shop(categoryName)values(%L)", categoryName);
model.client.query(insertCategoryQuery, function (err, result) {
if (!err) {
console.log("success");
}
// passing non-null if you want to stop async.eachSeries() in case of error
callback(null); // <<<<< need to call async.each()'s callback() here
});
}
else {
//insert product into product table
doInsert(params, function(err, result) {
callback(null); // <<<<< need to call async.each()'s callback() here
});
}
});
});
Also, it's probably good practice to check for error returned.. specifically model.client.query() in this case
Related
I have an array of Courses [course1, course2, ..., courseN]. Each course has a Hero UUID, which matches the UUID of an object in another collection.
// This is actually another db query but imagine it's an array
var courses = [{
"courseName": "Sample course name 1",
"hero": "a3f6f088-7b04-45e8-8d3b-d50c2d5b3a2d"
}, {
"courseName": "Sample course name 2",
"hero": "1b46227a-c496-43d2-be8e-1b0fa07cc94e"
}, {
"courseName": "Sample course name 3",
"hero": "c3bae6bf-2553-473a-9f30-f5c58c4fd608"
}];
I need to iterate over all courses, get the hero uuid and do a query to the Heroes collection then when the query is complete add the hero information to the course object.
The problem is that all queries are fired so rapidly that MongoDB returns them in arbitrary order. It receives all 3 hero uuids in order but it will sometimes return the third one before the first one, etc. Is there a way for one query to complete then do the other one, etc.?
What I am doing right now is:
var newCourses = courses;
var counter = 0;
courses.forEach(function (course) {
var courseHeroUuid = course.hero;
// This function does the query by uuid and returns the doc
getHeroByUuid(courseHeroUuid, function (err, result) {
if (err) {
next(err);
}
// Replace the hero UUID with the hero document itself
newCourses[counter].hero = result[0];
if (++counter == courses.length) {
next(null, newCourses);
}
}
});
This is a function inside an async.waterfall array, this is why I track the counter and call next() to go on. I know I can use async.each for the iteration, I tried it didn't help out.
This is the query I am doing.
function getHeroByUuid(heroUuid, callback) {
Hero.find({uuid: heroUuid}, function (err, result) {
if (err) {
callback(err);
}
callback(null, result);
})
}
This happens:
http://i.imgur.com/mEoQfgH.png
Sorry to answer my own question but I figured it out. What I needed was right under my nose.
I ended up using the async.whilst() function, documentation is right here and does exactly what I need - execute the next iteration of the loop after the result from the previous one is returned.
My code now looks like this:
var newCourses = courses;
var courseItemsLength = courses.length;
var counter = 0;
async.whilst(
function () {
return counter < courseItemsLength;
}, function (callback) {
var heroUuid = allCourses[counter].hero;
getHeroByUuid(heroUuid, function (err, result) {
if (err) {
next(err);
}
newCourses[counter].hero = result.name;
counter++;
if (err) {
callback(err);
}
callback();
});
}, function (err) {
if (err) {
next(err);
}
next(null, newCourses);
});
In the code
var stuff_i_want = '';
stuff_i_want = get_info(parm);
And the function get_info:
get_info(data){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
console.log(stuff_i_want); // Yep. Value assigned..
}
in the larger scope
stuff_i_want = null
What am i missing regarding returning mysql data and assigning it to a variable?
============ New code per Alex suggestion
var parent_id = '';
get_info(data, cb){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
return cb(results[0].objid); // Scope is larger than function
}
==== New Code in Use
get_data(parent_recording, function(result){
parent_id = result;
console.log("Parent ID: " + parent_id); // Data is delivered
});
However
console.log("Parent ID: " + parent_id);
In the scope outside the function parent_id is null
You're going to need to get your head around asynchronous calls and callbacks with javascript, this isn't C#, PHP, etc...
Here's an example using your code:
function get_info(data, callback){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
return callback(results[0].objid);
})
}
//usage
var stuff_i_want = '';
get_info(parm, function(result){
stuff_i_want = result;
//rest of your code goes in here
});
When you call get_info this, in turn, calls connection.query, which takes a callback (that's what function(err, results) is
The scope is then passed to this callback, and so on.
Welcome to javascript callback hell...
It's easy when you get the hang of it, just takes a bit of getting used to, coming from something like C#
I guess what you really want to do here is returning a Promise object with the results. This way you can deal with the async operation of retrieving data from the DBMS: when you have the results, you make use of the Promise resolve function to somehow "return the value" / "resolve the promise".
Here's an example:
getEmployeeNames = function(){
return new Promise(function(resolve, reject){
connection.query(
"SELECT Name, Surname FROM Employee",
function(err, rows){
if(rows === undefined){
reject(new Error("Error rows is undefined"));
}else{
resolve(rows);
}
}
)}
)}
On the caller side, you use the then function to manage fulfillment, and the catch function to manage rejection.
Here's an example that makes use of the code above:
getEmployeeNames()
.then(function(results){
render(results)
})
.catch(function(err){
console.log("Promise rejection error: "+err);
})
At this point you can set up the view for your results (which are indeed returned as an array of objects):
render = function(results){ for (var i in results) console.log(results[i].Name) }
Edit
I'm adding a basic example on how to return HTML content with the results, which is a more typical scenario for Node. Just use the then function of the promise to set the HTTP response, and open your browser at http://localhost:3001
require('http').createServer( function(req, res){
if(req.method == 'GET'){
if(req.url == '/'){
res.setHeader('Content-type', 'text/html');
getEmployeeNames()
.then(function(results){
html = "<h2>"+results.length+" employees found</h2>"
html += "<ul>"
for (var i in results) html += "<li>" + results[i].Name + " " +results[i].Surname + "</li>";
html += "</ul>"
res.end(html);
})
.catch(function(err){
console.log("Promise rejection error: "+err);
res.end("<h1>ERROR</h1>")
})
}
}
}).listen(3001)
Five years later, I understand asynchronous operations much better.
Also with the new syntax of async/await in ES6 I refactored this particular piece of code:
const mysql = require('mysql2') // built-in promise functionality
const DB = process.env.DATABASE
const conn = mysql.createConnection(DB)
async function getInfo(data){
var sql = "SELECT a from b where info = data"
const results = await conn.promise().query(sql)
return results[0]
}
module.exports = {
getInfo
}
Then, where ever I need this data, I would wrap it in an async function, invoke getInfo(data) and use the results as needed.
This was a situation where I was inserting new records to a child table and needed the prent record key, based only on a name.
This was a good example of understanding the asynchronous nature of node.
I needed to wrap the all the code affecting the child records inside the call to find the parent record id.
I was approaching this from a sequential (PHP, JAVA) perspective, which was all wrong.
Easier if you send in a promise to be resolved
e.g
function get_info(data, promise){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
promise.resolve(results[0].objid);
}
}
This way Node.js will stay fast because it's busy doing other things while your promise is waiting to be resolved
I've been working on this goal since few weeks, without any result, and I finally found a way to assign in a variable the result of any mysql query using await/async and promises.
You don't need to understand promises in order to use it, eh, I don't know how to use promises neither anyway
I'm doing it using a Model class for my database like this :
class DB {
constructor(db) {
this.db = db;
}
async getUsers() {
let query = "SELECT * FROM asimov_users";
return this.doQuery(query)
}
async getUserById(array) {
let query = "SELECT * FROM asimov_users WHERE id = ?";
return this.doQueryParams(query, array);
}
// CORE FUNCTIONS DON'T TOUCH
async doQuery(queryToDo) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
async doQueryParams(queryToDo, array) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, array, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
}
Then, you need to instantiate your class by passing in parameter to constructor the connection variable given by mysql. After this, all you need to do is calling one of your class methods with an await before. With this, you can chain queries without worrying of scopes.
Example :
connection.connect(function(err) {
if (err) throw err;
let DBModel = new DB(connection);
(async function() {
let oneUser = await DBModel.getUserById([1]);
let allUsers = await DBModel.getUsers();
res.render("index.ejs", {oneUser : oneUser, allUsers : allUsers});
})();
});
Notes :
if you need to do another query, you just have to write a new method in your class and calling it in your code with an await inside an async function, just copy/paste a method and modify it
there are two "core functions" in the class, doQuery and doQueryParams, the first one only takes a string as a parameter which basically is your mysql query. The second one is used for parameters in your query, it takes an array of values.
it's relevant to notice that the return value of your methods will always be an array of objects, it means that you'll have to do var[0] if you do a query which returns only one row. In case of multiple rows, just loop on it.
I have a table where Persons are saved an their time (in seconds) where they were active.
I would like to write a function that gathers the total time in another table called gather.
For each row I am checking if an entry in the gather table exists. Depending on that result I make an insert or an update.
db.serialize(function() {
db.each("SELECT * from TEST", function(err, row) {
db.get("SELECT * from GATHER where name = " + row.name "", function(err, row) {
if(row === undefined || row === null){
var stmt = db.prepare("INSTER INTO gather (name, time) VALUE(?,?)");
stmt.run([name, seconds], function(error){
console.log('lastInsert ' + this.lastID);
});
stmt.finalize();
}else{
seconds += row.time;//increment time
var stmt = db.prepare("UPDATE gather SET time = ? WHERE name = ?");
stmt.run([seconds, row.name], function(error){
console.log('lastInsert ' + row.idProcessed);
});
stmt.finalize();
}
});
});
});
The problem that I ecounter is that sqlite runs asynchronously. Therefore multiple entries are created in my gather table although lines should be updated.
What would be the right way to run this function sychronously? Should I limit the lines and call the function every second or is there a smarted way?
You could use async. For example (but first you should read final notes at the end):
var async = require('async');
var data = {}
var yourFirstSelect() = function(callback){
//You do your first select
//...
db.get("SELECT * from TEST", function(err, row) {
if(row){
data.name = row.name;
data.otherInterestingAttribute = row.otherInterestingAttribute;
callback(err, data);
}else{
callback('Row is null');
}
})
//..
}
var yourSecondSelect() = function(data, callback){
//You do your second select
//...
db.get("SELECT * from GATHER where name = " + data.name "", function(err, row) { //Note that I'm using data instead of row
if(row){
data.seconds = row.seconds;
data.otherInterestingAttribute = row.otherInterestingAttribute;
callback(err, data);
}else{
callback('Row is null');
}
})
//..
}
var decide() = function(data, callback){
if (data.somethingExists) { //Do your validations
data.type = 'update';
callback(err, data);
} else {
data.type = 'insert';
callback(err, data);
}
}
var update() = function(data,callback){
if (data.type === 'update') {
//...
//Do your stuff in order to update
seconds += row.time;//increment time
var stmt = db.prepare("UPDATE gather SET time = ? WHERE name = ?");
stmt.run([seconds, row.name], function(error){
console.log('update ' + row.idProcessed);
});
stmt.finalize();
//...
} else {
callback(err,data);
}
}
var insert() = function(data,callback){
if (data.type === 'insert') {
//...
//Do your stuff in order to insert
var stmt = db.prepare("INSTER INTO gather (name, time) VALUE(?,?)");
stmt.run([data.name, data.seconds], function(error){
console.log('lastInsert ' + this.lastID);
callback(err,data);
});
stmt.finalize();
//...
} else {
callback(err,data);
}
}
var niceWorkflow = function(){
async.waterfall([
yourFirstSelect,
yourSecondSelect,
decide,
update,
insert
],
function (err, result) {
console.log('Done');
})
}
//and call your workflow
niceWorkflow();
Off course this is not a 100% working code, I wrote it in order you look another way to do what you are trying. Many variables, validations and more are just examples and I intentionally forgot the db.each to avoid being too extense and confusing you and trying to answer your final question Is there a smarter way?.
You can also use Q.all from Q library.
It will return a promise after all the promises are resolved. It one promise fails, then Q.all will be rejected.
I need to query rows from a database, process some information per row, and then update each row with the result.
This is my example code where the intention is to loop over each row and update the label:
var mysql = require('mysql');
var db = mysql.createConnection(config.database);
db.connect(function() {
db.query('SELECT id FROM testTable', function (err, rows) {
if (err) {
console.log(err);
} else {
if (rows.length) {
for (var i = 0, len = rows.length; i < len; i++) {
var row = rows[i];
console.log(row);
var label = "Label_"+row.id;
db.query('UPDATE testTable SET label = ? WHERE id = ?', [label, row.id], function(err, result) {
if (err) {
console.log(err);
} else {
console.log("Set label on row %s", row.id);
}
})
}
}
}
})
});
The output of this is:
{ id: 1 }
{ id: 2 }
{ id: 3 }
{ id: 4 }
Set label on row 4
Set label on row 4
Set label on row 4
Set label on row 4
So, as you can see, I've updated row 4 four times instead of four rows once each. Whilst I new the queries would be non-blocking, I thought the values would change for each one.
I know I can change my code to use rows.forEach(function(){...}) and that then executes each UPDATE one after the other and that would be ok. But to help my understanding I would like to know how I can correctly execute the updates asynchronously.
Your row variable is a closure in the callback function. The callback function doesn't get called until you've looped through all your results list. The sql queries are correct, but printing out the value of row.id in each callback just gives you the last iteration of the for loop each time because that is the state of the closure for every callback.
You can avoid this by using the underscore module. It can also help in making you logic simpler.
npm install underscore
Then your code would look like this:
var mysql = require('mysql');
var _ = require('underscore');
var db = mysql.createConnection(config.database);
db.connect(function() {
db.query('SELECT id FROM testTable', function (err, rows) {
if (err) { console.log(err); return; }
_.each(rows, function(one) {
console.log(one);
var label = "Label_"+one.id;
var sql = 'UPDATE testTable SET label = ? WHERE id = ?';
db.query(sql, [label, one.id], function(err, result) {
if(err) { console.log(err); return; }
console.log("Set label on row %s", one.id);
});
});
});
});
I'm new to Node.js and Async coding. I need to write the equivalent of a nested for loop which will work with Node. I understand that my question is very similar to the one posted here: nested loops asynchronusly in nodejs, next loop must start only after one gets completed, but even after looking at that post in detail, I was unable to fix my code.
I am working with an XML feed. The 'parser' uses the xml2js package. The
loop runs exactly as expected if I remove the sql query (for which I'm using the mysql node package), but when I put the sql query in, then all the orders get processed first, the the "DONE" is output, and then the query fails as it tries to look up items for just the last order repeatedly.
I've tried replacing the for loops with async.forEach loops, but this did not help.
Any help or advice on how to recode this in a way more idiomatic to node would be greatly appreciated.
Many thanks!
Sixhobbits
parser.parseString(data, function (err, result) {
if(err) throw(err);
var numOrders = result['Root']['Orders'][0]['Order'].length;
var curr, currItem, currOrdId, items, sellersCode;
console.log("Logging IDs of", numOrders, "orders");
// for each order
for (var j=0; j<numOrders; j++){
//current order
curr = result['Root']['Orders'][0]['Order'][j];
currOrdId = curr['OrderId'][0]
items = curr['Items'][0]['Item'];
console.log("Order ID:", currOrdId, "--",items.length, "Items");
// for each item
for (var k=0; k<items.length; k++){
currItem = items[k];
sellersCode = currItem['SellersProductCode'][0];
var sqlQuery = 'select data_index, fulltext_id, cataloginventory_stock_item.product_id from catalogsearch_fulltext inner join cataloginventory_stock_item where catalogsearch_fulltext.data_index like "' + sellersCode + '|%"' + 'and cataloginventory_stock_item.item_id = catalogsearch_fulltext.product_id';
var query = connection.query(sqlQuery,function(err,rows,fields){
if (err) throw(err);
console.log(" Item ID :",currItem['ItemId'][0]);
console.log(" Full Text ID :", rows[0]['fulltext_id']);
console.log(" Product ID :", rows[0]['product_id']);
});
}//for
}//for
console.log("DONE");
});//parseString
You were on the right track by looking to use async.forEach. Here's how you would rework this code to use that:
parser.parseString(data, function (err, result) {
if(err) throw(err);
var numOrders = result['Root']['Orders'][0]['Order'].length;
var currOrdId, items, sellersCode;
console.log("Logging IDs of", numOrders, "orders");
// for each order
async.forEach(result['Root']['Orders'][0]['Order'], function (curr, callback1) {
currOrdId = curr['OrderId'][0];
items = curr['Items'][0]['Item'];
console.log("Order ID:", currOrdId, "--",items.length, "Items");
async.forEach(items, function (currItem, callback2) {
sellersCode = currItem['SellersProductCode'][0];
var sqlQuery = 'select data_index, fulltext_id, cataloginventory_stock_item.product_id from catalogsearch_fulltext inner join cataloginventory_stock_item where catalogsearch_fulltext.data_index like "' + sellersCode + '|%"' + 'and cataloginventory_stock_item.item_id = catalogsearch_fulltext.product_id';
var query = connection.query(sqlQuery,function(err,rows,fields){
console.log(" Item ID :",currItem['ItemId'][0]);
console.log(" Full Text ID :", rows[0]['fulltext_id']);
console.log(" Product ID :", rows[0]['product_id']);
callback2(err);
});
}, callback1);
}, function (err) {
console.log("DONE");
});
});//parseString
Each iteration of async.forEach must call its callback parameter when all of its async processing has completed. You've got two levels in this case which makes it a little more difficult to keep track of in your head, but it's the same concept.
This is a classic closure-in-a-loop problem. You need to break the closure by passing currItem as an argument:
for (var k=0; k<items.length; k++){
currItem = items[k];
sellersCode = currItem['SellersProductCode'][0];
var sqlQuery = 'select data_index, fulltext_id, cataloginventory_stock_item.product_id from catalogsearch_fulltext inner join cataloginventory_stock_item where catalogsearch_fulltext.data_index like "' + sellersCode + '|%"' + 'and cataloginventory_stock_item.item_id = catalogsearch_fulltext.product_id';
var query = connection.query(sqlQuery,(function(CI){
return function(err,rows,fields){
if (err) throw(err);
console.log(" Item ID :",CI['ItemId'][0]);
console.log(" Full Text ID :", rows[0]['fulltext_id']);
console.log(" Product ID :", rows[0]['product_id']);
}
})(currItem)); // Break closure by passing currItem as argument
}//for
I realize this is an old post, but you might find this function useful
eachKVAsync = function(elements,userInfo,onKeyValue,ondone) {
var onDone = ondone;
var ret = null;
var done=false;
var isArray = typeof elements.forEach===$f$;
var keys = isArray ? null : [],
values = isArray ? elements : [];
if (keys) {
for (var k in elements) {
keys.push(k);
values.push(elements[k]);
}
}
var aborted=false;
var endLoop = function (userInfo){
aborted=true;
if (onDone) {
onDone(userInfo,aborted);
onDone = null;
}
}
var i = 0;
var iterate = function (userInfo) {
if (i < values.length) {
var ix=i;
i++;
onKeyValue((keys?keys[ix]:i),values[ix],userInfo,iterate,endLoop);
} else {
if (onDone) {
onDone(userInfo,aborted);
onDone = null;
return;
}
}
}
iterate(userInfo);
},
use example
eachKVAsync(
elements, {
aValue: 2004
},
function onItem(key, value, info, nextItem, endLoop) {
if (value.isOk) {
info.aValue += value.total;
setTimeout(nextItem,1000,info);
} else {
endLoop(info);
}
},
function afterLastItem(info, loopEndedEarly) {
if (!loopEndedEarly) {
console.log(info.aValue);
}
}
);