node.js : For each over rows and update asynchronously? - node.js

I need to query rows from a database, process some information per row, and then update each row with the result.
This is my example code where the intention is to loop over each row and update the label:
var mysql = require('mysql');
var db = mysql.createConnection(config.database);
db.connect(function() {
db.query('SELECT id FROM testTable', function (err, rows) {
if (err) {
console.log(err);
} else {
if (rows.length) {
for (var i = 0, len = rows.length; i < len; i++) {
var row = rows[i];
console.log(row);
var label = "Label_"+row.id;
db.query('UPDATE testTable SET label = ? WHERE id = ?', [label, row.id], function(err, result) {
if (err) {
console.log(err);
} else {
console.log("Set label on row %s", row.id);
}
})
}
}
}
})
});
The output of this is:
{ id: 1 }
{ id: 2 }
{ id: 3 }
{ id: 4 }
Set label on row 4
Set label on row 4
Set label on row 4
Set label on row 4
So, as you can see, I've updated row 4 four times instead of four rows once each. Whilst I new the queries would be non-blocking, I thought the values would change for each one.
I know I can change my code to use rows.forEach(function(){...}) and that then executes each UPDATE one after the other and that would be ok. But to help my understanding I would like to know how I can correctly execute the updates asynchronously.

Your row variable is a closure in the callback function. The callback function doesn't get called until you've looped through all your results list. The sql queries are correct, but printing out the value of row.id in each callback just gives you the last iteration of the for loop each time because that is the state of the closure for every callback.
You can avoid this by using the underscore module. It can also help in making you logic simpler.
npm install underscore
Then your code would look like this:
var mysql = require('mysql');
var _ = require('underscore');
var db = mysql.createConnection(config.database);
db.connect(function() {
db.query('SELECT id FROM testTable', function (err, rows) {
if (err) { console.log(err); return; }
_.each(rows, function(one) {
console.log(one);
var label = "Label_"+one.id;
var sql = 'UPDATE testTable SET label = ? WHERE id = ?';
db.query(sql, [label, one.id], function(err, result) {
if(err) { console.log(err); return; }
console.log("Set label on row %s", one.id);
});
});
});
});

Related

Get count of items that matches a condition from mongo db in node js

I am new to mongo db. I have a scenario where I need to check into collection and the count of items having gameDate = current date.
I am using the following query in node js.
const count = this.games.find({gameDate:currentDate}).count();
console.log(count + "items present")
I have 1 matching record in database. But instead of getting count am getting the following error in console.
function(filter, callback) {
this.op = 'count';
if (typeof filter === 'function') {
callback = filter;
filter = undefined;
}
filter = utils.toObject(filter);
if (mquery.canMerge(filter)) {
this.merge(filter);
}
if (!callback) {
return this;
}
this.exec(callback);
return this;
}
How can I get the count here.
You can try the countDocuments function.
games.countDocuments({gameDate:currentDate}, function (err, count) {
if (err){
console.log(err)
}else{
console.log("Total:", count)
}
});

How do I remove the keys, but leave the values of result array? [duplicate]

This question already has answers here:
From an array of objects, extract value of a property as array
(24 answers)
Closed 3 years ago.
I'm pulling multiple rows from a node.js MySQL query, and I would like the format to be:
["value1","value2","value3"]
however my current result is:
[{"temperature":value1},{"temperature":value2},{"temperature":value3}]
I don't want the key, it's all data from the same MySQL column.
var sql = "SELECT temperature FROM temperatures";
con.query(sql, function (err, result) {
if (err) throw err;
console.log(result);
let data = JSON.stringify(result);
fs.writeFileSync('results.json', data);
I got time to rewrite the code. Thanks all, the map function worked excellently.
var sql = "SELECT temperature FROM temperatures";
con.query(sql, function (err, result) {
if (err) throw err;
var newresult = result.map( value => value.temperature);
let data = JSON.stringify(newresult);
console.log('Queried JSON Data: \n' + data + '\n');
fs.writeFileSync('results.json', data);
});
};
you can use forEach or Map
var kvArray = [{"temperature":"value1"},{"temperature":"value1"},{"temperature":"value1"}];
//example for map.
var rObj = kvArray.map( value => value.temperature);
//example for foreach.
var rObj = [];
kvArray.forEach( (value) => {
return rObj.push(value.temperature);
});
console.log(kvArray , rObj)
// rObj = (3) ["value1", "value1", "value1"]
// kvArray is still:
//[{"temperature":"value1"},{"temperature":"value1"},{"temperature":"value1"}];
I'm scanning the array of dictionaries and each time joining the values of each dictionary. This is more general approach, meaning even if you have multiple values in each dictionary it will work the same.
let make_results = (result) => {
let values_only = [];
result.forEach((r) => {
values_only = values_only.concat(Object.values(r));
});
return values_only;
}
let result = [{
"temperature": 1
}, {
"temperature": 2
}, {
"temperature": 3
}];
console.log(make_results(result));
Then I'm just using make_results function in your code:
var sql = "SELECT temperature FROM temperatures";
con.query(sql, function (err, result) {
if (err) throw err;
console.log(result);
let data = JSON.stringify(make_results);
fs.writeFileSync('results.json', data);

async.each table insertion trouble?

I wrote a code using async.each for inserting data into tables.
var categoryList = [{"categoryName": "biriyani","productName":"chicken biriyani"}, {"categoryName":"biriyani","productName":"mutton biriyani"}]
async.each(categoryList, function(item,callback)
{
var categoryName=item.categoryName;
var productName=item.productName;
var categoryCheckQuery = pgFormat("select * from shop where categoryName LIKE '%"+categoryName+"%'");
model.client.query(categoryCheckQuery,function (err,result) {
if(result.rowCount==0){
var insertCategoryQuery = pgFormat("insert into shop(categoryName)values(%L)",categoryName);
model.client.query(insertCategoryQuery,function (err,result) {
if (!err) {
console.log("success");
}
});
}
else{
//insert product into product table
}
});
Explanation:
1)Here first Json array containing categoryName->biriyani is entered into shop table
2)when async.each fetching next json array containing categoryName->biriyani,
categoryCheckQuery checks the shop table whether categoryname = 'biriyani' is already exists.
3)If exists it wont be saved
Problem:
Here for both the data result.rowCount ==0 and both the data which have categoryname = biriyani is entered into shop table.
There are couple problems in this code.
One is the use of async.each(), async.eachSeries() should be used instead of async.each() because the operation of the next item in categoryList depends on current item's operation. async.eachSeries() ensures that first item is done before moving on to next item.
Another is async.each()'s callback() should be called to signal it that you're done with it.
Here is the revised code:
var categoryList = [{"categoryName": "biriyani","productName":"chicken biriyani"}, {"categoryName":"biriyani","productName":"mutton biriyani"}]
// Use async.eachSerices() instead of async.each()
async.eachSeries(categoryList, function(item,callback) {
var categoryName = item.categoryName;
var productName = item.productName;
var categoryCheckQuery = pgFormat("select * from shop where categoryName LIKE '%" + categoryName + "%'");
model.client.query(categoryCheckQuery, function (err, result) {
if (result.rowCount == 0) {
var insertCategoryQuery = pgFormat("insert into shop(categoryName)values(%L)", categoryName);
model.client.query(insertCategoryQuery, function (err, result) {
if (!err) {
console.log("success");
}
// passing non-null if you want to stop async.eachSeries() in case of error
callback(null); // <<<<< need to call async.each()'s callback() here
});
}
else {
//insert product into product table
doInsert(params, function(err, result) {
callback(null); // <<<<< need to call async.each()'s callback() here
});
}
});
});
Also, it's probably good practice to check for error returned.. specifically model.client.query() in this case

nodejs get sqlite3 query result using promise or wait

This is my first personal project in Nodejs. I'm trying to get in live soon.
I have a Nodejs server that uses sqlite3. There are only 3000 rows with word, transform and a precalculated value each in a column of the table, which is already populated.
I need to just lookup the word in the DB to be sure it is valid.
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.get("SELECT * FROM tab WHERE w = ?", word, function(err, row) {
if(err) { console.log("Lookup:",word,", Error => ",err); return false; }
return true;
});
The problem is that the caller of this code has a lot of context and need the operation to wait. So, I tried this
function dbLookup(db, w) {
return function(cb) {
var rows = [];
db.exec('SELECT w FROM tab WHERE w = "'+w+'"')
.on('row', function(r) {
rows.push(r)
})
.on('result', function() {
cb(rows);
});
}
async.each([word], function(w) {
dbLookup(this.db, w);
}, function(err) {
if(err) {console.log("...ERROR..."); return false; }
else {console.log("...SUCCESS..."); return true; }
});
This doesn't solve the wait issue as the callback can fire at its own pace.
I read that promise using something like bluebird can solve my problem
but now I'm not able to get the value/result of the query out:
I've been pulling my hair for so long. Please help me either get the async working or get the result back from the promise approach.
var async = require('async');
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
function check(word, callback) {
db.get("SELECT count(1) cnt FROM tab WHERE w = ?", word, callback)
}
async.map(words, check, function(err, results) {
if (err)
return console.log('Query error')
var all_checked = results.filter(function(r) {
return r.cnt > 0
});
...
});
Or
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.all("SELECT distinct w FROM tab", function(err, rows) {
var all_checked = words.filter(function (w) {
return rows.indexOf(w) != -1;
})
...
})

node.js synchronous call gather results

I have a table where Persons are saved an their time (in seconds) where they were active.
I would like to write a function that gathers the total time in another table called gather.
For each row I am checking if an entry in the gather table exists. Depending on that result I make an insert or an update.
db.serialize(function() {
db.each("SELECT * from TEST", function(err, row) {
db.get("SELECT * from GATHER where name = " + row.name "", function(err, row) {
if(row === undefined || row === null){
var stmt = db.prepare("INSTER INTO gather (name, time) VALUE(?,?)");
stmt.run([name, seconds], function(error){
console.log('lastInsert ' + this.lastID);
});
stmt.finalize();
}else{
seconds += row.time;//increment time
var stmt = db.prepare("UPDATE gather SET time = ? WHERE name = ?");
stmt.run([seconds, row.name], function(error){
console.log('lastInsert ' + row.idProcessed);
});
stmt.finalize();
}
});
});
});
The problem that I ecounter is that sqlite runs asynchronously. Therefore multiple entries are created in my gather table although lines should be updated.
What would be the right way to run this function sychronously? Should I limit the lines and call the function every second or is there a smarted way?
You could use async. For example (but first you should read final notes at the end):
var async = require('async');
var data = {}
var yourFirstSelect() = function(callback){
//You do your first select
//...
db.get("SELECT * from TEST", function(err, row) {
if(row){
data.name = row.name;
data.otherInterestingAttribute = row.otherInterestingAttribute;
callback(err, data);
}else{
callback('Row is null');
}
})
//..
}
var yourSecondSelect() = function(data, callback){
//You do your second select
//...
db.get("SELECT * from GATHER where name = " + data.name "", function(err, row) { //Note that I'm using data instead of row
if(row){
data.seconds = row.seconds;
data.otherInterestingAttribute = row.otherInterestingAttribute;
callback(err, data);
}else{
callback('Row is null');
}
})
//..
}
var decide() = function(data, callback){
if (data.somethingExists) { //Do your validations
data.type = 'update';
callback(err, data);
} else {
data.type = 'insert';
callback(err, data);
}
}
var update() = function(data,callback){
if (data.type === 'update') {
//...
//Do your stuff in order to update
seconds += row.time;//increment time
var stmt = db.prepare("UPDATE gather SET time = ? WHERE name = ?");
stmt.run([seconds, row.name], function(error){
console.log('update ' + row.idProcessed);
});
stmt.finalize();
//...
} else {
callback(err,data);
}
}
var insert() = function(data,callback){
if (data.type === 'insert') {
//...
//Do your stuff in order to insert
var stmt = db.prepare("INSTER INTO gather (name, time) VALUE(?,?)");
stmt.run([data.name, data.seconds], function(error){
console.log('lastInsert ' + this.lastID);
callback(err,data);
});
stmt.finalize();
//...
} else {
callback(err,data);
}
}
var niceWorkflow = function(){
async.waterfall([
yourFirstSelect,
yourSecondSelect,
decide,
update,
insert
],
function (err, result) {
console.log('Done');
})
}
//and call your workflow
niceWorkflow();
Off course this is not a 100% working code, I wrote it in order you look another way to do what you are trying. Many variables, validations and more are just examples and I intentionally forgot the db.each to avoid being too extense and confusing you and trying to answer your final question Is there a smarter way?.
You can also use Q.all from Q library.
It will return a promise after all the promises are resolved. It one promise fails, then Q.all will be rejected.

Resources