how to promises in a loop? - node.js

I am trying write a cron function in nodejs which fetches user_ids of all the users from the db and then I want to parse through each user_id.
Here is my code :
cron.schedule('43 11 * * *', function(){
var now = moment()
var formatted = now.format('YYYY-MM-DD HH:mm:ss')
console.log('Starting the cron boss!');
var dbSelectPromise = function(db, sql1) {
return new Promise(function(resolve, reject) {
db.select(sql1, function(err, data) {
if (err) {
reject(err)
} else {
resolve(data)
}
})
})
}
var users =[]
var sql = "select distinct(user_id) from user_level_task"
dbSelectPromise(db,sql).then(function(secondResult){
for(i=0;i<secondResult.length;i++){
var sql1 = "select max(level_id) as level from user_level_task where user_id ="+secondResult[i].user_id
dbSelectPromise(db,sql1).then(function(thirdResult){
console.log(thirdResult)
console.log(current)
var sql2 = "select task_id form user_level_task where user_id = '"+secondResult[i].user_id+"' and level_id = '"+thirdResult[0].level+"' "
dbSelectPromise(db,sql2).then(function(fourthResult){
var leng = fourthResult.length
for(i=0;i<leng;i++){
console.log(fourthResult[i])
}
})
})
}
})
});
The problem i am facing is i cannot access value of i in third and fourth promises. Please help!

I think what's happening is that i is no longer the same when you create those new promises because the for loop is still running. It appears that what you really need is the user_id and level_id. I suggest you restructure your code a bit to reduce nesting and pass on the values you need for future promises.
Perhaps something similar to this:
dbSelectPromise(db, sql)
.then(secondResult => {
const levelPromises = [];
secondResult.forEach(res => {
levelPromises.push(getLevelByUserId(res.user_id, db));
});
return Promise.all(levelPromises); // Promise.all only if you want to handle all success cases
})
.then(result => {
result.forEach( level => {
{ userId, queryResult } = level;
// ...
})
//...
})
.catch(err => {
console.log(err);
});
function getLevelByUserId(userId, db) {
const query = `select max(level_id) as level from user_level_task where user_id = ${userId}`;
return dbselectPromise(db, query).then(result => { userId, result });
}
It creates an array of all the get level queries as promises and then passes it along to the next step using Promise.all() which will only resolve if all queries were successful. At that point, you will have access to the userId again of each result because we returned it in our new function for your next set of queries.
I think you should abstract your queries a bit further instead of using a generic dbSelectPromise and don't forget to catch() at the end otherwise you won't know what's happening.
Note: It assumes your db variable instantiated properly and your original db.select doesn't need to be returned based on whatever library you're using. There's also some new syntax there.

The problem i am facing is i cannot access value of i in third and fourth promises. Please help!
This is because you're using reinitializing i without using let. When the loop is in process, the value will be different than what you expect.
each promise is dependant on the other and need to run synchronously
For this to work, You need to chain promises. Also, you can make use of Promise.all() to execute a bunches of promises at once. Remember, Promise.all() is all or nothing.
Making those changes to your code, I get the following structure.
'use strict';
let _ = require('lodash');
function dbSelectPromise(db, sql1) {
return new Promise((resolve, reject) => {
return db.select(sql1, (err, data) => {
if (err) {
return reject(err);
}
return resolve(data);
});
});
}
function process(secondResult) {
let sql1 = "select max(level_id) as level from user_level_task where user_id =" + secondResult[i].user_id;
return dbSelectPromise(db, sql1).then(function (thirdResult) {
console.log(thirdResult);
let sql2 = "select task_id form user_level_task where user_id = '" + secondResult[i].user_id + "' and level_id = '" + thirdResult[0].level + "' ";
return dbSelectPromise(db, sql2);
});
}
function getUsers() {
let sql = "select distinct(user_id) from user_level_task";
return dbSelectPromise(db, sql).then((users) => {
return users;
}).catch(() => {
return [];
});
}
cron.schedule('43 11 * * *', function () {
var now = moment();
var formatted = now.format('YYYY-MM-DD HH:mm:ss');
getUsers().then((users) => {
let batches = _.map(users, function (user) {
return process(user);
});
return Promise.all(batches);
}).then((fourthResult) => {
console.log('Your fourthResult [[],..]', fourthResult);
}).catch(() => {
console.log('err while processing', err);
});
});

Related

Azure Redis Cache cache.set() Not operable within Promise.then?

This one has had me spinning my tires for about 2 days now, I'm ready to reach out for help :)
I have a google firebase functions app, running as a middle-ware to an angular SPA. Hoping to avoid some of the pay-by-use cost of Azure SQL, I wanted to implement a caching option for the most common queries.
I thought I knew redis, I've worked with it before. There's a simple enough example on the repo: https://www.npmjs.com/package//redis
Everything works fine, if it is top level.
But the way my application is built, i need the ability to set a cache value from within the .then of a Promise, and when I try to do that, all operation just stops, with no identifiable error logging, or even response from redis. Even in Azure insights, i'm not getting much feed-back, only that the 'set' operation isn't being counted in metrics.
So, just to clarify, this works:
// "cache", the object, set globally
export const testCache = functions.https.onRequest(
async (req: any, res) => {
await cache.connect();
var redisKey = 'testing_global';
var result = await cache.get(redisKey);
await cache.set(redisKey, 'testing new class')
console.log("\nDone");
cache.disconnect();
res.send('done');
})
But, this does not:
import { createClient } from 'redis';
const cache = createClient({
url: "rediss://" + process.env.REDIS_HOST_NAME + ":6380",
password: process.env.REDIS_KEY,
});
export const getValues = functions.https.onRequest(
(req: any, response) => {
cors(req, response, async () => {
response.set('Access-Control-Allow-Origin', origin);
var searchText = req.body['search'];
var offset = req.body['offset'];
var fetch = req.body['fetch'];
var x = req.body['x'];
var y = req.body['y'];
var districts = req.body['districtFilter'];
var sort = req.body['sort'];
if (offset) {
if (!/^\d+$/.test(offset))
throw new Error('bad number');
} else {
offset = 0;
}
if (fetch) {
if (!/^\d+$/.test(fetch))
throw new Error('bad number');
} else {
fetch = 200;
}
if (x) {
if (!/^-?\d+$/.test(x))
throw new Error('bad number');
} else {
x = null;
}
if (y) {
if (!/^-?\d+$/.test(y))
throw new Error('bad number');
} else {
y = null;
}
if (districts && districts.length > 0) {
districts = sanitizeStringArray(districts);
} else {
districts = null;
}
if (sort) {
switch (sort) {
case "scoreAsc":
case "scoreDesc":
case "priceAsc":
case "priceDesc":
break;
default:
sort = '';
}
}
var redisKey = `get_values_${searchText}_${offset}_${fetch}_${x}_${y}_${districts}_${sort}`;
var cacheResult: any = null;
await cache.connect();
// because I want to end up in the 'else', for testing
cacheResult = await cache.getFromCache('someOtherKey');
if (null !== cacheResult) {
response.send({
"status": "success",
"totalCount": cacheResult.totalCount,
"data": cacheResult.result
});
cache.disconnect();
} else {
var connection = new Connection(sqlConfig);
var totalCount: number = 0;
connection.on('connect', function (err: any) {
// If no error, then good to proceed.
console.log("Connected");
var sql = `EXEC SomeSPC;`;
const sqlRequest = new Request(sql, function (err: any) {
if (err) {
console.log(err);
}
});
const countRequest = new Request(
`EXEC SomeOtherSPC;`
, function (err: any) {
if (err) {
console.log(err);
}
}
)
sqlRequest.connection = connection;
countRequest.connection = connection;
var result: any[] = [];
sqlRequest.on('row', function (columns: any[]) {
var rowResult: any = {};
columns.forEach(function (column: any) {
rowResult[column['metadata']['colName']] = column['value'];
});
result.push(rowResult);
});
sqlRequest.on("requestCompleted", function (rowCount: any, more: any) {
console.log(rowCount + ' rows returned');
connection.execSql(countRequest);
countRequest.on('row', function (columns: any[]) {
totalCount = columns[0]['value'];
});
countRequest.on('requestCompleted', async function (rowCount: any, more: any) {
connection.close();
cacheResult = {
totalCount: totalCount
, result: result
};
// ******************************************************************
// Does Not Work, Just Fails, Without Much to Go On
await cache.set(redisKey, 'in Promise')
response.send({
"status": "success",
"totalCount": totalCount,
"data": result
});
})
});
connection.execSql(sqlRequest);
});
connection.on('infoMessage', infoError);
connection.on('errorMessage', infoError);
connection.on('end', end);
connection.on('debug', debug);
connection.connect();
console.log("Reading rows from the Table...");
}
})
}
)
There's **update- no longer ** a fair amount of psuedo-code here, so please **update- No Need to ** disregard any inconsistent lines. I went ahead and put in the full function, including all the fluff, since trimming the fat seems to make things difficult for others to understand what is being asked.
The sql stuff all works, if i take out the cache.set() everything is fine, but that line, in the result of the Promise, just fails, and I can't figure out why.
I've tried using cache locally and globally, extracting the cache operations to a function, and then to a separate class, and in all cases, i'm getting the same result.
Is there a known reason this wouldn't work?
As far as I can understand, given you didn't provide a reproducible example, your code is binding to the requestCompleted event for the second request only after it runs execSql(): I would suggest moving the binding block before that, otherwise the event may be skipped.

in sqlite3 (node package), how to run multiple queries by sequence?

I am using this pakcage: sqlite3 to manipulate database queries, code like this:
single query is quit simple:
var sqlite3 = require('sqlite3').verbose();
let db = new sqlite3.Database('database.sqlite3');
let entity_1_name = ''
db.serialize(function(){
db.get('select * from table1 where id = 1', function(error, row){
entity_1_name = row.name
})
})
db.close()
But how to execute multiple queries and in sequence?
(e.g. I want to run query1, then query2, then query3 )
just like:
var sqlite3 = require('sqlite3').verbose();
let db = new sqlite3.Database('database.sqlite3');
let entity_1_name = ''
let entity_2_name = ''
db.serialize(function(){
// this is query1
db.get('select * from table1 where id = 1', [], function(error, row){
entity_1_name = row.name
})
// this is query2
db.get('select * from table2 where dependency_column_name = ' + entity_1_name, function(error,row){
// this sql returns null value, because when this query was executed, the variable "entity_1_name" was not determined by query1.
})
})
db.close()
https://stackoverflow.com/a/40903302/16140221 May help you out
Their solution was to place each query within a function, then simply calling them.
(This is directly the code taken from the answer, it is not mine.)
function db_query1(your_param,...., callback){
  // database operation
db.run( sql , [param,...] , function(err,rows){
if(err) // return
else{
// get rows with callback
callback(null, rows);
}
});
}
function db_query2(your_param,...., callback){
  // database operation
db.run( sql , [param,...] , function(err,rows){
if(err) // return
else{
// get rows with callback
callback(null, rows);
}
});
}
Then calling the functions
db_query1(....,function(err,result1){
if(err) ...// return
// do the things with result1
// And then call query2
db_query2(....,function(err,result2){
if(err) ...// return
// do the things with result1
});
});
OK, at last, I used promise + async + await to make it done.
Step1. define a function return new Promise()
do_query_1(material_name){
// 1. return this new Promise()
return new Promise( (resolve, reject) => {
let db = this.$database_tool.get_db()
let that = this
db.serialize(function(){
db.get(`select * from table1 where id = 1`, [], function(error, row){
// 2. this is the point, put anything you want to result to resolve()
resolve(row.id)
})
})
db.close();
})
},
Step2. define a async function, to call the method defined in Step1.
async do_query_2(){
let that = this
// other lines was omitted.
// HERE , use await to get the result from STEP1.
let entity_1_name = await that.do_query_1(row.material_name)
db.serialize(function(){
let sql = `'select * from table2 where dependency_column_name = ' + entity_1_name`
db.run(sql)
})
db.close();
},
Step3. call do_query_2() in normal way:
do_query_2()
That's it!

Node waiting for sqlite to return results

I'm fairly new to Node from Python so I'm used to my code executing in the order in which I write it and this has got me a little stumped.
I'm basically getting rows back from my SQLite database and performing an operation on each row then keeping a running total. I then want the final total to reply back to the user (This is a Discord bot)
I've attempted to wrap my code inside a Promise but not when it runs my bot crashes saying 'query is not defined'
async function feelingCommand(username,receivedMessage){
let sqlStatement = `SELECT Message FROM discord WHERE User = ? ORDER BY Time DESC LIMIT 2`;
let total = 0;
const query = await new Promise((resolve,reject) => {
db.run(sqlStatement, (err) => {
if(err) {
return console.log(err.message);
reject(err);
}
db.each(sqlStatement, [username], (err, row) => {
if (err) {
reject(err);
}
let score = sentiment.analyze(`${row.Message}`).comparative;
total = total + score;
});
resolve(query);
});
});
console.log(total)
}
I'd be very grateful for any help in pointing me in the right direction. To be honest, I'm surprised I got this far :)
Well my addition searching indicated that db.each is a pain to promisify so I switched to using db.all and a foreach loop with a promise
async function feelingCommand(username,receivedMessage){
let sqlStatement = `SELECT Message FROM discord WHERE User = ? ORDER BY Time DESC LIMIT 2`;
let total = 0;
const query = await new Promise((resolve,reject) => {
db.all(sqlStatement, [username], (err, rows) => {
if (err) {
reject(err);
}
rows.forEach((row) => {
console.log(row.name);
let score = sentiment.analyze(`${row.Message}`).comparative;
total = total + score;
});
resolve(total);
});
})

nodejs get sqlite3 query result using promise or wait

This is my first personal project in Nodejs. I'm trying to get in live soon.
I have a Nodejs server that uses sqlite3. There are only 3000 rows with word, transform and a precalculated value each in a column of the table, which is already populated.
I need to just lookup the word in the DB to be sure it is valid.
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.get("SELECT * FROM tab WHERE w = ?", word, function(err, row) {
if(err) { console.log("Lookup:",word,", Error => ",err); return false; }
return true;
});
The problem is that the caller of this code has a lot of context and need the operation to wait. So, I tried this
function dbLookup(db, w) {
return function(cb) {
var rows = [];
db.exec('SELECT w FROM tab WHERE w = "'+w+'"')
.on('row', function(r) {
rows.push(r)
})
.on('result', function() {
cb(rows);
});
}
async.each([word], function(w) {
dbLookup(this.db, w);
}, function(err) {
if(err) {console.log("...ERROR..."); return false; }
else {console.log("...SUCCESS..."); return true; }
});
This doesn't solve the wait issue as the callback can fire at its own pace.
I read that promise using something like bluebird can solve my problem
but now I'm not able to get the value/result of the query out:
I've been pulling my hair for so long. Please help me either get the async working or get the result back from the promise approach.
var async = require('async');
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
function check(word, callback) {
db.get("SELECT count(1) cnt FROM tab WHERE w = ?", word, callback)
}
async.map(words, check, function(err, results) {
if (err)
return console.log('Query error')
var all_checked = results.filter(function(r) {
return r.cnt > 0
});
...
});
Or
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.all("SELECT distinct w FROM tab", function(err, rows) {
var all_checked = words.filter(function (w) {
return rows.indexOf(w) != -1;
})
...
})

How to properly return a result from mysql with Node?

In the code
var stuff_i_want = '';
stuff_i_want = get_info(parm);
And the function get_info:
get_info(data){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
console.log(stuff_i_want); // Yep. Value assigned..
}
in the larger scope
stuff_i_want = null
What am i missing regarding returning mysql data and assigning it to a variable?
============ New code per Alex suggestion
var parent_id = '';
get_info(data, cb){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
return cb(results[0].objid); // Scope is larger than function
}
==== New Code in Use
get_data(parent_recording, function(result){
parent_id = result;
console.log("Parent ID: " + parent_id); // Data is delivered
});
However
console.log("Parent ID: " + parent_id);
In the scope outside the function parent_id is null
You're going to need to get your head around asynchronous calls and callbacks with javascript, this isn't C#, PHP, etc...
Here's an example using your code:
function get_info(data, callback){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
return callback(results[0].objid);
})
}
//usage
var stuff_i_want = '';
get_info(parm, function(result){
stuff_i_want = result;
//rest of your code goes in here
});
When you call get_info this, in turn, calls connection.query, which takes a callback (that's what function(err, results) is
The scope is then passed to this callback, and so on.
Welcome to javascript callback hell...
It's easy when you get the hang of it, just takes a bit of getting used to, coming from something like C#
I guess what you really want to do here is returning a Promise object with the results. This way you can deal with the async operation of retrieving data from the DBMS: when you have the results, you make use of the Promise resolve function to somehow "return the value" / "resolve the promise".
Here's an example:
getEmployeeNames = function(){
return new Promise(function(resolve, reject){
connection.query(
"SELECT Name, Surname FROM Employee",
function(err, rows){
if(rows === undefined){
reject(new Error("Error rows is undefined"));
}else{
resolve(rows);
}
}
)}
)}
On the caller side, you use the then function to manage fulfillment, and the catch function to manage rejection.
Here's an example that makes use of the code above:
getEmployeeNames()
.then(function(results){
render(results)
})
.catch(function(err){
console.log("Promise rejection error: "+err);
})
At this point you can set up the view for your results (which are indeed returned as an array of objects):
render = function(results){ for (var i in results) console.log(results[i].Name) }
Edit
I'm adding a basic example on how to return HTML content with the results, which is a more typical scenario for Node. Just use the then function of the promise to set the HTTP response, and open your browser at http://localhost:3001
require('http').createServer( function(req, res){
if(req.method == 'GET'){
if(req.url == '/'){
res.setHeader('Content-type', 'text/html');
getEmployeeNames()
.then(function(results){
html = "<h2>"+results.length+" employees found</h2>"
html += "<ul>"
for (var i in results) html += "<li>" + results[i].Name + " " +results[i].Surname + "</li>";
html += "</ul>"
res.end(html);
})
.catch(function(err){
console.log("Promise rejection error: "+err);
res.end("<h1>ERROR</h1>")
})
}
}
}).listen(3001)
Five years later, I understand asynchronous operations much better.
Also with the new syntax of async/await in ES6 I refactored this particular piece of code:
const mysql = require('mysql2') // built-in promise functionality
const DB = process.env.DATABASE
const conn = mysql.createConnection(DB)
async function getInfo(data){
var sql = "SELECT a from b where info = data"
const results = await conn.promise().query(sql)
return results[0]
}
module.exports = {
getInfo
}
Then, where ever I need this data, I would wrap it in an async function, invoke getInfo(data) and use the results as needed.
This was a situation where I was inserting new records to a child table and needed the prent record key, based only on a name.
This was a good example of understanding the asynchronous nature of node.
I needed to wrap the all the code affecting the child records inside the call to find the parent record id.
I was approaching this from a sequential (PHP, JAVA) perspective, which was all wrong.
Easier if you send in a promise to be resolved
e.g
function get_info(data, promise){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
promise.resolve(results[0].objid);
}
}
This way Node.js will stay fast because it's busy doing other things while your promise is waiting to be resolved
I've been working on this goal since few weeks, without any result, and I finally found a way to assign in a variable the result of any mysql query using await/async and promises.
You don't need to understand promises in order to use it, eh, I don't know how to use promises neither anyway
I'm doing it using a Model class for my database like this :
class DB {
constructor(db) {
this.db = db;
}
async getUsers() {
let query = "SELECT * FROM asimov_users";
return this.doQuery(query)
}
async getUserById(array) {
let query = "SELECT * FROM asimov_users WHERE id = ?";
return this.doQueryParams(query, array);
}
// CORE FUNCTIONS DON'T TOUCH
async doQuery(queryToDo) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
async doQueryParams(queryToDo, array) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, array, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
}
Then, you need to instantiate your class by passing in parameter to constructor the connection variable given by mysql. After this, all you need to do is calling one of your class methods with an await before. With this, you can chain queries without worrying of scopes.
Example :
connection.connect(function(err) {
if (err) throw err;
let DBModel = new DB(connection);
(async function() {
let oneUser = await DBModel.getUserById([1]);
let allUsers = await DBModel.getUsers();
res.render("index.ejs", {oneUser : oneUser, allUsers : allUsers});
})();
});
Notes :
if you need to do another query, you just have to write a new method in your class and calling it in your code with an await inside an async function, just copy/paste a method and modify it
there are two "core functions" in the class, doQuery and doQueryParams, the first one only takes a string as a parameter which basically is your mysql query. The second one is used for parameters in your query, it takes an array of values.
it's relevant to notice that the return value of your methods will always be an array of objects, it means that you'll have to do var[0] if you do a query which returns only one row. In case of multiple rows, just loop on it.

Resources