In my Nodejs application I need to select a set of data from the database by some queries. An amount of queries not defined at the start and I need to check after each DB query whether I need to make another query or not. So the process like this
var total_result = [];
var n = 0;
db.query('.....query...', function(result, error){
// Callback with db data response
total_results.push(...result...);
n += result.length;
if (n < req_n) {
// ... Here is the code to repeat the same query ...
// ?????
}
});
Thank you.
var total_result = [],
n = 0
while(await query())
async function query() {
return new Promise((resolve, reject) => {
db.query('...query...', (result, error) => {
if(error) reject(error)
// Callback with db data response
total_results.push(...result...)
n += result.length
if (n < req_n) resolve(true)
else resolve(false)
})
})
}
Related
var user_id = '98-XXXXXXXX'
Contact.find({user_id: user_id})
.exec(function (err, results) {
if (err) {
return next(err);
}
var finalArray = [];
for(var i = 0; i< results[0].Total; i++) {
if(results[0].Contacts[i].name != "SPAM") {
for(var j = 0; j < results[0].Contacts[i].phoneNumbers.length; j++){
var number = results[0].Contacts[i].phoneNumbers[j].number
number = number.replace(/ /g,'');
var user_id = number.substr(number.length - 10);
Login.find({user_id:user_id})
.exec(function(err,results) {
if(err) {
return next(err); }
var intCount = results.length;
if (intCount > 0)
{
console.log('called')
finalArray.push(results[0])
console.log(finalArray)
}
});
}
}
//console.log(i,results[0].Total - 1);
//if(i == results[0].Total - 1)
}
console.log('Ended Here',finalArray)
var responseTosend = {"message":finalArray,"user_id":user_id}
return res.send(responseTosend);
});
EndedHere[] this is coming up first empty, after that i got the result of login.find query which is correct. Any ideas how to get the finalArray after all the calculation.
Thanks in advance
Since the functions are returning promises within the loops, the code execution has to wait till all those promises are resolved. Consider using Promise.all or Promise.map to wait. Reference
As already mentioned, a structure like this, will not return the results, but the intermediate functions or objects before they are finished, since nodejs does not know it should await the results first.
const x = [1,2,3]
let results = []
for (let i = 0; i < x.length; i++){
results.push(someAsyncJobLikeADatabaseCall(x[i]))
}
// this will not return the results, but the intermediate async objects/functions
console.log(results)
Here is a better version using promises and the .map function. Notice, how we replaced the for loop with .map() (which you could see as a shorthand for .forEach + push() or for() + push(). Mongoose returns Promises if configured right, so you don't even have to manually define them and we can directly return them in .map.
const x = [1,2,3]
let results = []
async function getAsyncResults(array){
// map returns an array, this time, an array of promises
const promises = x.map(number => someAsyncJobLikeADatabaseCall(number))
// Promise.all resolves, if all promises in the array have been resolved
return Promise.all(promises)
}
try {
let results = await getAsyncResults(x)
// this will return the results you expect.
console.log(results)
} catch (err) {
console.log('Some error', err)
}
I'm working on an application which have to read excel files and send each line to a server;
Now I would like to do it asynchronously, I mean I would like to get the result of server.postRequest(1); before sending another request server.postRequest(2);
for (let row = 2; row <= nrows; row++) {
Excel.mapExcelLine(workbook.Sheets[firstSheetName], row, ncols)
.then((data) => {
let idExcel = data.id;
database.get("SELECT * FROM sync WHERE idExcel=?", [idExcel], async function (err, row) {
if (row == undefined) {
let promise = await server.postRequest(row.id);
}
}
});
}
Here is the postRequest function
postRequest(id){
return new Promise(function (resolve, reject) {
options.body = id;
request.post(options, function (error, response, body) {
if (response.headers['msg'] == 'authrequired') {
request.post(authOptions, function (error, response, body) {
if(response.headers['msg'] == 'ok')
postRequest(id);
else resolve(402);
});
}
else if(response.headers['msg'] == 'ok'){
logger.info('Demand' + id + 'created'
resolve(200);
}
});
});
}
Now when I'm trying to loop through this array for example:
[1,2,3,4]
messages from postResquest function don't appear in the right order.
How to wait for each request to finish before sending another one ?
the expected output is:
Demand 1 created
Demand 2 created
Demand 3 created
Demand 4 created
You must use await for the requests you are sending asynchronously
I'm trying to write a Node.js code that does the below.
Connect to a Salesforce instance.
Get the past 7 days, and loop through them.
Run 2 queries inside them and push the result to an Array.
Display the value in another function.
Here is my JS code.
var jsforce = require("jsforce");
var moment = require('moment');
function connectToEP() {
var main_Obj = {};
var response_Obj = {};
var pastSevenDaysArray = [];
var conn = new jsforce.Connection();
var beforeSevenDays = moment().subtract(7, 'days').format('YYYY-MM-DD');
var today = moment().startOf('day');
var i = 0;
conn.login("myUid", "myPwd").then(() => {
console.log("Connected To Dashboard");
for (var m = moment(beforeSevenDays); m.diff(today, 'days') <= 0; m.add(1, 'days')) {
conn.query("SELECT SUM(Total_ETA_of_all_tasks__c), SUM(Total_ETA__C) from Daily_Update__c where DAY_ONLY(createddate)= " + m.format('YYYY-MM-DD')).then(() => {
console.log("B1");
var z = response_Obj.aggrRes;
response_Obj.aggrRes = res;
pastSevenDaysArray.push({ z: res });
console.log("B1 Exit");
}).then(() => {
conn.query("SELECT count(Id), Task_Type__c FROM Daily_Task__c where DAY_ONLY(createddate) = " + m.format('YYYY-MM-DD') + " group by Task_Type__c").then(() => {
console.log("B2");
var z = response_Obj.aggrRes;
response_Obj.aggrRes = res;
pastSevenDaysArray.push({ z: res });
console.log("B2 Exit");
})
})
}
return Promise.resolve(pastSevenDaysArray);
}).then((data) => {
console.log(typeof data);
updateMessage(JSON.stringify(data));
console.log(typeof data);
});
}
function updateMessage(message) {
console.log("XXXXXXXXXXXX");
console.log(message);
console.log("XXXXXXXXXXXX");
}
function socketNotificationReceived() {
console.log("socket salesforce rec");
connectToEP();
}
socketNotificationReceived();
when I run this code, the output that I get is.
socket salesforce rec
Connected To Dashboard
object
XXXXXXXXXXXX
[]
XXXXXXXXXXXX
object
B1
B1
B1
B1
B1
B1
B1
B1
I'm very new to this js platform, unable to get the promises concepts :(. please let me know on were am I going wrong and how can I fix it.
An explanation of what's going is very helpful in my future projects.
Thanks
The thing I always do when I get confused is to decompose. Build the pieces one by one, and make sure each works. Trying to understand your code, I get something like this...
A function each for logging in, getting a "task sum" from the db and getting a "task count" from the db. (Task sum/count is what I guessed the queries were up to. Rename as you see fit).
var jsforce = require("jsforce");
var moment = require('moment');
function login(conn) {
return conn.login("myUid", "myPwd");
}
function queryTaskSumForDay(conn, m) {
return conn.query("SELECT SUM(Total_ETA_of_all_tasks__c), SUM(Total_ETA__C) from Daily_Update__c where DAY_ONLY(createddate)= " + m.format('YYYY-MM-DD'));
}
function queryTaskCountForDay(conn, m) {
return conn.query("SELECT count(Id), Task_Type__c FROM Daily_Task__c where DAY_ONLY(createddate) = " + m.format('YYYY-MM-DD') + " group by Task_Type__c");
}
With those working, it should be easy to get a sum and a count for a given day. Rather than returning these in an array (containing two objects that each have a "z" property as your code did), I opted for the simpler single object that has a sum and count property. You may need to change this to suit your design. Notice the use of Promise.all() to resolve two promises together...
function sumAndCountForDay(conn, m) {
let sum = queryTaskSumForDay(conn, m);
let count = queryTaskCountForDay(conn, m);
return Promise.all([sum, count]).then(results => {
return { sum: results[0], count: results[1] };
});
}
With that working, it should be easy to get an array of sum-count objects for a period of seven days using your moment logic and the Promise.all() idea...
function sumAndCountForPriorWeek(conn) {
let promises = [];
let beforeSevenDays = moment().subtract(7, 'days').format('YYYY-MM-DD');
let today = moment().startOf('day');
for (let m = moment(beforeSevenDays); m.diff(today, 'days') <= 0; m.add(1, 'days')) {
promises.push(sumAndCountForDay(conn, m));
}
return Promise.all(promises);
}
With that working (notice the pattern here?), your OP function is tiny and nearly fully tested because we tested all of it's parts...
function connectToEP() {
let conn = new jsforce.Connection();
return login(conn).then(() => {
return sumAndCountForPriorWeek(conn)
}).then(result => {
console.log(JSON.stringify(result));
return result;
}).catch(error => {
console.log('error: ' + JSON.stringify(error));
return error;
});
}
I think your general structure should be something like this. The biggest issue is not returning promises when you need to. A "for loop" of promises is a little difficult to step into, but if you can do them in parallel then the easiest thing to do is Promise.all If you need to aggregate the data before you can perform the next query then you need to do multiple Promise.all().then()'s. The reason you get an empty array [] is because your for loop creates the promises but doesn't wait until they finish.
var jsforce = require("jsforce");
var moment = require('moment');
function connectToEP() {
// connectToEP now returns a promise
return conn.login("myUid", "myPwd").then(() => {
console.log("Connected To Dashboard");
let myQueries = [];
for (start ; condition ; incrementer) {
myQueries.push( // Add all these query promises to the parallel queue
conn.query(someQuery)
.then((res) => {
return res;
})
.then((res) => {
return conn.query(someQuery).then((res) => {
return someData;
})
})
)
}
return Promise.all(myQueries); // Waits for all queries to finish...
}).then((allData) => { // allData is an array of all the promise results
return updateMessage(JSON.stringify(allData));
});
}
I am trying to call CompaniesHouse API and fetch companies registered between November and February. The approach I took is to pick a starting index(a company registered in November) and a stop index(a company registered in February) and loop through to get the companies registered between the start and stop index. Like so:
var needle = require("needle");
var startIdx = 11059000;
var stopIdx = 11211109;
for(idx = startIdx; idx < stopIdx; idx++)
{
needle('get', "https://api.companieshouse.gov.uk/company/"+idx, {
username: key,password:""
})
.then(function(data) {
})
.catch(function(err) {
console.log('Call the locksmith!' + err)
})
}
But this doesn't work as gives either a timeout or socket hangup error.
The API is currently in beta and some features are still yet to be implemented.
Because the for loop runs synchronously and your calls to needle() are asynchronous and therefore do not block, you end up attempting to start more than 100,000 network requests at once. This overwhelms either your local computer or the target server and you start getting socket errors.
For this many requests, you need to run them X at a time so no more than X are in flight at the same time. To maximize performance, you will have to figure out what value of X you want to use because it will depend upon the target server and how it handles lots of simultaneous requests. It is generally safe to start with a value of 5 and then increase it from there to test higher values.
If you were processing an array, there are a number of pre-built options to run X requests at once. The simplest is to use a pre-built concurrency management operation such as Bluebird. Or you can write your own. You can see examples of both here: Make several requests to an API that can only handle 20 request a minute
But, since you are not processing an array, but are just incrementing a number for each successive request, I couldn't find a pre-built option that does that. So, I wrote a general purpose one where you can fill in the function that will increment your index:
// fn gets called on each iteration - must return a promise
// limit is max number of requests to be in flight at once
// cnt is number of times to call fn
// options is optional and can be {continueOnError: true}
// runN returns a promise that resolves with results array.
// If continueOnError is set, then results array
// contains error values too (presumed to be instanceof Error so caller can discern
// them from regular values)
function runN(fn, limit, cnt, options = {}) {
return new Promise((resolve, reject) => {
let inFlightCntr = 0;
let results = [];
let cntr = 0;
let doneCnt = 0;
function run() {
while (inFlightCntr < limit && cntr < cnt) {
let resultIndex = cntr++;
++inFlightCntr;
fn().then(result => {
--inFlightCntr;
++doneCnt;
results[resultIndex] = result;
run(); // run any more that still need to be run
}).catch(err => {
--inFlightCntr;
++doneCnt;
if (options.continueOnError) {
// assumes error is instanceof Error so caller can tell the
// difference between a genuine result and an error
results[resultIndex] = err;
run(); // run any more that still need to be run
} else {
reject(err);
}
});
}
if (doneCnt === cnt) {
resolve(results);
}
}
run();
});
}
Then, you could use this like this:
const needle = require("needle");
const startIdx = 11059000;
const stopIdx = 11211109;
const numConcurrent = 5;
let idxCntr = startIdx;
runN(function() {
let idx = idxCntr++;
return needle('get', "https://api.companieshouse.gov.uk/company/"+idx, {
username: key,password:""
});
}, numConcurrent, stopIdx - startIdx + 1, {continueOnError: true}).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
To minimize memory use, you can use a .then() handler on your call to needle() and trim down the response to only what you need in the final array:
const needle = require("needle");
const startIdx = 11059000;
const stopIdx = 11211109;
const numConcurrent = 5;
let idxCntr = startIdx;
runN(function() {
let idx = idxCntr++;
return needle('get', "https://api.companieshouse.gov.uk/company/"+idx, {
username: key,password:""
}).then(response => {
// construct the smallest possible response here and then return it
// to minimize memory use for your 100,000+ requests
return response.someProperty;
});
}, numConcurrent, stopIdx - startIdx + 1, {continueOnError: true}).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
var needle = require("needle");
var startIdx = 11059000;
var stopIdx = 11211109;
const promises = [];
for(idx = startIdx; idx < stopIdx; idx++)
{
promises.push(
needle('get', "https://api.companieshouse.gov.uk/company/"+idx, {
username: key,password:""
})
)
}
Promise.all(promises).then(results => {console.log(results);}).catch(err => console.log(err));
A simple Promise.all implementation can help.
//Importing: postges DB connection
var pg = require('pg');
var conString = "postgres://readxxx:p#ssword#vmwoxxx-tst:8888/worxxx";
var prvsiteid = '';
var cursiteid = '';
var qurystring = '';
pg.connect(conString, function(err, client, done) {
if (err) {
return console.error('error fetching client from pool', err);
return;
}
client.query("select site_id,created_at,started_at,completed_at,notes,finish_code from _background_tasks where finish_code > 0 and site_id > 0 and abs(extract(Epoch from (now()::timestamp without time zone - completed_at)))/60 <= 4600 order by site_id asc", function(err, result1) {
done();
if (err) {
return console.error('error running query', err);
return;
}
for (var i = 0; i < result1.rowCount; i++) {
cursiteid = result1.rows[i].site_id;
if (prvsiteid != cursiteid) {
prvsiteid = cursiteid;
qurystring = "select trim(su.name) as name, case When trim(su.email) is null then su.name || '#netapp.com' when trim(su.email) > '' then su.name || '#netapp.com' else su.name end uemail,su.friendly_name as frdname from system_users su where su.state = 'active' and su.id in (select distinct(system_user_id) from users u where u.site_id = " + cursiteid + "and u.admin_level >= 5)"
client.query(qurystring, function(err, result2) {
done();
if (err) {
return console.error('error running query', err);
return;
}
for (var j = 0; j < result2.rowCount; j++) {
console.log(cursiteid, result2.rows[j].name, result2.rows[j].uemail, result2.rows[j].frdname);
}
});
}
console.log(result1.rows[i].site_id, result1.rows[i].created_at, result1.rows[i].started_at, result1.rows[i].completed_at);
}
});
});
I know NodeJS programs are asynchronous but this scenario I intend it to be synchronous.
for loop(outer) --> for loop(inner) when outer forloop changes with new site id i want to send email to all the emailids from inner loop and also the resultant rows of the each site of outer loop has to printed.
If the library you are using doesn't support synchronous operations, you can't use it synchronously without abominations (You do not want abominations in your code. No matter how much the boss is pressing).
You can fairly simply perform loops and other actions on asynchronous operations by using Promises, specifically, bluebird which supports the ability to take callback-based asynchronous functions and transform them into Promise-ready functions.