Nodejs run promises sequentially - node.js

Dears ,
How can i run promises in nodejs sequentially , in the following example am looping through array of hours then for each fetched hour get result from the database , the issue here : am getting results but i want it sequentially same order that i got hours .
angular.forEach(SharedVar.getCategories(), function (h) {
t = h.split('-', 2);
t = t[0];
RESTApi.getAnswerdCallsByHour(t).then(function (answerdTotal) {
$scope.answerdCallsTotalByHour.push(answerdTotal);
var d = SharedVar.getDataS();
d[count] = answerdTotal;
SharedVar.setDataS(d);
count++;
});
});
Thanks ,

var promise = Promise.resolve(); // make an empty promise in the way you do it with your promise library
angular.forEach(SharedVar.getCategories(), function (h) {
promise.then(function() {
return RESTApi.getAnswerdCallsByHour(t).then(function (answerdTotal) {});
});
});

The way to do it sequently would be to do one Request and do the next request inside the promise.
I think the better approach by far is to extend your SharedVar.setDataS(d) function in a way, that it does not depend on getting the data sequentially. Like having a SharedVar.setDataS(d, index) and using the config var in your $http.get (or whatever) functioncall inside your RESTApi to promote that index all the way to the promise.
If your RESTApi looks like this:
var RESTApi = {
getAnswerdCallsByHour : function(hour) {
var url = "bla.com/myservice?hour=" + hour;
return $http.get(url).data;
}
// Some other Methods...
Then you need a way to pass something to "reorder" your Data when it arrives asynchronously, this could be a index you count up or in your case maybe the hour Variable:
var RESTApi = {
getAnswerdCallsByHour : function(hour) {
var url = "bla.com/myservice?hour=" + hour;
var config = [];
config.hour = hour;
return $http.get(url, config); // Return the promise not just data or a specific field
}
// Some other Methods...
Now when your promise is fullfiled you can access your "hour" Variable like so:
var d = SharedVar.getDataS();
d[promise.config.hour] = promise.data;
SharedVar.setDataS(d);
Now you know what piece of data correlates to which request and you do not need to recieve Data in order. The last piece only works properly when hours runs sequential from 0 to 23, if that isn't the case you need to:
var RESTApi = {
getAnswerdCallsByHour : function(hour, index) {
var url = "bla.com/myservice?hour=" + hour;
var config = [];
config.index = index;
return $http.get(url, config);
}
// Some other Methods...
...
...
var d = SharedVar.getDataS();
d[promise.config.index] = promise.data;
SharedVar.setDataS(d);

Safari's answer is how I typically handle this. (Sorry, I don't have enough rep to comment yet...) You were experiencing problems with it because the example provided does not capture and use the new promise in subsequent loops. See my comments on the slightly modified version here:
var promise = Promise.resolve();
angular.forEach(SharedVar.getCategories(), function (h) {
t = h.split('-', 2);
t = t[0];
// You must capture the new promise here; the next loop will wait
// for the promise returned from getAnswerdCallsByHour to resolve.
promise = promise.then(function() {
// Halt downstream promises until this returned promises finishes
return RESTApi.getAnswerdCallsByHour(t).then(function (answerdTotal) {
$scope.answerdCallsTotalByHour.push(answerdTotal);
var d = SharedVar.getDataS();
d[count] = answerdTotal;
SharedVar.setDataS(d);
count++;
});
});
});

Related

module.exports return value undefined

Little info, i have an arp.js file which takes a subnet address "192.168.2" and gets all strings returned from arp -a and stores in an array.
I can't figure out why my arpList function is returning an undefined value in my index.js file.
All the console.logs are returning the correct values in the arp.js page when called from the index.js, but the ipObj is coming up undefined. Even the console.log before i return of ipObj works.
Any help would be greatly appreciated.
var { spawn } = require('child_process');
const arpLs = spawn('arp', ['-a']);
var bufferData;
module.exports = {
arpList: function (subnet) {
arpLs.stdout.on('data', data => {
bufferData += data
})
arpLs.stderr.on('data', data => {
console.log('error: ' + data);
});
arpLs.on('exit', function (code) {
if (code != 0) {
console.log("Error exiting"); //if error occurs
}
console.log("exit start 1"); // checking internal processes at stages
var dataArray = bufferData.split(' ');
var ipArray = [];
for (i = 0; i < dataArray.length; i++) {
if (dataArray[i].includes(subnet)) {
ipArray.push(dataArray[i]);
console.log("loop working");
}
}
var ipObj = { "lanIps": ipArray };
console.log("Object is there: "+ipObj)
return ipObj; // this obj should be returned to the index.js call using
})
},
sayMyName: function () {
return "Hello";
}
}
//arpList(ipSubnet);
//INDEX.js
//the index page looks like this
//var arp = require('./arp.js);
//var ipSubnet = "192.168.2";
//var lanIps = arp.arpList(ipSubnet);
//console.log(lanIps);
I ended up adding a callback function to arpList - function (subnet, callback)
Then instead of returning the value pass it into the callback
Then on the index.js side instead of
var lanIps = arp.arpList(value)
i used
arp.arpList(value, function(res){lanIps = res}
return ipObj; // this obj should be returned to the index.js call using
It won't be returned. The reference say nothing about return value. Node-style callbacks rarely work like that because they are potentially asynchronous and returned value cannot be taken into account.
This a special case of this well-known problem. The process is asynchronous and is finished after arp.arpList(ipSubnet) call, there's nothing to assign to lanIps. This is a use case for promises. There are already third-party promisified counterparts like child-process-promise.
The problem can be also solved by moving to synchronous API. child_process functions have synchronous counterparts, including spawnSync.

Using $.Deferred() with nested $.each statements

I am working with SharePoint making JSOM calls. I need to get results and not move on until ALL data is retrieved. I have tried so many examples (none are complete enough for me to understand how to tailor to my issue with nested $.each loop statements. I seem to get close, but never anything working correctly.
I have edited my actual code (minus the top 3 variables as they are passed in from another page) to incorporate Tomalak's work so it we can figure it out better. Currently, the results are coming up with empty objects. Trying to figure out what I;m doing wrong.
[EDITED 06 AUG 2018]
Finally got it to work. there was only two tiny problems I found with the provided code :-). I will try and bold them.
var fya = [2008,2009]; //Fiscal Year Array which we use to know what lists to look at
var assignedRecords = []; //Global Reusable Variable
var assignedCourses = ['Math','Science','Reading']; //There might not be records who are associated with a particular course in each list. Wee need to pull the student records (assignedRecords) assoiciated with the above in 2008 and 2009.
SP.ClientContext.prototype.executeQueryPromise = function (items) {
var result = $.Deferred();
this.load(items);
this.executeQueryAsync(function (sender, args) { result.resolve(items) },
function (sender, args) { result.reject(args) });
return result.promise();
};
Move 'var arr = [];' outside of arrayFromCollection
Add 'var' to the e variable in that same function
var arr = [];
function arrayFromCollection(coll) {
var e = coll.getEnumerator();
while (e.moveNext()) arr.push(e.get_current());
return arr;
};
function queryListPromise(title, course) {
var list = hostWeb.get_lists().getByTitle(title);
var camlQuery = new SP.CamlQuery();
camlQuery.set_viewXml('<View><Query><Where>'
+ '<Eq><FieldRef Name="Course"/><Value Type="Text">' + course + '</Value></Eq>'
+ '</Where></Query></View>');
return context.executeQueryPromise(list.getItems(camlQuery)).then(arrayFromCollection);
};
function GetAssignedApplications() {
assignedRecords = []; //Need to start empty
var myCourses = assignedCourses;
$.each(myCourses, function (i, myCoursesItem) {
var courseName = myCoursesItem.get_lookupValue();
Change '$.forEach' to '$.each'
$.each(fya, function (n, fyaItem) {
var listTitle = "FY" + String(fyaItem).substring(2); //FY18 & FY19 are the names of the actual lists.
assignedRecords.push(queryListPromise(listTitle, courseName));
});
});
$.when(assignedRecords).then(function (results) {
return Array.prototype.concat.apply([], results);
}).then(function (items) {
items.forEach(function (item) {
var a = item; //item is empty and this actually runs before arrayFromCollection and it returns duplicate records (4) when there is only 2.
});
}).fail(onError);
};
So first things first, waiting on multiple promises in jQuery is done with $.when().
$.when(p1, p2, p3).then(function (results) {
// results is an array of length 3
});
In order to make this work with a variable argument count, we use Function#apply:
var promises = [p1, p2, p3]; // could be be of any length now
$.when.apply($, promises).then(function () {
var results = [].slice.call(arguments);
// results is an array of the same length as promises
});
In order to make this reusable, let's call that $.whenAll:
$.whenAll = function (promises) {
return $.when.apply($, promises).then(function () {
return [].slice.call(arguments);
});
};
Now we can do:
$.whenAll([p1, p2, p3]).then(function (results) {
// results is an array of length 3
});
Next, we need to turn the callback-based executeQueryAsync() into a promise-based function. In general, promisification follows this scheme:
function somethingPromise(args) {
var result = $.Deferred();
somethingAsync(/* onSuccess */ result.resolve, /* onError */ result.reject);
return result.promise();
}
This expects that the onSuccess and onError handlers receive data as arguments. That's not the case with executeQueryAsync - it modifies the items directly. However, we still want to resolve the promise with items, so we need to do that explicitly.
Let's slap that onto the SP.ClientContext prototype right-away for re-use.
SP.ClientContext.prototype.executeQueryPromise = function (items) {
var result = $.Deferred();
this.load(items);
this.executeQueryAsync(
function (sender, args) { result.resolve(items); },
function (sender, args) { result.reject(args); }
);
return result.promise();
};
Next we should set up a helper that turns those unwieldy collections into something usable.
function arrayFromCollection(coll) {
var arr = [], e = coll.getEnumerator();
while (e.moveNext()) arr.push(e.get_current());
return arr;
}
With that all done, we can extract a function that takes a title and returns a promise for an item array.
function queryListPromise(title) {
var list = web.get_lists().getByTitle(title);
var q = new SP.CamlQuery();
//...
return ctx.executeQueryPromise(list.getItems(q)).then(arrayFromCollection);
}
And finally, since you want a flat list of items, we should start with a flat list of queries:
function GetNextFunction() {
var queries = [];
// build flat list of queries
$.each(arr1, function(i, arr1Item) {
$.each(arr2, function(n, arr2Item) {
queries.push(queryListPromise(arr2Item));
});
});
// wait on all queries
$.whenAll(queries).then(function (results) {
// flatten incoming array of arrays
return Array.prototype.concat.apply([], results);
}).then(function (items) {
// items contains all items from all queries
items.forEach(function (item) { /* ... */ });
}).fail(onError);
}
Here queries becomes a flat array of promises, each promise will resolve to an array of items. Therefore, results will be an array of arrays of items.

Bluebird async series call

I have this below code, but as soon as widgetsAddCall is added into array, it gets executed, and promise.each is of no use.
Function widgetsAddCall is making async request to API server and returns bluebird promisified request. I want that API call is made one after another, so that each call sends chunk of data one after another.
var chunkCnt = Math.ceil(widgetsIds.length/4000);
var responseT = Array();
var Promise = require('bluebird');
for(var cntTemp =0 ; cntTemp<chunkCnt;cntTemp++){
var tempWidgs = widgetsIds.slice(cntTemp,cntTemp+4000);
var query = {
id: tempWidgs.join(',')
};
responseT.push(widgetsAddCall(tempWidgs,query,campRemoteId,campaign));
}
return Promise.each(responseT,function(responses) {
// Use the responses here
return getWidgets(campRemoteId,campaign).then((ids) => {
var toRemove = [];
for(var id of ids){
if(widgetsIds.indexOf(id)===-1){
toRemove.push(id);
}
}
if(toRemove.length) {
return removeWidgets(campaign, campRemoteId, toRemove);
}
});
})
I want that API call is made one after another, so that each call
sends chunk of data one after another.
The simplest way I can think of to serialize all your calls is to prebuild your chunks into an array and then use Bluebird's Promise.mapSeries() to serially iterate through the array:
var Promise = require('bluebird');
// pre-build chunks into an array
var chunks = [];
var chunkCnt = Math.ceil(widgetsIds.length / 4000);
for (var chunkIndex = 0; chunkIndex < chunkCnt; chunkIndex++) {
chunks.push(widgetsIds.slice(cntIndex,cntIndex + 4000));
}
// now serially iterate the array
Promise.mapSeries(chunks, function(item) {
return widgetsAddCall(item, {id: item.join(',')}, campRemoteId, campaign);
}).then(function(results) {
// now process the results of widgetsAddCall() here
});
FYI, your original use of Promise.each() was not making any sense to me because you were iterating an array of promises, but then not using any info from the iteration. There seemed to be no point to the iteration. Plus, you weren't properly serializing your calls to widgetsAddCall() anyway since you were launching all those calls in parallel.

Perform arbitrary set of asynchronous tasks

My input is streamed from another source, which makes it difficult to use async.forEach. I am pulling data from an API endpoint, but I have a limit of 1000 objects per request to the endpoint, and I need to get hundreds of thousands of them (basically all of them) and I will know they're finished when the response contains < 1000 objects. Now, I have tried this approach:
/* List all deposits */
var depositsAll = [];
var depositsIteration = [];
async.doWhilst(this._post(endpoint_path, function (err, response) {
// check err
/* Loop through the data and gather only the deposits */
for (var key in response) {
//do some stuff
}
depositsAll += depositsIteration;
return callback(null, depositsAll);
}, {limit: 1000, offset: 0, sort: 'desc'}),
response.length > 1000, function (err, depositsAll) {
// check for err
// return the complete result
return callback(null, depositsAll);
});
With this code I get an async internal error that iterator is not a function. But in general I am almost sure the logic is not correct as well.
If it's not clear what I'm trying to achieve - I need to perform a request multiple times, and add the response data to a result that at the end contains all the results, so I can return it. And I need to perform requests until the response contains less than 1000 objects.
I also looked into async.queue but could not get the hang of it...
Any ideas?
You should be able to do it like that, but if that example is from your real code you have misunderstood some of how async works. doWhilst takes three arguments, each of them being a function:
The function to be called by async. Gets argument callback that must be called. In your case, you need to wrap this._post inside another function.
The test function (you would give value of response.length > 1000, ie. a boolean, if response would be defined)
The final function to be called once execution is stopped
Example with each needed function separated for readability:
var depositsAll = [];
var responseLength = 1000;
var self = this;
var post = function(asyncCb) {
self._post(endpoint_path, function(err, res) {
...
responseLength = res.length;
asyncCb(err, depositsAll);
});
}
var check = function() {
return responseLength >= 1000;
};
var done = function(err, deposits) {
console.log(deposits);
};
async.doWhilst(post, check, done);

Nodejs: Resolving promises with generator function

I know there are a lot of good examples over the web and I read a lot of them, but currently I'm stucked with resolving promises with the new functionality of generators in nodejs 0.11.x.
For e.g. I have the following function:
SolrBaseDomain.prototype.promisedQuery = function(query, callback) {
var solrClient = solr.createClient(this.configuration);
var defer = Q.defer();
solrClient.search(query, function(err,obj){
if (!err) {
if (obj.response.numFound > 0) {
defer.resolve(obj.response.docs);
} else {
defer.resolve(null);
}
} else {
defer.reject(err);
}
});
var promise = defer.promise;
return Q.async(function* (){
var result = yield promise;
return result;
});
};
I expected that every call to this method will wait until the promise is fullfilled and the return-statement gives back the result of the promise.
But currently it seems that instead the code inside "Q.async..." will not be executed or the async call arrives after the return statement of the method was executed.
It's strange, in every example I know, this is one of the recommended ways in order to wait for async calls in nodejs but currently it does not work for me.
I've tried a lot of different variations of the above example, but the result is everytime the same, I get not back a valid result.
I have nodejs installed in version 0.11.10 and the --harmony-flag is set, when the code is executede.
Can anyone point me to right direction? I'm wondering if I oversee something ... :)
Thanks for your feedback.
Best regards
Udo
I expected that every call to this method will wait until the promise is fullfilled and the return-statement gives back the result of the promise.
No. Generators will not make functions synchronous, you cannot (and don't want to) block while waiting for a result. When calling a generator function and running sequentially through the async steps that it yields, the result you will get back in the end is still asynchronous - and therefore a promise. Only inside of the generator, your code can use synchronous control flow and yield.
This means that the (then-) callback-based code
SolrBaseDomain.prototype.promisedQuery = function(query) {
var promise = Q.ninvoke(solr.createClient(this.configuration), "search", query);
return promise.then(function(obj) {
if (obj.response.numFound > 0) {
return obj.response.docs;
} else {
return null;
}
});
};
becomes
SolrBaseDomain.prototype.promisedQuery = Q.async(function* (query) {
var promise = Q.ninvoke(solr.createClient(this.configuration), "search", query);
var obj = yield promise;
// ^^^^^
if (obj.response.numFound > 0) {
return obj.response.docs;
} else {
return null;
}
});
Try this
SolrBaseDomain.prototype.promisedQuery = Q.async(function*(query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
This does the same thing for promises as this does for callbacks:
SolrBaseDomain.prototype.query = function (query, callback) {
var solrClient = solr.createClient(this.configuration);
solrClient.search(query, function(err, obj) {
if (err) return callback(err);
callback(null, obj.response.numFound > 0 ? obj.response.docs : null);
});
};
Therefore if the first return a promise that resolves to undefined so will the callback version call the callback with undefined.
according to your suggestions, my code looks now like this:
...
SolrBaseDomain.prototype.query = Q.async(function* (query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
...
I share the above query-function over all data access layers in order to have a central method which is querying the different indexes in an asynchronous way.
For e.g. in the domain data access layer, the code which deals with that function looks like this:
SolrHostDomain.prototype.getByName = Q.async(function* (domain) {
var queryObject = {
"domain": domain
};
var query = this.getQuery("byName", queryObject);
var docs = yield this.query(query);
var domain = null;
if (docs != null && docs.length > 0) {
domain = this.dataMapper.merge(docs[0]);
}
return domain;});
Currently I'm not sure if the generator in the "getByName"-function is necessary, but it seems to work. Dealing with promises is some unclear concept for me, since I'm new to nodejs.
So maybe, if you can help me on that topic and point me in the right direction, this would be helpfull.
The main question for me is, how can I ensure, that a synchronous method can call an asynchronous method and get back not a promise but the final result of this promise.
I've searched a long time, but I could not find a good documentation which describes the use of generator functions or promises in conjunction with synchronous calls. Even examples are focusing only of using the mechanism but not working together with synchronous function.
Best regards and many thanks for your help
Udo
Got it!!!
After a few trial and errors, I think I got it now and I have a working solutions:
Query function:
SolrBaseDomain.prototype.query = Q.async(function* (query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
Calling method:
SolrHostDomain.prototype.getByName = function(domain) {
var queryObject = {
"domain": domain
};
var query = this.getQuery("byName", queryObject);
var docsPromise = this.query(query);
var _self = this;
return docsPromise.then(function(docs) {
var domain = null;
if (docs != null && docs.length > 0) {
domain = _self.dataMapper.merge(docs[0]);
}
return domain;
});
};
The solution was to understand, that the "query"-method still returns a promise instead of the concrete result even if yield is used.
So I have to add every code which is working on the result of the promise within the "then"-functions (or "done" if no other caller up in the calling hierarchy of methods will follow).
After the settlement of the promise, each code which is set within the "then"-functions will be processed.
BR
Udo

Resources