Q.allSettled not returning all the results - node.js

I'm trying to use Q.allSettled in NodeJS (LoopbackJS) for the following scenario.
As an input to my method (REST API) I get an array of objects. Each of these objects internally has 2 objects: ObjA & ObjB.
For each item in the array:
If ObjA exists in the database get its ID and send mail #1
If it doesn't exist then insert ObjA and then get its ID and send mail #2.
Set ObjA.ID in ObjB and save ObjB.
Once all the objects are saved and emails are sent, then send the response of REST API call. If any of the previous tasks have failed, add the error details in the response.
Here is the pseudo code:
myModel.myMethod = function(input, cb){
var defResp = Q.defer();
var promises = [];
try {
var defObjAList = Q.defer();
promises.push(defObjAList.promise);
getObjAIfItExists(input).done(function(inputWIds) { // input[] with IDs populated
inputWIds.forEach(function(item){
var defObjA = Q.defer();
if(item.objA.id){ // objA already exists in DB
var options = { ... }; // options for sending mail #1
promises.push(Q.ninvoke(Email, "send", options)); // using loopback Email which internally uses nodemailer
defObjA.resolve(item.objA.id);
} else {
Q.ninvoke(ObjA, "save", item.objA).done(function (savedA) {
var options = { ... }; // options for sending mail #2
promises.push(Q.ninvoke(Email, "send", options)); // using loopback Email which internally uses nodemailer
console.log(promises.length); // prints 3
defObjA.resolve(savedA.id);
}, function(err){
defResp.reject(err);
});
}
var defObjB = Q.defer();
promises.push(defObjB.promise);
defObjA.promise.done(function(objAId){
item.objB.objAId = objAId;
promises.push(Q.ninvoke(ObjB, "save", item.objB));
console.log(promises.length); // prints 4
}, function(err){
defResp.reject(err);
});
console.log(promises.length); // prints 2
defObjAList.resolve("Process Complete");
}); // inputWIds.forEach
}, function(err){
defResp.reject(err);
}); //getObjAIfItExists
console.log(promises.length); // prints 1
Q.allSettled(promises).done(function (results) {
console.log(results.length); // prints 1
console.log(JSON.stringify(results)); // prints result[] with single item
var response = {};
response.errors = [];
// iterate on results and check if any promise was failed, if yes add the reason to errors array
defResp.resolve(response);
});
} catch (err) {
defResp.reject(err);
}
return defResp.promise.nodeify(cb);
}
For the testing purpose my input array only contains one item. So the total number of promises that get added to promises[] are 4. But in spite of that the result array contains only 1 item.
My code is working for normal case, but if there is an error e.g. while sending mail, I need to send it in response. That is not working because the results array doesn't contain the email sending promise's output.
Can someone tell me what am I doing wrong? And if I need to handle it in some other way?

I've found out a way to handle the scenario described above. But I'm still open for alternate approaches.
Instead of resolving defObjAList after adding 2nd promise to promises array, I'm waiting till all the promises are added to the array.
Then inside the allSettled handler (which was monitoring only 1 promise), I again call Q.allSettled by passing promises array again (which now contains 4 promises). Now this second handler for allSettled gets the results array with 4 items for all the promises that were added.
Alternatively I could simple wait for defObjAList promise to fulfill and then call Q.allSettled for rest of the promises. This will be better in terms of performance. But for the time being I've kept 2 calls to Q.allSettled.
:
:
inputWIds.forEach(function(item, ind){ // added ind param
// handle ObjA
var defObjB = Q.defer();
promises.push(defObjB.promise);
defObjA.promise.done(function(objAId){
item.objB.objAId = objAId;
promises.push(Q.ninvoke(ObjB, "save", item.objB));
console.log(promises.length); // prints 4
if(ind == inputWIds.length-1){ // check if its the last iteration
defObjAList.resolve("Process Complete");
}
}, function(err){
defResp.reject(err);
});
console.log(promises.length); // prints 2
// defObjAList.resolve("Process Complete"); - removed from here
}); // inputWIds.forEach
:
:
Q.allSettled(promises).done(function (resultsOld) {
Q.allSettled(promises).done(function (results) {
console.log(results.length); // prints 4
// handle results array and send response
});
});

Related

Getting response before firebase transaction done

I'm trying to retrieve all the child then when there's match display.
I print the value in the console and my code work well there after few second, but when I print it in the agent as a message it show not available before the response because it does not wait.
Here is my code:
function retrieveContact(agent) {
var query = admin.database().ref("/contacts").orderByKey();
query.once("value")
.then(function(snapshot) {
snapshot.forEach(function(childSnapshot) {
var key = childSnapshot.key;
var childName = childSnapshot.child('name').val();
if (agent.parameters.name == childName) {
console.log('find ' + childName);
agent.add('The email address for ' + childName + ' is ' + childSnapshot.child('email').val());
}
// console.log('testMode'+childName);
}); //// .then
}); //// .once }
SO, how can I wait my response then let the agent show the result?
How can I include the promise concept in my code ?
You don't show your entire Handler function, but if you're doing async operations (such as reading from the firebase db) you must return the Promise. This is how the Handler Dispatcher knows to wait for the Promise to complete before returning a response to the user.
In your case, it is probably as simple as
return query.once("value")
// etc

Twilio Node JS - filter sms per phone number

I would like to filter sms per phone number and date the SMS was sent using REST API, however the output of the following code is not available outside of client.messages.each() block.
Please advise how I can use the latest sms code sent to the filtered number:
const filterOpts = {
to: '+13075550185',
dateSent: moment().utc().format('YYYY-MM-DD')
};
let pattern = /([0-9]{1,})$/;
let codeCollection = [];
client.messages.each(filterOpts, (record) => {
codeCollection.push(record.body.match(pattern)[0]);
console.log(record.body.match(pattern)[0], record.dateSent);
});
console.log(codeCollection,'I get an empty array here');//how to get
the latest sms and use it
doSomethingWithSMS(codeCollection[0]);
Twilio developer evangelist here.
The each function doesn't actually return a Promise. You can run a callback function after each has completed streaming results by passing it into the options as done like this:
const codeCollection = [];
const pattern = /([0-9]{1,})$/;
const filterOpts = {
to: '+13075550185',
dateSent: moment().utc().format('YYYY-MM-DD'),
done: (err) => {
if (err) { console.error(err); return; }
console.log(codeCollection);
doSomethingWithSMS(codeCollection[0]);
}
};
client.messages.each(filterOpts, (record) => {
codeCollection.push(record.body.match(pattern)[0]);
console.log(record.body.match(pattern)[0], record.dateSent);
});
Let me know if that helps at all.
Do you have access to the length of the array of messages? If so, you can do something like this
const filterOpts = {
to: '+13075550185',
dateSent: moment().utc().format('YYYY-MM-DD')
};
let pattern = /([0-9]{1,})$/;
let codeCollection = [];
var i = 0
client.messages.each(filterOpts, (record) => {
if (i < messages.length){
codeCollection.push(record.body.match(pattern)[0]);
console.log(record.body.match(pattern)[0], record.dateSent);
i++;
else {
nextFunction(codeCollection);
}
});
function nextFunction(codeCollection){
console.log(codeCollection,'I get an empty array here');
doSomethingWithSMS(codeCollection[0]);
}
messages.each() is running asynchronously, so your main thread moves on to the next call while the client.messages() stuff runs on a background thread. So, nothing has been pushed to codeCollection by the time you've tried to access it. You need to somehow wait for the each() to finish before moving on. Twilio client uses backbone style promises, so you can just add another .then() link to the chain, like below. You could also use a library like async which lets you use await to write asynchronous code in a more linear looking fashion.
const filterOpts = {
to: '+13075550185',
dateSent: moment().utc().format('YYYY-MM-DD')
};
let pattern = /([0-9]{1,})$/;
let codeCollection = [];
client.messages.each(filterOpts, (record) => {
codeCollection.push(record.body.match(pattern)[0]);
console.log(record.body.match(pattern)[0], record.dateSent);
}).then(
function() {
console.log(codeCollection,'I get an empty array here');
if( codeCollection.count > 0 ) doSomethingWithSMS(codeCollection[0]);
}
);

res.send() is not sending current response, instead keeps last one

This is some of my code that I have in my index.js. Its waiting for the person to visit url.com/proxy and then it loads up my proxy page, which is really just a form which sends back an email and a code. From my MongoDB database, I grab the users order using the code, which contains some information I need (like product and the message they're trying to get). For some reason, it seems like its responding before it gets this information and then holds onto it for the next time the form is submitted.
The newline in my res.send(product + '\n' + message) isnt working either, but thats not a big deal right now.
But.. for example, the first time I fill out the form ill get a blank response. The second time, I'll get the response to whatever I filled in for the first form, and then the third time ill get the second response. I'm fairly new to Web Development, and feel like I'm doing something obviously wrong but can't seem to figure it out. Any help would be appreciated, thank you.
app.get('/proxy', function(req,res){
res.sendFile(__dirname+ "/views/proxy.html");
});
var message = "";
var product = "";
app.post('/getMessage', function(req,res)
{
returnMsg(req.body.user.code, req.body.user.email);
//res.setHeader('Content-Type', 'text/plain');
res.send(product + "\n" + message);
});
function returnMsg(code, email){
MongoClient.connect(url, function(err, db){
var cursor = db.collection('Orders').find( { "order_id" : Number(code) })
cursor.each(function(err, doc){
assert.equal(err, null);
if (doc!= null)
{
message = doc["message"];
product = doc["product"];
}
else {
console.log("wtf");
// error code here
}
});
console.log(email + " + " + message);
var document = {
"Email" : email,
"Message" : message
}
db.collection("Users").insertOne(document);
db.close();
});
}
You need to do lots of reading about your asynchronous programming works in node.js. There are significant design problems with this code:
You are using module level variables instead of request-level variables.
You are not correctly handling asynchronous responses.
All of this makes a server that simply does not work correctly. You've found one of the problems already. Your async response finishes AFTER you send your response so you end up sending the previously saved response not the current one. In addition, if multiple users are using your server, their responses will tromp on each other.
The core design principle here is first that you need to learn how to program with asynchronous operations. Any function that uses an asynchronous respons and wants to return that value back to the caller needs to accept a callback and deliver the async value via the callback or return a promise and return the value via a resolved promise. The caller then needs to use that callback or promise to fetch the async value when it is available and only send the response then.
In addition, all data associated with a request needs to stay "inside" the request handle or the request object - not in any module level or global variables. That keeps the request from one user from interfering with the requests from another user.
To understand how to return a value from a function with an asynchronous operation in it, see How do I return the response from an asynchronous call?.
What ends up happening in your code is this sequence of events:
Incoming request for /getMessage
You call returnMsg()
returnMsg initiates a connection to the database and then returns
Your request handler calls res.send() with whatever was previously in the message and product variables.
Then, sometime later, the database connect finishes and you call db.collection().find() and then iterate the cursor.
6/ Some time later, the cursor iteration has the first result which you put into your message and product variables (where those values sit until the next request comes in).
In working out how your code should actually work, there are some things about your logic that are unclear. You are assigning message and product inside of cursor.each(). Since cursor.each() is a loop that can run many iterations, which value of message and product do you actually want to use in the res.send()?
Assuming you want the last message and product value from your cursor.each() loop, you could do this:
app.post('/getMessage', function(req, res) {
returnMsg(req.body.user.code, req.body.user.email, function(err, message, product) {
if (err) {
// send some meaningful error response
res.status(500).end();
} else {
res.send(product + "\n" + message);
}
});
});
function returnMsg(code, email, callback) {
let callbackCalled = false;
MongoClient.connect(url, function(err, db) {
if (err) {
return callback(err);
}
var cursor = db.collection('Orders').find({
"order_id": Number(code)
});
var message = "";
var product = "";
cursor.each(function(err, doc) {
if (err) {
if (!callbackCalled) {
callback(err);
callbackCalled = true;
}
} else {
if (doc != null) {
message = doc["message"];
product = doc["product"];
} else {
console.log("wtf");
// error code here
}
}
});
if (message) {
console.log(email + " + " + message);
var document = {
"Email": email,
"Message": message
}
db.collection("Users").insertOne(document);
}
db.close();
if (!callbackCalled) {
callback(null, message, product);
}
});
}
Personally, I would use promises and use the promise interface in your database rather than callbacks.
This code is still just conceptual because it has other issues you need to deal with such as:
Proper error handling is still largely unfinished.
You aren't actually waiting for things like the insert.One() to finish before proceeding.

node's module function return value empty/undefined?

I'm trying to get the html encoded table row value, returned from the slqLite based logger. As I'm new to node modules I'm stuck at:
var sqlite3 = require('sqlite3').verbose();
var db = new sqlite3.Database(':memory:');
var html = '';
module.exports = {
readHtml: function() {
var html = ''; // optional but does not work here as well
db.serialize(function() {
db.each("SELECT rowid AS id, info FROM logger", function(err, row) {
html = html + '<tr><td>' + row.info + '<td><tr>'; << html is growing
console.log('Log: ' + row.info); << working
});
});
console.log(html); // html gets empty here!
return html;
}
}
So have no value returned from:
var sysLog = require('logger');
sysLog.init();
sysLog.write('test string1');
sysLog.write('test string2');
console.log(sysLog.readHtml());
It has to be very simple to be solved ...
node is 6.7
You problem is directly related to a very common issue when starting with JavaScript:
How do I return the response from an asynchronous call?
Which shows the simplest way to receive results of an asynchronous operation, such as db.each is using a callback.
function readHtml()
var html = ''
db.serialize(function() {
db.each(..., function(err, row) {
// this line executes sometime later
// after we already returned from readHtml()
});
});
// this line executes right after the call to db.serialize
// but before db.each calls the callback we give to it.
// At this point, html is '' because we still didn't read any rows
// (they are read asynchronously, sometime later)
return html;
}
readHtml(); // <-- this is always '' because rows are read at a later point
To solve this, you would need a function that will be called with a callback like this:
readHtml(function(html) { // <-- this callback gets called once all rows are read
console.log(html);
});
Your situation also has an additional complication that db.each calls its callback once for every row. By looking at the docs, you can see that db.each accepts an additional complete callback when all rows are read. You can use this callback to signalize reading is done and pass the html results.
Here's how you can define readHtml:
function readHtml(callback) { // pass in a callback to call once all rows are read and all html is accumulated
var html = '';
db.serialize(function() {
// read all the rows and accumulate html as before
db.each("SELECT rowid AS id, info FROM logger", function(err, row) {
html = html + '<tr><td>' + row.info + '<td><tr>';
}, function() {
callback(html); // use the second callback to signal you are done and pass the html back
});
});
}

Getting original request object during multiple asynchronous calls in nodejs-request

I have multiple HTTP requests in a nodejs app that each returns a word of a sentence. The replies will come at different times, so I'm saving them in a dictionary, with the key being the original sentence's word index. Problem is, when I access the request object, I only get the last one.
var completed_requests = 0;
sentence = req.query.sentence;
sentence = "sentence to be translated"
responses=[];
words = sentence.split(" ");
for(j=0;j<words.length;j++){
var word = words[j];
var data={
word:word
};
var options = {
url: 'example.com',
form:data,
index:j
};
request.post(options, function(err,httpResponse,body){
options = options;
if(!err){
responses.push({options.index: body});
completed_requests+=1;
if(completed_requests==words.length){
var a="";
for(var k=0;k<words.length;k++){
a+=responses[k]+" ";
}
res.render('pages/index', { something: a });
}
}
else{
//err
}
});
}
Basically, when I access the object.index object, the object returned isn't the one used for the original request, but the last one (for some reason). How should I resolve this?
When we take a look at how the code is evaluated by JavaScript due to it's async nature in node.js the problem becomes obvious:
For the first word the loop for(j=0;j<words.length;j++){ is executed.
The value of j is assigned to options.index. For the loop run this options.index has now the value 0.
request.post(options, function(err,httpResponse,body){ is executed but the callback handler will be invoked later.
For the first word the loop for(j=0;j<words.length;j++){ is executed.
The value of j is assigned to options.index. options.index has now the value 1.
request.post(options, function(err,httpResponse,body){ is executed but the callback handler will be invoked later.
The problem becomes obvious now since no new options objects are created but the value of j is assigned to options.index in every loop run. When the first callback handler is invoked options.index has the value words.length - 1.
To fix the problem we will wrap creating the options object in a function executeRequest
var completed_requests = 0;
sentence = req.query.sentence;
sentence = "sentence to be translated"
responses=[];
words = sentence.split(" ");
for(j=0;j<words.length;j++){
var word = words[j];
var data={
word:word
};
function executeRequest(url, form, index) {
var options = {
url: url,
form: form,
index: index
};
request.post(options, function(err,httpResponse,body){
// options = options; Superfluous
if(!err){
responses.push({ [index]: body});
completed_requests+=1;
if(completed_requests==words.length){
var a="";
for(var k=0;k<words.length;k++){
a+=responses[k]+" ";
}
res.render('pages/index', { something: a });
}
}
else{
//err
}
});
}
executeRequest('example.com', data, j);
}
A good read about scoping and hoisting in JavaScript can be found here http://www.adequatelygood.com/JavaScript-Scoping-and-Hoisting.html
You need to use an async routine such as forEach or map, also I suggest you read up on the async nature of node to help understand how to handle callbacks for io.

Resources