Error Promise.all - node.js

I'm trying to use Promise.all function but actually as I start nodeJS and I discover asynchronous technology I don't know where is my problem in my code...
Basically I would like to use Promise.all to make my own callback in a function then In my loop For I create several Promise and if I can save my data then I'll resolve my current Promise.
But apparently my promise.all is executed immediately and it don't wait for my others Promise..
See below ..
function persistMAP(jsonData,callback){
//Deck persistance
const promises =[];
for(var i=0; i<1; i++){
(function(i) {
var rowData = new DeckDatabase({
_id: new mongoose.Types.ObjectId(),
DeckNumber: Number(jsonData.Deck[i].DeckNumber),
x: Number(jsonData.Deck[i].x),
y: Number(jsonData.Deck[i].y),
});
rowData.save(function (err) {
if (err) return console.log(err);
for(var index=0; j=jsonData.Units.length,index<j; index++){
(function(index) {
promises.push(
new Promise(function(resolve,reject){
var unit = new MapDatabase({
UnitID:jsonData.Units[index].UnitID,
TypeID: Number(jsonData.Units[index].TypeID),
x: Number(jsonData.Units[index].x),
y: Number(jsonData.Units[index].y),
_id: mongoose.Types.ObjectId(((jsonData.Units[index].Code).toLowerCase()) + 'dd40c86762e0fb12000003'), // mongoose.Types.ObjectId(jsonData.Units[i].Code + 'dd40c86762e0fb12000003')
MainClass: jsonData.Units[index].MainClass,
Orientation: jsonData.Units[index].Orientation,
Postion: jsonData.Units[index].Postion,
Deck : String(rowData._id)
});
unit.save(function (err) {
if (err) {
reject();
throw err
}
else{
console.log('save');
resolve();
}
});
})
);
})(index);
}
});
})(i);
}
Promise.all(promises)
.then(()=>{
console.log('start find');
callback();
})
};
and here is where I call my function
else{
var jobj = JSON.parse(response.body);
console.log("persist begin");
persistMAP(jobj,function(){
console.log('retrieve Done');
MapDatabase.find()
.populate('Deck')
.exec(function(err, finalData){
console.log('send');
res.send(finalData);
})
});
}
So why It doesn't wait ? :(

You do an async operation in your for:
for(var i=0; i<1; i++){
....
rowData.save(function (err) {
So your promises array isn't filled with all the promise.
To loop async use async module :
var async = require("async");
var models = []; // my models array
// async loop, with 10 in same time, e is the element, i the index in models
async.eachOfLimit(models, 10, function(e, i, cb){
anAsyncFunction(function(err){
return cb(err);
});
}, function(err, result{
// all done
});

Related

Not able to return value from promise in Nodejs

I have written the following code in Nodejs which is saving data in MongoDB:
function insertDoc(db,data){
return new Promise(resolve => {
callback=db.collection('AnalysisCollection').insertOne(data).then(function(response,obj){
console.log("Inserted record");
resolve(obj);
//console.log(obj);
// response.on('end',function(){
// resolve(obj);
// });
//return resolve(obj);
}).then(() => { return obj }
).catch(function(error){
throw new Error(error);
});
})
}
I am calling the above function from the main function like this:
async function cosmosDBConnect(nluResultJSON){
try{
//console.log("Inserting to cosmos DB");
console.log(nluResultJSON);
var url = config.cosmos_endpoint;
var result="";
var data = JSON.parse(JSON.stringify(nluResultJSON));
MongoClient.connect(url, function(err, client) {
assert.equal(null, err);
var db = client.db('NLUAnalysisDB');
// insertDoc(db, data, function() {
result=insertDoc(db, data, function() {
console.log(result);
client.close();
//return data._id;
});
});
}
catch (e) {
console.log(e);
}
}
module.exports = { cosmosDBConnect };
But in cosmosDBConnect, I am getting 'undefined' for the result, though in insertDoc I am getting the output for'obj' with _id for the inserted record.
Please help me to return this _id to cosmosDBConnect.
You are use callbacks inside of async function, which creates internal scopes. So your return aplies to them instead of whole function. You should use Promise-based methods inside of async function using await (without callbacks) or wrap whole function into own Promise otherwise.
Example:
function cosmosDBConnect(nluResultJSON) {
return new Promise((resolve, reject) => {
var url = config.cosmos_endpoint;
var result = '';
var data = JSON.parse(JSON.stringify(nluResultJSON));
MongoClient.connect(url, function(err, client) {
if (err) return reject(err);
assert.equal(null, err);
var db = client.db('NLUAnalysisDB');
insertDoc(db, data).then(obj => {
console.log(obj);
client.close();
return resolve(data._id);
});
});
});
}
Also you need to understand that your insertDoc return Promise and do not accept callback you tried to pass.
Ref: async function
result = insertDoc(db, data).then((data) => {
console.log(data);
}).catch(err => console.error(err));

How to promise in a nested Asynchronous operations

I've been struggling to figure out how to do this for the past two hours. But I can't seem to get the Promise to wait for my searchFacesFunc to complete before it solves. What is the correct way to do so?
async function searchFacesFunc(faceId){
var searchFacesParams = {
CollectionId: "my-collection",
FaceId: faceId,
FaceMatchThreshold: 80,
MaxFaces: 10
};
await rekognition.searchFaces(searchFacesParams, function(err, data) {
if(err){
throw err;
}else{
var matching_percent = data.FaceMatches[0].Similarity;
console.log('Matching Percent: ' + matching_percent);
}
});
}
return new Promise((resolve, reject) => {
rekognition.indexFaces(indexParams, function(err, data) {
if(err){
throw err;
}else{
const faceRecords = data.FaceRecords;
for(let i = 0; i < faceRecords.length; i++){
var faceId = faceRecords[i].Face.FaceId;
console.log('FaceId: ' + faceId);
searchFacesFunc(faceId); //The promise is finished before these multiple functions finish
}
resolve(null, 'success');
}
});
});
If the rekognition.indexFaces function accepts an asynchronous callback, you can solve this issue easily:
return rekognition.indexFaces(indexParams, async (err, data) => {
if (err) {
throw err;
} else {
const faceRecords = data.FaceRecords;
for (let i = 0; i < faceRecords.length; i++) {
var faceId = faceRecords[i].Face.FaceId;
console.log("FaceId: " + faceId);
await searchFacesFunc(faceId); // Await the promise
}
return "success";
}
});
};
However, if this is not the case, you can still solve this the following way:
Use util.promisify to "promisify" the rekognition.indexFaces function
Construct a recursive callback function that only resolves the original promise you constructed when it executed faceRecords.length times.

node csv files upload synchronous import followed by mongoose find

I am not able to get results from a mongoose find query after successfully uploading csv files. The data from the csvs is being saved in my collection OK. But I am getting no results from a following find query. What causes promises to not conform to the sequence? I feel I may be close to understanding promises as they are handled in the event loop but this case escapes me?
let uploadfiles = function (req,res) {
return new Promise( function (resolve, reject) {
upload(req, res, (err) => {
promises = [];
if (!err) {
if (Object.keys(req.files).length>0) {
simpleLoop(req.files, function (value, key, lovey) {
promises.push(insertStreamData(req.files[key]["originalname"]));
});
console.log('hither');
}
}
return Promise.all(promises);
});
resolve('something');
});
};
// insert uploaded csv data from files into db
function insertStreamData(filename){
var originalFileName = filename;
var thefilename = originalFileName.split('.');
var csvfile = "./public/files/"+originalFileName;
var stream = fs.createReadStream(csvfile, { headers: true});
var csvStream = csv().on("data", function(data){
if(data[0] != 'Product #' && data[7] != 0){
var item = new Product({
'Invoice':thefilename[0],
'Product #': data[0],
'SKU': data[1],
'UpcCode': data[2],
'Description': data[3],
'MfgNo': data[4],
'Vendor': data[5],
'Order Qty': data[6],
'Ship Qty': data[7],
'Min Sell': data[8],
'Retail': data[9],
'Cost': data[10],
'Ext Cost': data[11],
'Box': data[12]
});
item.save(function(error){ if(error){ throw error; } });
}
}).on("end", function(){
console.log('the end');
});
stream.pipe(csvStream);
}
let getincomingproducts = function(){
return new Promise(function (resolve, reject) {
resolve('done');
});
};
router.post('/uploaddata', function(req, res) {
uploadfiles(req,res).then(function(result){
return getincomingproducts();
}).then(function(result){
console.log(result);
res.redirect('/showdata');
}).catch(err => console.log("Caught " + err));
});
Output 'done' is logged to the console prior to the first promise. I want to replace done with mongoose find results. But a simple console log demonstrates the unsynchronous result.
done
hither
the end
the end
the end
the end
Please don't mix Promise constructors and such with async and await. This could be a lot easier if you just use await and async functions. And only wrap the callback taking functions once.
A clean version wouldn't have any business logic in the Promise constructors and would used await rather than then in an async function.
I found the second point in the program where it doesn't wait for the completion of an action that is that the insertStreamData does not return a promise that is fulfilled when it is done which is what is causing the out of order issue. I will update my answer.
I see a issue in upload files where it has:
return Promise.all(promises);
});
resolve('something');
which should be
return Promise.all(promises).then(function () {
resolve('something');
});
This change should cause the output to be hither, the end * 4, done. If you want the insertStreamData to occur in serial rather than parallel, that requires a different change.
My version of the script:
function upload_wrapper(req, res) {
return new Promise(function(resolve, reject) {
upload(req, res, function (err) {
if (err) {
reject(err);
} else {
resolve();
}
});
})
}
let uploadfiles = async function(req, res) {
await upload_wrapper(req, res);
let promises = [];
if (Object.keys(req.files).length > 0) {
simpleLoop(req.files, function(value, key, lovey) {
promises.push(insertStreamData(req.files[key]["originalname"]));
});
console.log('hither');
}
await Promise.all(promises);
return 'something';
};
// insert uploaded csv data from files into db
function insertStreamData(filename) {
return new Promise(function(resolve, reject) {
var originalFileName = filename;
var thefilename = originalFileName.split('.');
var csvfile = "./public/files/" + originalFileName;
var stream = fs.createReadStream(csvfile, {
headers: true
});
var csvStream = csv();
csvStream.on("data", function(data) {
if (data[0] != 'Product #' && data[7] != 0) {
var item = new Product({
'Invoice': thefilename[0],
'Product #': data[0],
'SKU': data[1],
'UpcCode': data[2],
'Description': data[3],
'MfgNo': data[4],
'Vendor': data[5],
'Order Qty': data[6],
'Ship Qty': data[7],
'Min Sell': data[8],
'Retail': data[9],
'Cost': data[10],
'Ext Cost': data[11],
'Box': data[12]
});
item.save(function(error) {
if (error) {
//throw error;
csvStream.pause(); // here the stream should be destroyed if this stops at the first error
reject(error);
}
});
}
}).on("end", function() {
console.log('the end');
resolve('the end');
});
stream.pipe(csvStream);
});
}
let getincomingproducts = function() {
return Promise.resolve('done');
};
router.post('/uploaddata', async function(req, res) {
try {
let result = await uploadfiles(req, res);
let result = await getincomingproducts();
console.log(result);
res.redirect('/showdata');
} catch (err) {
console.log("Caught " + err)
}
});
Untested. It still could be improved.

Chaining multiple chained promises with Q (loopback app)

Here is a scenario, I've implemented a loopback remote method which imports some data from REST connector to local postgresql connector.
I can do this for a single model
var importData = function (model, cb) {
migrateModel(model, cb)
.then(findImportInfo)
.then(fetchRemoteData)
.then(processFetchedData)
.then(updateImportInfo)
.then(countLocalData)
.then(importCompleted)
.catch(function (err) {
importFailed(err, cb);
})
.done(function () {
console.log('done');
});
};
So the chain does many thing and at the end importCompleted calls the provide cb which is the callback that returns the response to the REST API.
But I can't figure how to do this with multiple models and return each result. I tried something like this, it works actually but REST API never receives a result.
var importDataAll = function (app, cb) {
var models = app.models();
var deferred = Q.defer();
var promises = [];
var results = [];
function doCallback() {
cb(null, results);
}
models.forEach(function (model) {
if (typeof model.importData === 'function') {
migrateModel(model, model.definition.name, null)
.then(findImportInfo)
.then(fetchRemoteData)
.then(processFetchedData)
.then(updateImportInfo)
.then(countLocalData)
.then(function (prevResult) {
var deferred = Q.defer();
var remoteCount = prevResult.dataCount;
var localCount = prevResult.recordCount;
var result =
{
'time': new Date(),
'remoteCount': remoteCount,
'localCount': localCount,
'started': prevResult.started,
'completed': new Date()
}
results.push(result);
deferred.resolve(result);
return deferred.promise;
})
.catch(function (err) {
promises.reject(err);
})
}
});
return Q.allSettled(promises).then(doCallback);
};
I'm lost at that point, any ideas?
EDIT
Trying #Otze's answer I tried this also
var importDataAll = function (app, cb) {
var models = app.models().filter(function (element, index, array) {
return typeof element.importData === 'function';
});
var promises = models.map(function (model) {
migrateModel(model, model.definition.name, null)
.then(findImportInfo)
.then(fetchRemoteData)
.then(processFetchedData)
.then(updateImportInfo)
.then(countLocalData)
.then(importResult)
.catch(function (err) {
promises.reject(err);
})
});
Q.all(promises)
.then(function (resolvedPromises) {
cb(null, results);
});
};
But the result is the same, cb gets called early but the code actually runs in order. I just can't get the result to the response. I think it's never ends so the REST API gets no content after some time.
Have a look at Q.all or any of the other promise combination functions:
http://documentup.com/kriskowal/q/#combination
With Q.all you could do somehting like this:
var promises = myModels.map(doAllThePromiseThings);
Q.all(promises)
.then(function(resolvedPromises) {
doStuff();
});
Note that you need to return a promise from doAllThePromiseThings.
Since .then returns a promise you can simply do:
.then(function (prevResult) {
return {
'time': new Date(),
'remoteCount': prevResult.dataCount,
'localCount': prevResult.recordCount,
'started': prevResult.started,
'completed': new Date()
};
})
instead of
.then(function (prevResult) {
var deferred = Q.defer();
var remoteCount = prevResult.dataCount;
var localCount = prevResult.recordCount;
var result =
{
'time': new Date(),
'remoteCount': remoteCount,
'localCount': localCount,
'started': prevResult.started,
'completed': new Date()
}
results.push(result);
deferred.resolve(result);
return deferred.promise;
})
I use bluebird library's map method to accomplish such use cases:
https://github.com/petkaantonov/bluebird/blob/master/API.md#mapfunction-mapper--object-options---promise
var Promise = require('bluebird');
var importDataAll = function (app, cb) {
var models = app.models().filter(function (element, index, array) {
return typeof element.importData === 'function';
});
Promise.map(
models,
function(model) {
return migrateModel(model, model.definition.name, null) // don't want to break the promise chain
.then(findImportInfo)
.then(fetchRemoteData)
.then(processFetchedData)
.then(updateImportInfo)
.then(countLocalData)
.then(importResult)
.then(function(){
...
return Promise.resolve(); // don't want to break the promise chain
});
},
{concurrency: 1}
)
.then(function () {
debug('finished working on all the models one-by-one');
cb(null);
})
.catch(function (err) {
cb(err);
});

Why is this promise resolving early?

The promise from Product.find() is resolving before the loop updating the highest price has completed. How can I resolve the promise only when the data manipulation is complete?
Product.find({'prodType': req.params.type}).lean().exec(function (err, product) {
if (err) {
res.status(400);
}
for (var i = 0; i < product.length; i++) {
(function(u) {
Option.find({'prodId': product[i].productId}).lean().sort({price: -1}).limit(1).exec(function (err, highest) {
product[u].price = highest;
// not updated in returned object
});
})(i);
}
}).then(function (product) {
res.send(product);
});
Move all of the code from the .exec() callback to a .then, and return a new promise to it that resolves when all the other promises are complete.
Product.find({'prodType': req.params.type}).lean().exec()
.then(function (product) {
var promises = [];
for (var i = 0; i < product.length; i++) {
(function(u) {
promises.push(Option.find({'prodId': product[i].productId}).lean().sort({price: -1}).limit(1).exec(function (err, highest) {
product[u].price = highest;
// not updated in returned object
}));
})(i);
}
return Promise.all(promises).then(function () {
// we want to pass the original product array to the next .then
return product;
});
}).then(function (product) {
res.send(product);
}).catch(function (err) { // the power of promise error catching!
// If any error occurs in any of the db requests or in the code, this will be called.
res.status(400);
});
Also, since you're dealing with an array, .map makes this easier and removes the need for the inner IIFE.
Product.find({'prodType': req.params.type}).lean().exec()
.then(function (product) {
var promises = product.map(function (p) {
return Option.find({'prodId': p.productId}).lean().sort({price: -1}).limit(1).exec(function (err, highest) {
p.price = highest;
}));
});
return Promise.all(promises).then(function () {
// we want to pass the original product array to the next .then
return product;
});
}).then(function (product) {
res.send(product);
}).catch(function (err) { // the power of promise error catching!
// If any error occurs in any of the db requests or in the code, this will be called.
res.status(400);
});

Resources