async each is running to fast? - node.js

I have the following code running through an object with async:
async.each(Object.keys(shopList), function(key, callback){
var shop = shopList[key];
saveOrder(payId, shopList[key], key, req.body, req.user, function(err, newOrder){
if (err) {
callback(err);
}else{
orderCount++;
console.log("succes!", orderCount, newOrder.number);
callback();
}
})
}, function(err){
if (err) {
console.log("ERROR!", err);
}else{
console.log("done!");
}
})
In this function a another function is called. This code looks like this:
saveOrder = function(payId, shop, nameSlug, body, user, callback){
console.log("saveOrder");
var orderNumber = 0;
Order.findOne().sort({_id:-1}).exec(function(err, latestOrder) {
if(latestOrder.number){
orderNumber = latestOrder.number.split("-")[1];
}
var order = new Order();
var date = new Date();
order.number = date.getFullYear().toString() + date.getMonth().toString() + "-" + (parseInt(orderNumber)+1);
order.date = date;
order.payId = payId;
order.status = {
status: "Created",
comment: "",
date: new Date()
};
order.comment = body.comment;
order.shop = {
name: shop.name,
nameSlug: nameSlug
}
order.billingDetails = {
//order details
}
order.sendDetails = {
//more order details
}
order.user = {
//yep, order details
}
var orderItems = [];
for(p = 0; p < shop.items.length; p++){
var product = shop.items[p];
var orderItem = {
_id: product._id,
name: product.name,
brand: product.brand[0].name,
price: product.price,
quantity: product.quantity
}
orderItems.push(orderItem);
}
order.items = orderItems;
order.save(function(err, result){
if (err){
console.log("err!", err);
return callback(err)
}else{
console.log("saved!");
return callback(null, result);
}
})
})
}
The problem is in the last function. There I try to create a ordernumber which must be unique. I get the last order, split the ordernumber and do a +1.
When I have more objects in my shopList, this function is triggered when he is not ready. With other words, the first order isn't saved then, and I will get the same ordernumber.
How can I fix this? I tried a setTimeout in the async.each but that isn't working.

You could use a mutex using locks.
The callbacks will wait that the mutex is unlocked to lock it making that you won't have simultaneous executions.
var locks = require('locks');
var mutex = locks.createMutex();
saveOrder = function(payId, shop, nameSlug, body, user, callback){
mutex.lock(function () {
console.log("saveOrder");
var orderNumber = 0;
Order.findOne().sort({_id:-1}).exec(function(err, latestOrder) {
if(latestOrder.number){
orderNumber = latestOrder.number.split("-")[1];
}
var order = new Order();
var date = new Date();
order.number = date.getFullYear().toString() + date.getMonth().toString() + "-" + (parseInt(orderNumber)+1);
order.date = date;
order.payId = payId;
order.status = {
status: "Created",
comment: "",
date: new Date()
};
order.comment = body.comment;
order.shop = {
name: shop.name,
nameSlug: nameSlug
}
order.billingDetails = {
//order details
}
order.sendDetails = {
//more order details
}
order.user = {
//yep, order details
}
var orderItems = [];
for(p = 0; p < shop.items.length; p++){
var product = shop.items[p];
var orderItem = {
_id: product._id,
name: product.name,
brand: product.brand[0].name,
price: product.price,
quantity: product.quantity
}
orderItems.push(orderItem);
}
order.items = orderItems;
order.save(function(err, result){
if (err){
console.log("err!", err);
return callback(err)
}else{
console.log("saved!");
return callback(null, result);
}
})
})
mutex.unlock(); //don't forget to unlock the mutex
});
}

You should use async.waterfall instead of async.each, because:
async.waterfall - runs the tasks array of functions in series, each passing their results to the next in the array. http://caolan.github.io/async/docs.html#waterfall
async.each - applies the function iteratee to each item in coll, in parallel.

Fixed this issue with using eachSeries() instead of each()
http://caolan.github.io/async/docs.html#eachSeries

Related

exports return null before executing mongodb save when i call it from another controller?

I have to call one controller function from another, but other function is not returning a proper value.
I called a function from one controller in line 'a',line 'b' is another controller.but inside forEach loop when it comes to line 'c', its not executing inside code instead it return null before executing the code from line 'd'.
a. var invoiceItemResp = InvoiceItemController.createInvoiceItems(itemList, doc.id);
b. exports.createInvoiceItems = (itemList, invoiceId) => {
var counter = 0;
var success = false;
itemList.forEach(element => {
var invoiceItem = new InvoiceItem({
_id: new mongoose.Types.ObjectId(),
invoice_id: invoiceId,
item: element.item,
amount: element.amount
});
c. invoiceItem.save((err, doc) => {
d. if(!err){
++counter;
if(counter == itemList.length){
response = {
"message": "success"
}
return response;
}
}else{
response = {
"message": "failed"
}
return response;
}
});
});
}
Your needs the loop to be async and wait for DB to write,
const invoiceItemResp = await InvoiceItemController.createInvoiceItems(itemList, doc.id);
exports.createInvoiceItems = async (itemList, invoiceId) =>
{
var counter = 0;
var success = false;
let resposneArray = [];
let response= {};
try
{
for (const element of itemList)
{
var invoiceItem = new InvoiceItem({
_id: new mongoose.Types.ObjectId(),
invoice_id: invoiceId,
item: element.item,
amount: element.amount
});
const err = await invoiceItem.save();
if (!err)
{
++counter;
if (counter == itemList.length)
{
response = {
"message": "success"
};
resposneArray.push(response);
}
} else
{
response = {
"message": "failed"
};
resposneArray.push(response)
}
});
return resposneArray;
}
catch (e)
{
throw e;
}
};
You should use async await while writing to db.
var invoiceItemResp = await InvoiceItemController.createInvoiceItems(itemList, doc.id);
exports.createInvoiceItems = async (itemList, invoiceId) => {
var counter = 0;
var success = false;
itemList.forEach(element => {
var invoiceItem = new InvoiceItem({
_id: new mongoose.Types.ObjectId(),
invoice_id: invoiceId,
item: element.item,
amount: element.amount
});
invoiceItem.save((err, doc) => {
if(!err){
++counter;
if(counter == itemList.length){
response = {
"message": "success"
}
return response;
}
}else{
response = {
"message": "failed"
}
return response;
}
});
});
}

Expressjs does not execute sequencially and function return does not work

I am new to node/express js, and trying to execute the following. The control executes the lines after function call "var nextVersion =getNextContractVersion(cid)", even before the function returns a response. As a result the value for newVersion is not updated to Contract object. Also, function getNextContractVersion(cid) returns undefined, unlike the updated nextVersion.
How do i fix this behavior, please suggest. Also, is the right way of invoking function?
// Package imports
const express = require('express');
var router = express.Router();
const mongoose = require('mongoose');
//Local imports
var { Customer } = require('../models/customer');
var { Contract } = require('../models/contract');
router.put('/:cid/contracts', (req, res) => {
var cid = req.params.cid;
var nextVersion =getNextContractVersion(cid);
var contract;
if (validateCustomerId(cid)) {
req.body.contract.forEach((item) => {
contract = new Contract({
customerID: cid,
startDate: item.startDate,
endDate: item.endDate,
conditions: item.conditions,
price: item.price,
author: item.author,
version: nextVersion
});
});
contract.save((err, docs) => {
if (!err) {
Customer.findOneAndUpdate({ customerID: cid }, { $push: { contract: contract } },
{ safe: true, upsert: true, new: true }).populate({ path: 'contract' }).exec((err1, docs1) => {
if (!err1) {
res.send(docs1).status(200);
} else {
console.log('Error is adding a new contract:' + JSON.stringify(err1, undefined, 2));
}
});
} else {
console.log('Error is updating a new customer:' + JSON.stringify(err, undefined, 2));
}
});
} else {
res.status(400).send('Bad Request - Invalid input!')
}
});
function getNextContractVersion(cid) {
var nextVersion=1;
Contract.findOne({ customerID: cid }).sort({version: 'descending'}).exec((err, doc) => {
if (!err && doc != null) {
var currentVersion = parseInt(doc.version);
nextVersion = currentVersion + 1;
}
});
return nextVersion;
}
You are mixing synchronous and asynchronous code.
Contract.findOne({ customerID: cid }).sort({version: 'descending'}).exec((err, doc) => {
if (!err && doc != null) {
var currentVersion = parseInt(doc.version);
nextVersion = currentVersion + 1;
}
});
The above code effectively says "Go to the database, find one of these objects and whenever in the future that is done, run this code that's in the exec block."
One of the ways to reason about asynchronous code from a synchronous mindset is that of promises.
Here's a semi pseudo implementation:
router.put('/:cid/contracts', (req, res) => {
var cid = req.params.cid;
return getTheMostRecentContract(cid)
.then(function(oldContract){
var nextVersion = oldContract.version +1;
if(!validateCustomerId(cid)){
return res.status(400).send('Bad Request - Invalid input!');
}
var contract;
var savePromises = [];
req.body.contract.forEach((item) => {
contract = new Contract({
customerID: cid,
startDate: item.startDate,
endDate: item.endDate,
conditions: item.conditions,
price: item.price,
author: item.author,
version: nextVersion
});
savePromises.push(contract.save());
});
return Promise.all(savePromises);
})
.then(function(resultOfAllSavePromises){
//rest of code here
}).catch(function(error){
console.log('Error is updating a new customer:' + JSON.stringify(err, undefined, 2));
return res.status(400);
})
});
function getTheMostRecentContract(cid) {
return Contract.findOne({ customerID: cid }).sort({version: 'descending'});
}
As a matter of practice though, have the database control your auto-increment values. This code won't work in a high traffic environment.

wait for function and then console result

I want to wait for my functions like fetching all result first, then consoling my result.
exports.listofAllFeaturedProd = (req, res) => {
Product.find({is_featured:'1'},function(error,fetchallFeatProds)
{
var allProducts = new Array();
var pp = 0;
var products = new Array();
for (var ProductId in fetchallFeatProds)
{
var pArr = [];
pArr['_id'] = fetchallFeatProds[ProductId]._id;
pArr['name'] = fetchallFeatProds[ProductId].name;
pArr['sku'] = fetchallFeatProds[ProductId].sku;
pArr['description'] = fetchallFeatProds[ProductId].description;
pArr['price'] = fetchallFeatProds[ProductId].price;
pArr['large_image'] = fetchingImage(fetchallFeatProds[ProductId]._id);
pArr['brand'] = fetchingBrand(fetchallFeatProds[ProductId].brand_id);
console.log('#################### IMAGE ####################');
console.log(pArr);
pp++;
}
console.log(products);
});
};
function fetchingImage(pid)
{
ProductImage.findOne({product_id:pid},function(error,fetchallFeatProdsImgs)
{
console.log(fetchallFeatProdsImgs.large_image);
return fetchallFeatProdsImgs.large_image;
});
}
function fetchingBrand(bid)
{
Brand.findOne({_id:bid},function(error,fetchAllBrands)
{
console.log(fetchAllBrands);
return fetchAllBrands;
});
}
node not wait for functions and console undefined after that console my function result. how i stop my code for fetching first result then console all data in array.
Output for console.log(pArr);
[ _id: 57bd996ebf8c930b2bcc06a1,
name: 'New Product',
sku: 'New-Product',
description: 'New Product',
price: 'test',
large_image: undefined,
brand: undefined ]
After that added console inside functions which gave results as below:
Output for fetchingImage
images/12.png
Output for fetchingBrand
{ user_id: '57b42b571fc35e49162de413',
brand_name: '10 Fork ',
brand_logo: 'uploads/brands_logo/1472027911329_5.png',
brand_desc: '10 Fork',
_id: 57bd5ce6cebed2a3189cedcf,
__v: 0 }
Desired output is:
[ _id: 57bd996ebf8c930b2bcc06a1,
name: 'New Product',
sku: 'New-Product',
description: 'New Product',
price: 'test',
large_image: images/12.png,
brand: { user_id: '57b42b571fc35e49162de413',
brand_name: '10 Fork ',
brand_logo: 'uploads/brands_logo/1472027911329_5.png',
brand_desc: '10 Fork',
_id: 57bd5ce6cebed2a3189cedcf,
__v: 0 } ]
Try below code:
var temp = [],
async = require('async');
async.eachSeries(fetchallFeatProds, function(ProductId, callback)
{
pArr['_id'] = ProductId._id;
pArr['name'] = ProductId.name;
pArr['sku'] = ProductId.sku;
pArr['description'] = ProductId.description;
pArr['price'] = ProductId.price;
pArr['large_image'] = fetchingImage(ProductId._id);
pArr['brand'] = fetchingBrand(ProductId.brand_id);
temp.push(pArr);
callback(null);
}, function(err){
console.log(temp); //This should give you desired result
});
If it is not working still, try using callbackfor the functions fetchingImage and fetchingBrand. Or you may try using async.parallel as well inside eachSeries.
EDIT:-
async-eachseries
Change your functions with callback.
function fetchingImage(pid, callback)
{
ProductImage.findOne({product_id:pid},function(error,fetchallFeatProdsImgs)
{
console.log(fetchallFeatProdsImgs.large_image);
callback(error,fetchallFeatProdsImgs.large_image);
});
}
function fetchingBrand(bid, callback)
{
Brand.findOne({_id:bid},function(error,fetchAllBrands)
{
console.log(fetchAllBrands);
calback(error,fetchAllBrands);
});
}
Use async.parallel so that it will wait till both the functions are done. Then push into temp array. Doc to refer
async.eachSeries(fetchallFeatProds, function(ProductId, callback)
{
var pArr = {};
pArr['_id'] = ProductId._id;
pArr['name'] = ProductId.name;
pArr['sku'] = ProductId.sku;
pArr['description'] = ProductId.description;
pArr['price'] = ProductId.price;
async.parallel([
function(callback)
{
fetchingImage(ProductId._id, function(err, res){
pArr['large_image'] = res;
callback(err); //Forgot to add
});
},
function(callback)
{
fetchingBrand(ProductId.brand_id,function(err, res){
pArr['brand'] = res;
callback(err); //Forgot to add
});
},
], function(err){
console.log(pArr); //Edit
temp.push(pArr);
callback(err);
})
}, function(err){
console.log(temp); //This should give you desired result
callback(err);
});

node js mongo db dependencies (doc not being found)

I have the following code:
var method = PushLoop.prototype;
var agent = require('./_header')
var request = require('request');
var User = require('../models/user_model.js');
var Message = require('../models/message_model.js');
var async = require('async')
function PushLoop() {};
method.startPushLoop = function() {
getUserList()
function getUserList() {
User.find({}, function(err, users) {
if (err) throw err;
if (users.length > 0) {
getUserMessages(users)
} else {
setTimeout(getUserList, 3000)
}
});
}
function getUserMessages(users) {
// console.log("getUserMessages")
async.eachSeries(users, function (user, callback) {
var params = {
email: user.email,
pwd: user.password,
token: user.device_token
}
messageRequest(params)
callback();
}, function (err) {
if (err) {
console.log(err)
setTimeout(getUserList, 3000)
}
});
}
function messageRequest(params) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ params.email +"&api_password="+ params.pwd +"&method=getSMS&type=1&limit=5"
request(url, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
if (responseObject["status"] == "success") {
async.eachSeries(messages, function(message, callback){
console.log(params.token)
saveMessage(message, params.token)
callback();
}, function(err) {
if (err) {
console.log(err)
}
// setTimeout(getUserList, 3000)
})
} else {
// setTimeout(getUserList, 3000)
}
} else {
console.log(err)
// setTimeout(getUserList, 3000)
}
});
setTimeout(getUserList, 3000)
}
function saveMessage(message, token) {
// { $and: [ { price: { $ne: 1.99 } }, { price: { $exists: true } }
// Message.find({ $and: [{ message_id: message.id}, {device_token: token}]}, function (err, doc){
Message.findOne({message_id: message.id}, function (err, doc){
if (!doc) {
console.log('emtpy today')
var m = new Message({
message_id: message.id,
did: message.did,
contact: message.contact,
message: message.message,
date: message.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: token
});
m.save(function(e) {
if (e) {
console.log(e)
} else {
agent.createMessage()
.device(token)
.alert(message.message)
.set('contact', message.contact)
.set('did', message.did)
.set('id', message.id)
.set('date', message.date)
.set('message', message.message)
.send();
}
});
}
}) //.limit(1);
}
};
module.exports = PushLoop;
Which actually works perfectly fine in my development environment - However in production (i'm using Openshift) the mongo documents get saved in an endless loop so it looks like the (if (!doc)) condition always return true therefore the document gets created each time. Not sure if this could be a mongoose issue - I also tried the "find" method instead of "findOne". My dev env has node 0.12.7 and Openshift has 0.10.x - this could be the issue, and i'm still investigating - but if anybody can spot an error I cannot see in my logic/code please let me know
thanks!
I solved this issue by using a "series" like pattern and using the shift method on the users array. The mongoose upsert findOneOrCreate is good however if there is a found document, the document is returned, if one isn't found and therefore created, it's also returned. Therefore I could not distinguish between the newly insert doc vs. a found doc, so used the same findOne function which returns null if no doc is found I just create it and send the push notification. Still abit ugly, and I know I could have used promises or the async lib, might refactor in the future. This works for now
function PushLoop() {};
var results = [];
method.go = function() {
var userArr = [];
startLoop()
function startLoop() {
User.find({},function(err, users) {
if (err) throw err;
users.forEach(function(u) {
userArr.push(u)
})
function async(arg, callback) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ arg.email +"&api_password="+ arg.password +"&method=getSMS&type=1&limit=5"
request.get(url, {timeout: 30000}, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
var status = responseObject.status
if (status === "success") {
messages.forEach(function(m) {
var message = new Message({
message_id: m.id,
did: m.did,
contact: m.contact,
message: m.message,
date: m.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: arg.device_token
});
var query = { $and : [{message_id: m.id}, {device_token: arg.device_token}] }
var query1 = { message_id: m.id }
Message.findOne(query).lean().exec(function (err, doc){
if (!doc || doc == null) {
message.save(function(e) {
console.log("message saved")
if (e) {
console.log("there is an error")
console.log(e)
} else {
console.log(message.device_token)
var messageStringCleaned = message.message.toString().replace(/\\/g,"");
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : messageStringCleaned
}
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = messageStringCleaned;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
}
})
}
});
});
}
else {
console.log(err)
}
}
});
setTimeout(function() {
callback(arg + "testing 12");
}, 1000);
}
// Final task (same in all the examples)
function series(item) {
if(item) {
async( item, function(result) {
results.push(result);
return series(userArr.shift());
});
} else {
return final();
}
}
function final() {
console.log('Done');
startLoop();
}
series(userArr.shift())
});
}
}
module.exports = PushLoop;

Sorting MapReduce results on MongoDB

I have my MapReduce working correctly to group my results by date. All works well, however I'd like to have the results to be returned from Most Recent to Oldest based on 'created' value.
findTimelineByQuery: function (query, fields, options, callback) {
var obj = {};
obj.map = function() {
emit(Date.UTC(this.created.getFullYear(), this.created.getMonth(), this.created.getDate()), {
created:this.created,
title:this.title,
type: this.type,
id: this._id,
owner: this.owner,
value: this.value
});
};
obj.reduce = function(previous, current) {
var array = [];
var res = {items:array};
current.forEach(function (v) {
res.items.push(v);
});
return res;
};
obj.verbose = true;
obj.query = query;
_Items.mapReduce(obj, function(error, model, stats) {
callback(model);
});
}

Resources