I am using mongodb find query inside loop as I need to run the find query 5 times. And I used below code for that:
let result = {};
let miles = ['5','10','15','25'];
let i = 0;
while (i < miles.length) {
Shops.find({ 'shopInfo.address':{ $geoWithin:{ $centerSphere: [ [ 75.83183541365247, 30.902146005639267 ], miles[i] / 3959 ] } } }).then(response=>{
if(i==4){
result[miles[i]] = response.length;
res.json(result);
}else{
result[miles[i]] = response.length;
i++;
}
})
.catch(err=>{
console.log(err)
});
}
And when I hit the api on browser. It's not returning with anything and getting below error in console:
Please help me, How can I solve the issue?
The following happens:
Your while loop runs and starts an async action. However as that async action is not finished yet, the i++ will not be executed, therefore the loop runs forever, creates more and more async actions that fill up your memory, and finally NodeJS crashes because it runs out of memory. To prevent that, you should not synchronously iterate with an asynchronous task. Either await the asynchronous task inside a loop, or .map the miles to the asynchronous tasks then await them all:
const entries = Promise.all(miles.map(mile =>
Shops.find({ 'shopInfo.address':{ $geoWithin:{ $centerSphere: [ [ 75.83183541365247, 30.902146005639267 ], mile / 3959 ] } } })
.then(entry => ([mile, entry.length]))
));
entries.then(entries => {
const result = Object.fromEntries(entries);
//...
});
// Polyfill: Object.fromEntries
if(!Object.fromEntries)
Object.defineProperty(Object, "fromEntries", {
value(entries) {
const result = {};
for(const [k, v] of entries)
result[k] = v;
return result;
}
});
Normal loops have issues with looping over async code. You can implement an asyncForEach helper function:
async function asyncForEach(array, callback) {
if (array)
for (let index = 0; index < array.length; index++) {
await callback(array[index], index, array);
}
}
And call it executing your mongoQueries:
await asyncForEach(miles, async mile => {
const response = await Shops.find({ 'shopInfo.address':{ $geoWithin:{ $centerSphere: [ [ 75.83183541365247, 30.902146005639267 ], miles[i] / 3959 ] } } })
// do soming with the response
})
Use async/await to wait mongo promise get resolved and then increment variable. Try this :
async function test() {
let result = {};
let miles = ["5", "10", "15", "25"];
let i = 0;
while (i < miles.length) {
try {
let response = await Shops.find({
"shopInfo.address": {
$geoWithin: {
$centerSphere: [
[75.83183541365247, 30.902146005639267],
miles[i] / 3959
]
}
}
});
if (i == 4) {
result[miles[i]] = response.length;
res.json(result);
} else {
result[miles[i]] = response.length;
i++;
}
} catch (err) {
console.log(err);
}
}
}
Below is the code which worked for me:
let miles = ['5','10','15','25','50'];
async.map(miles,getDistance , function(err, results) {
if(err){
console.log(err);
}
res.json(results);
});
function getDistance(mile, callback) {
Shops.count({ 'shopInfo.address':{ $geoWithin:{ $centerSphere: [ [ 75.83183541365247, 30.902146005639267 ], mile / 3959 ] } } }).then(response=>{
if(response){
callback(null,response);
}
})
}
I Hope It will work for you as well.
Related
I am performing the useMutation operation in the innermost loop and want to check the remaining cost upon every mutation. But it gets checked after all the mutations which is a problem because for some reason even if all the mutations get done(When the cost is under limits), It calls the .then() part for cost-checking and waiting for unknown reason.
Edit: I also noticed that even though the program is waiting again and again, the network status of chrome shows that all the mutations have happened and only the query of handleDiscountMore i.e. fetchMore is pending
const { loading, error, data, fetchMore, extensions, refetch } = useQuery(GET_COLLECTION, {
variables: { "id": coll.collection.id }
});
const [updatePrice] = useMutation(UPDATE_PRICE);
const redirectToModify = async (data, totalProducts) => {
wait(20000);
var cursor, fetchCount;
fetchCount = data.collection.products.edges.length;
totalProducts -= fetchCount;
data.collection.products.edges.map(async(product) => {
const results = await Promise.all(product.node.variants.edges.map(variant => {
if (selected == 'curr_price') {
//do stuff
}
else {
//do stuff
}
const productVariableInput = {
//Object
};
updatePrice({ variables: { input: productVariableInput } }).then(({ data, extensions }) => {
console.log("Remaining", extensions.cost.throttleStatus.currentlyAvailable)
console.log(data)
if (extensions.cost.throttleStatus.currentlyAvailable < 100) {
console.log("WAITING")
wait(18000);
}
}).catch(e => {
console.log(e)
})
console.log("AFTER")
return 0;
}))
})
if (totalProducts > 0) {
console.log("Calling")
wait(15000);
handleDiscountMore(data, cursor, totalProducts)
}
};
//Below function is Just for reference. It gets called before checking the throttleStatus above. afaik there's no problem with this
const handleDiscountMore = (data, cursor, pc) => {
console.log("Call received")
fetchMore({
variables: {
"id": data.collection.id,
"cursor": cursor
},
updateQuery: (
previousResult,
{ fetchMoreResult }
) => {
console.log("adding", fetchMoreResult);
redirectToModify(fetchMoreResult, pc);
// return fetchMoreResult;
}
})
}
Your map of maps is evaluating all promises at exactly the same time. Here's a cleaned up example that uses a nested for loop instead, which will wait for each request to finish before starting the next (note: I couldn't run it to test, so there's probably some bugs, but the idea is there):
const id = coll.collection.id;
const { loading, error, data, fetchMore, extensions, refetch } = useQuery(GET_COLLECTION, {
variables: { id }
});
const [updatePrice] = useMutation(UPDATE_PRICE);
// Given a product, returns a promise that resolves when all variants are processed
async function process_product(product){
const variants = product.node.variants.edges;
for (let i = 0; i < variants.length; i++){
await process_variant(variants[i]);
}
}
// Given a variant, returns a promise after the product is processed
async function process_variant(variant){
if (variant) {
console.log('doing stuff')
}
else {
console.log('doing other stuff')
}
const productVariableInput = {};
const variables = { input: productVariableInput };
try {
const {data, extensions} = await updatePrice({ variables });
const remaining_throttle = extensions.cost.throttleStatus.currentlyAvailable;
console.log("Remaining", remaining_throttle)
console.log(data)
// Change to a while loop to make sure you actually wait until resources are available
if (remaining_throttle < 100) {
console.log("WAITING")
await wait(18000);
}
} catch (e) {
console.log('error:', e);
}
console.log("AFTER")
return 0;
}
const redirectToModify = async (data, totalProducts) => {
await wait(20000);
let cursor;
const products = data.collection.product.edges;
totalProducts = totalProducts - products.length;
// Wait for all products to finish processing
for (var i = 0; i < products.length; i++){
await process_product(products[i]);
}
if (totalProducts > 0) {
console.log("Calling")
await wait(15000);
handleDiscountMore(data, cursor, totalProducts)
}
};
function updateQuery(previousResult, { fetchMoreResult }){
console.log("adding", fetchMoreResult);
redirectToModify(fetchMoreResult, pc);
return fetchMoreResult;
}
//Below function is Just for reference. It gets called before checking the throttleStatus above. afaik there's no problem with this
function handleDiscountMore(data, cursor, pc) {
console.log("Call received")
const variables = { id: data.collection.id, cursor };
fetchMore({ variables, updateQuery })
}
I'd like to check if uploading files are used in my database. So I'd like to list all files in the "uploads" folder and then check with SequelizeJs if item are finded with the right property.
My code seems not working as espected :
var promises = [];
fs.readdir('./public/uploads', (err, files) => {
if (!err) {
promises = files.filter(function(file) {
if (file !== 'csv' && file !== '.gitignore') {
// vérification dans FileModel
return models.FileModel.findOne({ where: { name: '/uploads/' + file } }).then(function(findedFile) {
if (!findedFile) {
return Promise.resolve(file).then(function() {
// Vérification dans VehiclePhotoModel
return models.VehiclePhotoModel.findOne({ where: { name: '/uploads/' + file } }).then(function(findedFile) {
if (!findedFile) {
return Promise.resolve(file);
}
});
});
}
});
}
});
}
});
Promise.all(promises).then(function(unusedFiles) {
response.render('file/_checkUnusedFiles', {
_layoutFile: false,
unusedFiles: unusedFiles
});
});
Edit #1 :
I've mixed up your code that doesn't work for me but I still get an issue, the Promise.resolve() is not returning the string name file.
const fileModels = [
{ model: models.FileModel, property: 'name' },
{ model: models.VehiclePhotoModel, property: 'name' }
];
function _checkUnusedFiles(request, response) {
var promises = [],
files = _.remove(fs.readdirSync('./public/uploads'), function(file) {
return file !== 'csv' && file !== '.gitignore';
});
promises = _.map(files, function(file) {
return fileModels.map(function(fileModel) {
var where = {};
where[fileModel.property] = file;
return fileModel.model.findOne({ where: where }).then(function(findedItem) {
if (!findedItem) {
return Promise.resolve(file);
}
});
});
});
Promise.all(promises).then(function(unusedFiles) {
response.render('optimizer/_checkUnusedFiles', {
_layoutFile: false,
unusedFiles: unusedFiles
});
});
}
With the above code soon you get into callback hell, which will result in issues like the ones you have mentioned. Anyways, so the problem is as follows
files.filter returns
<string[]> | <Buffer[]> | <fs.Dirent[]>
taken from documentation:https://nodejs.org/api/fs.html#fs_fs_readdir_path_options_callback
a better approach would be to loop and then do something like promises.append(...)
even better approach break down your code using async await to avoid callback hell and issues like this
You really don't need Promise.all in your case as it might complicated things even further and could be a performance bottleneck.
Something like this should work fine
const fs = require('fs')
const path = require('path')
// Mocking SequeliseJS (You can remove this and replace in code as well with original models
const modelsMock = {
FileModel: {
findOne: (query) => {
return new Promise((resolve, reject) => {
// rand will just randomize result between true and false
let rand = Math.floor(Math.random() * Math.floor(2))
resolve(rand == 1 ? resolve(true) : resolve(false))
})
}
},
VehiclePhotoModel: {
findOne: (query) => {
return new Promise((resolve, reject) => {
// rand will just randomize result between true and false
let rand = Math.floor(Math.random() * Math.floor(2))
resolve(rand == 1 ? resolve(true) : resolve(false))
})
}
}
}
run()
async function run() {
let promises = []
let existFiles = []
let doesntExistFiles = []
// If the goal is to use readdir async then uncomment the following and comment let files = fs.readdirSync('./public/uploads')
/*
const util = require('util')
const readdir = util.promisify(fs.readdir);
let files = await readdir('./public/uploads')
*/
let files = fs.readdirSync('./public/uploads')
// Filter files: (using array.filter is pretty useless to us here since we can't run it async unless we want to promisify it which there are plenty of examples out there, for now we will stick with good old fashion loop)
for(var i = 0; i < files.length; i++) {
const file = files[i]
if(path.extname(file) !== '.csv' && path.extname(file) !== '.gitignore') {
try {
let exist = await checkIfFileExist(file, modelsMock)
exist === true ? existFiles.push(file) : doesntExistFiles.push(file)
} catch (e) {
// Handle error here
}
}
}
response.render('file/_checkUnusedFiles', {
_layoutFile: false,
unusedFiles: unusedFiles //Note this will break the code as unusedFiles is undefined, I really dont know what is this variable hence I didn't change it but feel free to change it as you see it appropriate.
});
}
async function checkIfFileExist(file, models){
let query = `{ where: { name: '/uploads/' + ${file} } }`
try {
let existInFiles = await models.FileModel.findOne(query)
if(!existInFiles) {
let existInVehicle = await models.VehiclePhotoModel.findOne(query)
existInVehicle ? return true : return false
} else {
return true
}
}
catch(e) {
throw e
}
}
here is the full example working on Repl.it
https://repl.it/repls/DramaticAwfulSpreadsheets#index.js
EDIT1:
Like I mentioned in the comments I don't really suggest what you are doing (My opinion at least :)); however, if you flatten your array everything should be fine.
const fileModels = [
{ model: models.FileModel, property: 'name' },
{ model: models.VehiclePhotoModel, property: 'name' }
];
function _checkUnusedFiles(request, response) {
var promises = [],
files = _.remove(fs.readdirSync('./public/uploads'), function(file) {
// I think you want to check for extension here and not the file name? if so use this
// return (path.extname(file) !== '.csv' && path.extname(file) !== '.gitignore') //Make sure to require('path')
return file !== 'csv' && file !== '.gitignore';
});
promises = _.map(files, function(file) {
return fileModels.map(function(fileModel) {
var where = {};
// Your query seems wrong? at least different from last time. If it is intentional then you are good otherwise I think you meant to do
// where[fileModel.property] = `/uploads/${file}`
where[fileModel.property] = file;
return fileModel.model.findOne({ where: where }).then(function(findedItem) {
if (!findedItem) {
return Promise.resolve(file);
}
});
});
});
// Your promises array is more like a 2d array because each file will map to an array of 2 promises
// So if you flatten it, you should get a 1D promises array
promises = promises.flat()
Promise.all(promises).then(function(unusedFiles) {
response.render('optimizer/_checkUnusedFiles', {
_layoutFile: false,
unusedFiles: unusedFiles
});
});
}
If you don't flatten the output of promises looks something like the following
promises =
[
[ Promise { <pending> }, Promise { <pending> } ],
[ Promise { <pending> }, Promise { <pending> } ]
]
Each entry is a file which has 2 promises for each model
So when you do Promise.all(promises) it really doesn't know how interpret that in 2D arrays; However, when you flatten it, your output looks something similar to:
promises =
[
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> }
]
Now keep in mind unusedFiles output may look weird and that is because not every promise will actually resolve to something meaningful, at least that is what you are mapping.
I am trying to do a call which retrieves a list of categories. Inside this call I want to loop through the categories and retrieve the items for each category and return them all together. My call retrieves the categories perfectly before I added the loop to retrieve the items.
To double check my call to another controller works, I added a proof of concept block of code which you can see below is commented out. So I know it isn't the call to an external class.
Here is my code:
'use strict';
var mongoose = require('mongoose'),
MenuCategory = mongoose.model('MenuCategory');
module.exports = function(menuItemController) {
var mod = {
listEntireMenu(req, res) {
return new Promise(function(resolve, reject) {
var entireMenu = [];
MenuCategory.find({}, function(err, menuCategories) {
if (err) {
return reject(err)
} else {
//---------------------------
// PROOF OF CONCEPT THAT CALL TO OTHER CONTROLLER WORKS
//---------------------------
//
// var categoryWithItems = menuCategories[0].toObject();
// req.body.menuCategoryID = categoryWithItems._id;
// menuItemController.listAllMenuItemsByCategory(req, res).then((menuItems) => {
// if(menuItems)
// {
// return resolve(menuItems);
// }
// else
// {
// return { success: false }
// }
// });
//-----------------------------
for (var i = 0; i < menuCategories.length; i++) {
var categoryWithItems = menuCategories[i].toObject();
var subItems = [];
req.body.menuCategoryID = categoryWithItems._id;
menuItemController.listAllMenuItemsByCategory(req, res).then((menuItems) => {
if(menuItems)
{
subItems = menuItems;
}
else
{
return { success: false }
}
});
categoryWithItems.tester = { "itemsList" : subItems };
entireMenu.push(categoryWithItems);
}
return resolve(entireMenu)
}
});
}).then((menuCategories) => {
if(menuCategories)
{
return menuCategories
}
else
{
return { success: false }
}
});
},
}
return mod;
};
What I actually get returned is this :
[
{
"_id": "5ed16fxxxxxxxx95676e37",
"locationID": "5ed16xxxxxxxx7295676e36",
"menuCategoryName": "Category One",
"Created_date": "2020-05-29T20:26:34.991Z",
"__v": 0,
"tester": {
"itemsList": []
}
},
{
"_id": "5ed170xxxxxx95676e38",
"locationID": "5ed16xxxxxxxx7295676e36",
"menuCategoryName": "Category Two",
"Created_date": "2020-05-29T20:26:48.799Z",
"__v": 0,
"tester": {
"itemsList": []
}
}
]
Here is the call from the route.js :
app.get('/api/listEntireMenu', (req, res) => {
menuCategoryController.listEntireMenu(req, res).then(menuCategories => res.json(menuCategories));
})
It never writes the subItems into the object. Is this an async issue or something else? I am not sure how to solve this.
Thanks in advance.
i believe the reason the result of your call to resolve is being returned before the requests are able to complete...for this you need to wait until all the promises or requests have finished properly and returned.
There are two ways you can do this: you could either run them one by one and wait for each one to finish first or run them all concurrently until all of them are done.
Ofcourse the fastest way to do it would be to run them all concurrently so lets go for that way:
so to start, let us not use the for loop and instead remap the iterable array menuCategories to promises of the request, we will use your proof of concept code to make the array of promises
//...
Promise.all(
menuCategories.map((category) => {
let category_with_items = category.toObject();
req.body.menuCategoryID = category_with_items._id;
// here we need to return this since its the promise we are remapping to
return menuItemController.listAllMenuItemsByCategory(req, res)
.then((menuitems) => {
if(menuItems) {
return menuitems;
}
throw 'No menu items found'
});
});
)
// each promise will return menuitems so we have to wait for all the promises to complete
// then with the results of each promise we push the items into the entire menu
.then((itemslist) => {
itemslist.forEach((items) => entireMenu.push(items));
return entireMenu;
})
// lastly we need to handle any errors from the promises
.catch((error) => { success: false });
//...
So now we have...
listEntireMenu(req, res) {
return MenuCategory.find({}, function(err, menuCategories) {
if (err) {
throw err
} else {
entireMenu = [];
return /* the promise all call from above will go right here */;
}
}
I hope it works out, thanks...
I need help with code below. I get an array of items from the client then the goal is to save them in mongodb and return the list classified as 'saved' and 'failed' items. sample of failed items are those that are duplicate on a unique attribute.
I know the code below will not work because of variable scope. how do i get around it? the code below returns an empty array for both savedItems and failedItems. Thanks!
router.post('/addItems', async (req, res, next) => {
let items = req.body;
let result = {
savedItems: [],
failedItems: []
};
function saveData() {
for (i = 0; i < items.length; i++) {
item = items[i];
Model.create({ ...item }, (err, data) => {
if (err) {
result.failedItems.push(item);
} else {
result.savedItems.push(item);
}
});
}
return result;
}
saveData().then(result => {
res.send({
results: result
});
});
});
router.post('/addItems', async (req, res, next) => {
// use try catch when use async
try {
let items = req.body;
let result = {
savedItems: [],
failedItems: []
};
for (let i = 0; i < items.length; i++) {
const item = items[i];
// use the returned promise instead of callback for Model.create
const data = await Model.create({ ...item });
result.savedItems.push(item);
// if also need to handle failed item in result use anathor try catch inside
/*try {
const data = await Model.create({ ...item });
result.savedItems.push(item);
} catch( err ) {
result.failedItems.push(item);
}*/
}
res.send({
results: result
});
} catch( err ) {
// To all the errors unexpected errors + thrown rejected promises
res.send({
error: err
});
}
});
Your saveData method didn't return a promise, try this
function saveData() {
return new Promise(resolve => {
let items = req.body;
let result = {
savedItems: [],
failedItems: []
};
let promises = [];
for (i = 0; i < items.length; i++) {
item = items[i];
let promise = new Promise(resolve => {
Model.create({ ...item }, (err, data) => {
if (err) {
result.failedItems.push(item);
} else {
result.savedItems.push(item);
}
resolve();
});
});
promises.push(promise);
}
Promise.all(promises).then(() => resolve(result));
})
}
I need to iterate through an array. With each iteration, I need to update my database. I need to wait for the first update to be complete and then make the second update.
After searching through several answers, I found ASYNC/AWAIT feature of ES2017. However, I have not been able to implement it so far. The updates are happening randomly and not in a sequence. Please let me know how to implement ASYNC/AWAIT in this situation
Here is my code snippet:
function findRecipe(product, qty) {
return new Promise((resolve, reject) => {
Recipe.findOne({
product: product
}, (err, recipe) => {
if (err) {
reject(err)
} else {
for (let i = 0; i < recipe.items.length; i++) {
Item.findOne({
name: recipe.items[i].name
}, (err, item) => {
if (err) {
reject(err)
} else {
var lessAmt = recipe.quantities[i] * qty;
item.stock -= lessAmt;
item.save((err, item) => {
if (err) {
console.log(err)
} else {
resolve(item)
}
})
}
})
}
}
})
});
}
for (let i = 0; i < bill.product.length; i++) {
//Calling function for updates for each item
findRecipe(bill.product[i], bill.qty[i])
}
It looks like you are nearly there, Just Wrap the loop in a function and make it async.
async function updateAllRecipe(){
for(let i=0;i<bill.product.length;i++){
//Calling function for updates for each item
await findRecipe(bill.product[i],bill.qty[i])
}
}
But seriously though, I think you can leverage of parallelism here using the Promise.All. Is it really necessary to wait for the recipe to finish before queing the next findRecipe method? If not use the promise.all for it to perform faster
Async Await is easy to implement once you know the basic concept of asynchronous nature of Nodejs, I have used for...of loop here which also works asynchronously.
//Async function to perform database updates
async function findRecipe(product, qty) {
try {
let recipe = await Recipe.findOne({ product: product });
let i = 0;
for (itemObj of recipe.items) {
let item = await Item.findOne({ name: itemObj.name });
var lessAmt = recipe.quantities[i] * qty;
item.stock -= lessAmt;
let updatedItem = await item.save();
i++;
}
return true;
}
catch (err) {
return err;
}
}
async function someasyncFunction() {
for (let i = 0; i < bill.product.length; i++) {
//Calling function for updates for each item
await findRecipe(bill.product[i], bill.qty[i])
}
}
Use Promise.all and start process parallel. It'll increase the performance of API.
async function findRecipe(product, qty) {
try {
let recipe = await Recipe.findOne({
product: product
});
const items = await Promise.all(recipe.items.map(itemObj => Item.findOne({
name: itemObj.name
})));
const items = await Promise.all(items.map((item, i) => {
var lessAmt = recipe.quantities[i] * qty;
item.stock -= lessAmt;
return item.save();
}));
return true;
} catch (err) {
return err;
}
}
async function someasyncFunction() {
await Prmise.all(bill.product.map((product, i) => findRecipe(product, bill.qty[i])));
}