This question already has answers here:
Using async/await with a forEach loop
(33 answers)
Closed last year.
I'm trying to fetch json, check with mongodb if duplicate exist and if not, then to insert data into mongodb.
The problem is loops goes very fast, without waiting for duplicate check and insert.
What am I doing wrong here?
const fetch = require('node-fetch');
var MongoClient = require('mongodb').MongoClient;
async function fetchPhotos() {
MongoClient.connect("mongodb://localhost:27017", async function (err, dbo) {
const db = dbo.db('mydb')
if (err) throw err;
for (var i = 1; i < 100; i++) {
await fetch(`domain.com/?page=${i}`)
.then(res => res.json())
.then((response) => {
response.forEach(photo => {
console.log("checking if there is duplicate: " + photo.id);
var exists = db.collection("photos").countDocuments({"id": photo.id}, {limit: 1});
if (exists === 1) {
console.log("dupl found, next!");
} else {
db.collection("photos").insertOne(photo, function (err, res) {
if (err) throw err;
console.log("1 document inserted");
});
}
});
});
}
});
}
module.exports.fetchPhotos = fetchPhotos;
There are a few problems with your asynchronous code inside a loop.
a .forEach() loop does not wait for await. You have to use a for loop or while loop if you want it to wait for await.
You aren't using await on db.collection("photos").countDocuments({"id": photo.id}, {limit: 1});.
You aren't using await on db.collection("photos").insertOne(photo).
Do not mix plain callbacks and promises in the same flow of control. It makes it very difficult to code safely and with proper error handling.
You are missing appropriate error handling in a number of places.
You can restructure the whole thing to use the promise interface to your database and then sequence the iteration through the photos and simplify things like this:
const fetch = require('node-fetch');
var MongoClient = require('mongodb').MongoClient;
async function fetchPhotos() {
const dbo = await MongoClient.connect("mongodb://localhost:27017");
const db = dbo.db('mydb');
for (let i = 1; i < 100; i++) {
let response = await fetch(`http://example.com/?page=${i}`);
let data = await response.json();
for (let photo of data) {
console.log("checking if there is duplicate: " + photo.id);
let cnt = await db.collection("photos").countDocuments({"id": photo.id}, {limit: 1});
if (cnt > 0) {
console.log("dupl found, next!");
} else {
await db.collection("photos").insertOne(photo);
console.log("photo inserted");
}
}
}
}
// caller of fetchPhotos() gets a promise that tells you if the
// operation completed or had an error
module.exports.fetchPhotos = fetchPhotos;
Related
I have an array of addons and I want to insert them into the db table.
var addons = [sample,sample,.....]
return new Promise((resolve,reject) => {
addons.foEach(async addon => {
// first check if the items is in db
const response = await Kinex.where({}).from('table_name');
if(response.length == 0){
// insert new record
const insertResp = kinex('table_name').insert(addon)
addon.system_id = insertResp[0];
}else{
addon.system_id = response[0].id;
}
})
})
What I expected is to have unique record in the database, but the above code produced duplicate record in the database. Please help to find out the issue with the code.
The problem is running async function inside a loop. As mentioned by #Felix, forEach doesn't know about async functions and doesn't wait for your where query to return. If you wanna do things in async manner inside loops, you can do it with for..of loops. Also make sure to always use try/catch blocks while using async/await. Below is the code in your case:
const addons = [sample,sample,.....];
return new Promise(async (resolve, reject) => {
try {
for (let addon of addons) {
// first check if the items is in db
const response = await Kinex.where({}).from('table_name');
if (response.length) {
const insertResp = await kinex('table_name').insert(addon)
addon.system_id = insertResp[0];
} else addon.system_id = response[0].id;
resolve(); // resolve with whatever you wants to return
}
} catch (e) {
reject(e)
}
});
You can read more on for..of with async/await here.
As pointed by #Sándor, here's the code using Promise.all:
var addons = [sample, sample, .....]
return Promise.all(addons.map(async addon => {
// Do your async stuff here
// first check if the items is in db
const response = await Kinex.where({}).from('table_name');
if (response.length == 0) {
// insert new record
const insertResp = kinex('table_name').insert(addon)
addon.system_id = insertResp[0];
} else {
addon.system_id = response[0].id;
}
}))
I'm trying to read data from a MongoDB Atlas collection using Node.js. When I try to read the contents of my collection I get the error MongoError: Cannot use a session that has ended. Here is my code
client.connect(err => {
const collection = client
.db("sample_airbnb")
.collection("listingsAndReviews");
const test = collection.find({}).toArray((err, result) => {
if (err) throw err;
});
client.close();
});
I'm able to query for a specific document, but I'm not sure how to return all documents of a collection. I've searched for this error, I can't find much on it. Thanks
In your code, it doesn't wait for the find() to complete its execution and goes on to the client.close() statement. So by the time it tries to read data from the db, the connection has already ended. I faced this same problem and solved it like this:
// connect to your cluster
const client = await MongoClient.connect('yourMongoURL', {
useNewUrlParser: true,
useUnifiedTopology: true,
});
// specify the DB's name
const db = client.db('nameOfYourDB');
// execute find query
const items = await db.collection('items').find({}).toArray();
console.log(items);
// close connection
client.close();
EDIT: this whole thing should be in an async function.
Ran into the same issue when I updated the MongoClient from 3.3.2 to the latest version (3.5.2 as of this writing.) Either install only 3.3.2 version by changing the package.json "mongodb": "3.3.2", or just use async and await wrapper.
If still the issue persists, remove the node_modules and install again.
One option is to use aPromise chain. collection.find({}).toArray() can either receive a callback function or return a promise, so you can chain calls with .then()
collection.find({}).toArray() // returns the 1st promise
.then( items => {
console.log('All items', items);
return collection.find({ name: /^S/ }).toArray(); //return another promise
})
.then( items => {
console.log("All items with field 'name' beginning with 'S'", items);
client.close(); // Last promise in the chain closes the database
);
Of course, this daisy chaining makes the code more synchronous. This is useful when the next call in the chain relates to the previous one, like getting a user id in the first one, then looking up user detail in the next.
Several unrelated queries should be executed in parallel (async) and when all the results are back, dispose of the database connection.
You could do this by tracking each call in an array or counter, for example.
const totalQueries = 3;
let completedQueries = 0;
collection.find({}).toArray()
.then( items => {
console.log('All items', items);
dispose(); // Increments the counter and closes the connection if total reached
})
collection.find({ name: /^S/ }).toArray()
.then( items => {
console.log("All items with field 'name' beginning with 'S'", items);
dispose(); // Increments the counter and closes the connection if total reached
);
collection.find({ age: 55 }).toArray()
.then( items => {
console.log("All items with field 'age' with value '55'", items);
dispose(); // Increments the counter and closes the connection if total reached
);
function dispose(){
if (++completedQueries >= totalQueries){
client.close();
}
}
You have 3 queries. As each one invokes dispose() the counter increments. When they've all invoked dispose(), the last one will also close the connection.
Async/Await should make it even easier, because they unwrap the Promise result from the then function.
async function test(){
const allItems = await collection.find({}).toArray();
const namesBeginningWithS = await collection.find({ name: /^S/ }).toArray();
const fiftyFiveYearOlds = await collection.find({ age: 55 }).toArray();
client.close();
}
test();
Below is an example of how Async/Await can end up making async code behave sequentially and run inefficiently by waiting for one async function to complete before invoking the next one, when the ideal scenario is to invoke them all immediately and only wait until they all are complete.
let counter = 0;
function doSomethingAsync(id, start) {
return new Promise(resolve => {
setTimeout(() => {
counter++;
const stop = new Date();
const runningTime = getSeconds(start, stop);
resolve(`result${id} completed in ${runningTime} seconds`);
}, 2000);
});
}
function getSeconds(start, stop) {
return (stop - start) / 1000;
}
async function test() {
console.log('Awaiting 3 Async calls');
console.log(`Counter before execution: ${counter}`);
const start = new Date();
let callStart = new Date();
const result1 = await doSomethingAsync(1, callStart);
callStart = new Date();
const result2 = await doSomethingAsync(2, callStart);
callStart = new Date();
const result3 = await doSomethingAsync(3, callStart);
const stop = new Date();
console.log(result1, result2, result3);
console.log(`Counter after all ran: ${counter}`);
console.log(`Total time to run: ${getSeconds(start, stop)}`);
}
test();
Note: Awaiting like in the example above makes the calls sequential again. If each takes 2 seconds to run, the function will take 6 seconds to complete.
Combining the best of all worlds, you would want to use Async/Await while running all calls immediately. Fortunately, Promise has a method to do this, so test() can be written like this: -
async function test(){
let [allItems, namesBeginningWithS, fiftyFiveYearOlds] = await Promise.all([
collection.find({}).toArray(),
collection.find({ name: /^S/ }).toArray(),
collection.find({ age: 55 }).toArray()
]);
client.close();
}
Here's a working example to demonstrate the difference in performance: -
let counter = 0;
function doSomethingAsync(id, start) {
return new Promise(resolve => {
setTimeout(() => {
counter++;
const stop = new Date();
const runningTime = getSeconds(start, stop);
resolve(`result${id} completed in ${runningTime} seconds`);
}, 2000);
});
}
function getSeconds(start, stop) {
return (stop - start) / 1000;
}
async function test() {
console.log('Awaiting 3 Async calls');
console.log(`Counter before execution: ${counter}`);
const start = new Date();
const [result1, result2, result3] = await Promise.all([
doSomethingAsync(1, new Date()),
doSomethingAsync(2, new Date()),
doSomethingAsync(3, new Date())
]);
const stop = new Date();
console.log(result1, result2, result3);
console.log(`Counter after all ran: ${counter}`);
console.log(`Total time to run: ${getSeconds(start, stop)}`);
}
test();
other people have touched on this but I just want to highlight that .toArray() is executed asynchronously so you need to make sure that it has finished before closing the session
this won't work
const randomUser = await db.collection('user').aggregate([ { $sample: { size: 1 } } ]);
console.log(randomUser.toArray());
await client.close();
this will
const randomUser = await db.collection('user').aggregate([ { $sample: { size: 1 } } ]).toArray();
console.log(randomUser);
await client.close();
client.connect(err => {
const collection = client
.db("sample_airbnb")
.collection("listingsAndReviews");
const test = collection.find({}).toArray((err, result) => {
if (err) throw err;
client.close();
});
});
I'm trying to return the hole master/detail object to client, but detail is coming as an empty array
Like this post I've also ended with the same problem:
"Can't call res.send(data) inside the loop because res.send() can only be called once."
"But if I call res.send(array) outside of the loop, the array is still empty"
What is the right way to do it?
I'm trying not to use use asyn
var getMasterDetail = function (req, res) {
const key = "Detail";
var list = {}
list[key] = []
var modelsMaster = objModels.ObjMaster
var modelsDetail = objModels.objDetail
modelsMaster.getMasters(objModels.hdb, (e, master) => {
if (e) {
return console.log(e);
}
for (i = 0; i < master.length; i++) {
modelsDetail.getDetails(objModels.hdb, master[i].nrMaster, (e, detail) => {
if (e) {
return console.log(e);
}
for (j = 0; j < detail.length; j++) {
list[key].push(detail[j])
}
})
master[i].DetailList = list
};
res.send({ MasterDetail: master })
})
};
Thanks.
UPDATE: The answer from #Hammerbot was almost right, but,
I have not notice at the time, that I was getting the same detail for all masters.
Ex. {master:{1,2,3,4,5,6}, master{1,2,3,4,5,6}} instead of {master:{1,2,3}, master{4,5,6}}
I Have no any idea why and how to fix it. I've tried to clean the list befor the loop, and move master master[i].DetailList, creating a second Promisse for the second loop, no success.
You should use promises for that. Here is an example that should resolve your problem:
var getMasterDetail = function (req, res) {
const key = "Detail";
var list = {}
list[key] = []
var modelsMaster = objModels.ObjMaster
var modelsDetail = objModels.objDetail
modelsMaster.getMasters(objModels.hdb, (e, master) => {
if (e) {
return console.log(e);
}
const promises = []
for (i = 0; i < master.length; i++) {
const promise = new Promise(resolve => {
master[i].DetailList = list
modelsDetail.getDetails(objModels.hdb, master[i].nrMaster, (e, detail) => {
if (e) {
return console.log(e);
}
for (j = 0; j < detail.length; j++) {
list[key].push(detail[j])
}
resolve()
})
})
promises.push(promise)
}
Promise.all(promises).then(() => {
res.send({ MasterDetail: master })
})
})
};
As you can see, before the loop I initiate a promises array. Inside the loop, I create a promise by iteration that gets resolved when the callback has finished.
I push the promise into the promises Array, and at the end I use Promise.all() to wait for all the promises to get resolved before sending the result in the response.
i have array of db like
const dbArr = ["http://localhost:5984", "http://xyz_couchdb.com:5984"]
data to insert
let data ={
_id: 324567,
name: Harry,
gerder: male
}
here is the logic i am using nano module
return new Promise((resolve, reject) => {
let res = [];
let rej = [];
let counter = 0;
for(let i = 0; i < dbArr.length ; i++){
dbArr[i].insert(data, (err, body) => {
err ? rej.push(err) : res.push(body)
if(counter === obj.dbArray.length -1){
rej.length ? reject(rej) : resolve(res)
}
counter++;
})
}
})
what can be the best possible way to achieve this using promise or async module or anything.
In the following example, we gotta use Array.map to create one promise for each element of dbArr, then we gotta wait all promises to end using Promise.all. The catch is here so we handle the errors.
function getAll(dbArr) {
return Promise.all(dbArr.map(x => x.insert(data)));
}
getAll(dbArr)
.then((rets) => {
// Handle the returns
// They are in an array
})
.catch((err) => {
// Handle the error
});
EDIT :
Ok after checking out the documentation of node-couchdb (the one I suppose you use) - I saw that the .insert() method do not return a Promise but only a callback.
So we gotta transform the method, so it will return a Promise using util.Promisify()
const {
promisify,
} = require('util');
function getAll(dbArr) {
return Promise.all(dbArr.map(x => promisify(x.insert)(data)));
}
getAll(dbArr)
.then((rets) => {
// Handle the returns
// They are in an array
})
.catch((err) => {
// Handle the error
});
I need to iterate between two values and create/touch files (I/O) on each iteration.
I'm using the fs-promise module to do so asynchronously:
const path = require('path');
const fsp = require('fs-promise');
function addPages(startAt, pages, mode) {
let htmlExt = mode.HTML;
let cssExt = mode.CSS;
fsp.readFile(path.join('.', 'templates', 'body.html'), { encoding: 'utf-8' })
.then((content) => {
// return Promise.all(() => {}).then().catch(); // Do this.
for (let i = startAt, endAt = startAt + pages; i < endAt; i++) {
console.log(i);
fsp.writeFile(path.join('.', 'manuscript', `page-${i}`, `style.${cssExt}`), '')
.then(() => { console.log('Yay!') })
.catch(console.log.bind(console));
// fsp.writeFile(path.join('.', 'manuscript', `page-${i}`, `style.${cssExt}`), '')
// .then((i, templateHTML) => {
// fsp.writeFile(path.join('.', 'manuscript', `page-${i}`, `body.${htmlExt}`), content);
// })
// .catch((err) => {
// console.log.bind(console);
// });
}
})
.catch((err) => {
if (err) return error('Couldn\'t create pages', err);
});
Now I did read that Promises.all([Array of promises]) is the way to go for looping inside the then() scope, but the question is why/how?
I'm unable to wrap my head around why the for-loop doesn't execute before the context moves out of the promised then() scope, and then how should I get to the expected outcome.
const path = require('path');
const fsp = require('fs-promise');
function addPages(startAt, pages, mode) {
let htmlExt = mode.HTML;
let cssExt = mode.CSS;
return fsp.readFile(path.join('.', 'templates', 'body.html'), { encoding: 'utf-8' })
.then((content) => {
var pendingWrites = [];
for (let i = startAt, endAt = startAt + pages; i < endAt; i++) {
let filename = path.join('.', 'manuscript', `page-${i}`, `style.${cssExt}`);
let thisWrite = fsp.writeFile(filename, '');
pendingWrites.push(thisWrite);
}
return Promise.all(pendingWrites);
})
.catch((err) => {
// either fully recover from the error or rethrow
console.log("Could not add pages: ", err);
throw err;
});
}
As elaborated in the comments, resist the temptation to introduce none-functional .catch() handlers into your promise chain.
Non-functional means in this case: It does not recover from the error and does not rethrow the error. A catch handler that does not throw marks an error as handled, i.e. it returns a resolved promise, not a rejected one. This makes proper error handling later in the promise chain impossible. It's bad practice and unhelpful.
If you want to log the error, log it and rethrow it. If you have fully recovered from the error and subsequent code is unimpeded, don't rethrow.