MongoDB Result can console.log() but not pushed to array NodeJS - node.js

I have a list of users I would like to look up in my MongoDB database and once I look them up, I want to store them in an array so I can do things with the data. I'm using the official mongodb package in NodeJS. Here's the code
var chatsData = []
for (let userID of chatIDs) {
db.collection('users').find({ '_id': ObjectId(userID) }).toArray((err, result) => {
if (err) throw err
chatsData.push(result)
console.log(result)
})
}
}
console.log('vvv Final data in array: vvv')
console.log(chatsData)
When I run the code this, I get this vvv
vvv Final data in array: vvv
[]
[
{
_id: 5eae4c90ad1dd6304c69a75a,
usnm: 'gohjunhao',
eml: 'junhao#gmail.com',
phnm: '00000000',
pswd: '$2a$10$IUaxiweNrUUwxZP6XEQfFeTTnbta13/kv6DdebwJ0WT/bM.3fc5ay',
register_date: 2020-05-03T04:46:08.054Z,
__v: 0
}
]
[
{
_id: 5ead401f8059852114bf9867,
usnm: 'gfox.2020',
eml: 'carrie#gmail.com',
phnm: '11111111',
pswd: '$2a$10$UYaEraoI4Kj0dI.nt5Hbr.LgDL1TNtDOsz7tcxETJW7HRtmgWo.UK',
register_date: 2020-05-02T09:40:47.684Z,
__v: 0
}
]
How do I get a proper array of data in my array so it can be used later? Is what I'm doing wrong? Do I need to use a .then() statement or an async await?
Here's the full code
MongoClient.connect(url, { useUnifiedTopology: true }).then(async chooseDB => {
db = chooseDB.db('nodejs')
// Get a list of all tables
db.listCollections().toArray((err, result) => {
/***** YOU DON'T NEED TO UNDERSTAND THIS PART OF THE CODE ******/
if (err) throw err
var chatList = []
var chatIDs = []
for (let i = 0; i < result.length; i++) {
const table = result[i]
if (table.name.indexOf(data) > 1) {
// add tables with personal id to chatList
chatList.push(table.name)
// add id's of other chats to out table
chatIDs.push(table.name.replace('dm', '').replace('~', '').replace(data, ''))
}
}
/***** IT'S JUST HOW I GET MY CHAT ID'S *****/
// Get data on users
var chatsData = []
for (let userID of chatIDs) {
try{
let temp = await db.collection('users').find({ '_id': toMongoObjectId(userID) }).toArray()
chatsData.push(temp)
}
catch(error) {
console.log(error)
}
}
console.log('vvv Final data in array: vvv')
console.log(chatsData)
toClient.userData = chatsData
toClient.users = chatList
socket.emit('res_chatList', toClient)
})
})

This can be solved using async-await, write async to the function of then block-like,
mongoclient.connect().then( async (cli) => {
db = cli.db(dbName);
...
})
your logic to fetch data will be
var chatsData = []
for (let userID of chatIDs) {
try{
let temp = await db.collection('users').find({ '_id': ObjectId(userID) }).toArray();
chatsData.push(temp);
}
catch(error) {
console.log(error);
}
}
console.log(chatsData);

Related

Getting an {"message":"Invalid update pipeline operator: \"_id\""} error

I am trying to update two rows in my players table based on the id. I am trying to use the updateMany method where id can be found in an array of id's but I am getting the {"message":"Invalid update pipeline operator: \"_id\""} error. I checked the array to make sure it is valid id's. Here is my code
const winningTeam = asyncHandler(async (req, res) => {
req.body.forEach((element) => {
element.wins += 1;
element.lastPlayed = Date.now();
element.percentage = (element.wins / (element.wins + element.wins)) * 1000;
});
let usersId = [];
usersId.push(req.body[0]._id);
if (req.body.length === 2) {
usersId.push(req.body[1]._id);
}
const player = await Player.updateMany({ _id: { $in: usersId } }, req.body);
if (player) {
res.status(200).json(player);
} else {
res.status(400);
throw new Error("Invalid Data");
}
});
You should use $set property for the update parameter. I'm not sure about the structure of your req.body but it should be something like this:
Player.updateMany({ _id: { $in: usersId } }, {$set: req.body});
instead of this:
Player.updateMany({ _id: { $in: usersId } }, req.body);
Take a look at docs for updateMany

Remove object array items by comparing object array from mongodb

There is document which is having array of object inside.
Like
Objectid('')
fruits : [{_id:'2'},{_id:'3'},{_id:'4'}]
I want to delete these items fruits_id = [{_id:'3'},{_id:'4'}].
fruit_group.updateOne(
{collection_id: collection_id},
{$pullAll: {"fruits": fruits_id}}
)
so far i found below logic which i think is not efficient.
routes.post('/removeFruits', async (request, response, next) => {
var post_data = request.body;
var collection_id = post_data.collection_id;
var fruits_ids = JSON.parse(post_data.fruits_ids);
var prev_fruits;
await fruit_group.findOne({'collection_id': collection_id}, function (err, result) {
if (err) {
console("Some error occurred");
response.json({'message': "Some error occurred", 'result': 'false'});
}
prev_fruits = result.fruits;
});
for (var i = 0; i < fruits_ids.length; i++) { // this will delete all occurring items from array
var key = fruits_ids[i].user_id;
prev_fruits.filter(x => x.user_id === key).forEach(x => prev_fruits.splice(prev_fruits.indexOf(x), 1));
}
await fruit_group.updateOne({'collection_id': collection_id}, {$set: {'fruits': prev_fruits}}, function (err, result) {
if (err) {
response.json({'message': "Some error occurred", 'result': 'false'});
}
response.json({'message': 'Deletion successfully', 'result': 'true'});
});
});
is there anyway to achieve the same result?
Assuming fruits_id = [{ _id: '3' }, { _id: '4' }], you could do something like this using $pull and $in:
await fruit_group.updateOne({'collection_id': collection_id}, { $pull: { fruits: { $in: fruits_id }}})
This follows the example of removing all items that equal a specified value.

How to get promises from nested arrays?

Can somebody help me with this?
I am trying to scrape a website and store the collected data in a Json file. I'm using cheerios and request-promise.
The Json structure goes like that: companys > packages > cities
"companies": [
{
"id": 0,
"name": "companyName",
"url": "https://www.url-company.net/",
"packages": [
{
"id": 0,
"name": "general",
"url": "https://www.url-company.net/package",
"cities": [
{
"id": 0,
"name": "cityName",
"url": "https://www.url-company.net/package/city",
},
...]
}
...]
}
..]
I have extracted the array of companies from this site.
Each COMPANY has a specific url --> from every url I scraped the
packages for each company.
Each PACKAGE has a specific url --> from
every url I want to scrape the cities for each package but I am NOT
able to do it.
I am only able to populate companies and packagesByCompany, but I'm lost when trying to populate citiesByPackage:
const rp = require('request-promise');
const cheerio = require('cheerio');
const jsonfile = require('jsonfile');
const baseUrl = 'https://www.base-url-example.net';
scrapeAll();
function scrapeAll() {
return scrapeCompanies().then(function (dataCompanys) {
//Map every endpoint so we can make a request with each URL
var promises = dataCompanys.map(function (company) {
return scrapePackagesByCompany(company) // Populate each company with all the array of packages from this company
});
return Promise.all(promises);
})
.then(function(promiseArray) { // Need help here!!!!
var promise4all = Promise.all(
promiseArray.map(function(company) {
return Promise.all( // This is NOT working, I do not know how to get promises from nested arrays
company.packages.map(function(package) {
return Promise.all(
scrapeCitiesByPackage(package) // Try to populate each package with all the array of cities from this package
);
})
);
})
);
return promise4all;
})
.then(function (data) {
saveScrapedDateIntoJsonFile(data);
return data;
})
.catch(function (err) {
return Promise.reject(err);
});
}
function scrapeCompanies() {
return rp(baseUrl)
.then(function(html){
const data = [];
let companysImg = '#content section .elementor-container > .elementor-row > .elementor-element.elementor-top-column .elementor-widget-wrap .elementor-widget-image >.elementor-widget-container > .elementor-image';
let $ = cheerio.load(html);
$(companysImg).each(function(index, element){
const urlCompany = $(element).find('a').attr('href');
const imgCompany = $(element).find('img').data('lazy-src');
if (urlCompany && imgCompany) {
const nameCompany = urlCompany;
const company = {
id : index,
name: nameCompany,
url : baseUrl + urlCompany,
img: imgCompany,
};
data.push(company);
}
});
return data;
})
.catch(function(err){
//handle error
console.error('errorrr2', err);
});
}
function scrapePackagesByCompany(company) {
return rp(company.url)
.then(function(html){
company.packages = [];
let packagesImg = '#content section .elementor-container > .elementor-row > .elementor-element.elementor-top-column .elementor-widget-wrap .elementor-widget-image >.elementor-widget-container > .elementor-image';
let $ = cheerio.load(html);
$(packagesImg).each(function(index, element){
const urlPackage = $(element).find('a').attr('href');
const imgPackage = $(element).find('img').data('lazy-src');
if (urlPackage && imgPackage) {
const namePackage = urlPackage.text();
const package = {
id : index,
name: namePackage,
url : urlPackage,
img: imgPackage,
};
company.packages.push(package);
}
});
return company;
})
.catch(function(err){
//handle error
console.error('errorrr2', err);
});
}
function scrapeCitiesByPackage(insurancePackage) {
return rp(insurancePackage.url)
.then(function(html){
insurancePackage.cities = [];
let citiesLinks = '#content section .elementor-container > .elementor-row > .elementor-element .elementor-widget.elementor-widget-posts .elementor-posts-container article';
let $ = cheerio.load(html);
$(citiesLinks).each(function(index, element) {
const $linkCity = $(element).find('a');
const urlCity = $linkCity.attr('href');
const nameCity = $linkCity.text();
if (urlCity && nameCity) {
const city = {
id : index,
name: nameCity,
url : urlCity,
};
insurancePackage.cities.push(city);
}
});
return insurancePackage;
})
.catch(function(err){
//handle error
console.error('errorrr2', err);
});
}
function saveScrapedDateIntoJsonFile(data) {
jsonfile.writeFile(
'./data/company.json',
{companies : data },
//data,
{spaces: 2},
function(err) {
console.error('errorrr', err);
});
}
Thanks in advance :)
What you are trying could be made to work but it's arguably better for scrapePackagesByCompany() and scrapeCitiesByPackage() simply to deliver data, and to perform all the "assembly" work (ie bundling the delivered arrays into higher level objects) in scrapeAll().
You can write something like this:
scrapeAll()
.catch(function(err) {
console.log(err);
});
function scrapeAll() {
return scrapeCompanies()
.then(function(companies) {
return Promise.all(companies.map(function(company) {
return scrapePackagesByCompany(company)
.then(function(packages) {
company.packages = packages; // assembly
return Promise.all(packages.map(function(package) {
return scrapeCitiesByPackage(package)
.then(function(cities) {
package.cities = cities; // assembly
});
}));
});
}))
.then(function() {
return saveScrapedDateIntoJsonFile(companies);
});
});
}
Then it's fairly trivial to simplify scrapePackagesByCompany() and scrapeCitiesByPackage(package) such that they deliver packages array and cities array respectively.

How to implement async in for loop?

I have a collection called 'alldetails' which have the details of some collection
{
"name" : "Test1",
"table_name" : "collection1",
"column_name" : "column1"
},
{
"name" : "Test2",
"table_name" : "collection2",
"column_name" : "column2"
},
{
"name" : "Test3",
"table_name" : "collection3",
"column_name" : "column3"
}
I have collection1,collection2 and collection3 which have column1,column2,colum3 respectively
I have to fetch all the name from the 'alldetails' and I have to get the min and max value of other table based on the column name.
So I want the output like below
{name: ["Test1","Test2","Test3"],
date: [{min_date: "2018-12-01", max_date: "2018-12-31", name: "Test1"},
{min_date: "2018-12-01", max_date: "2018-12-31", name: "Test2"},
{min_date: "2018-12-01", max_date: "2018-12-31", name: "Test3"}]
}
I tried the below code because of non blocking its not waiting the response.
alldetails.find({}, { _id: 0 }).then(async function(result) {
let result_data = {};
let resolvedFinalArray = {};
let array = [];
result_data["name"]= [];
result_data["date"] = [];
resolvedFinalArray = await Promise.all(result.map(async value => {
result_data["name"].push(value.name)
getResult(value.table_name,value.column_name,function(response){
result_data["date"].push({min_date: response.minvalue, max_date: response.maxvalue, name:value.name})
});
}));
setTimeout(function()
{
console.log(resolvedFinalArray);
}, 3000);
});
Please suggest me a solution.
If you want to wait for getResult then you need to return Promise from result.map callback.
You are not pushing anything to resolvedFinalArray so why bother with console.log(resolvedFinalArray)
alldetails.find({}, {_id: 0}).then(async (result) => {
let result_data = {};
result_data["name"] = [];
result_data["date"] = [];
await Promise.all(result.map(value => {
// create Promise that resolves where getResult callback is fired
return new Promise((resolve) => {
getResult(value.table_name, value.column_name, (response) => {
result_data["name"].push(value.name);
result_data["date"].push({
min_date: response.minvalue,
max_date: response.maxvalue,
name: value.name
});
resolve();
});
});
}));
console.log(result_data);
});
or using for loop
alldetails.find({}, {_id: 0}).then(async (result) => {
let result_data = {};
result_data["name"] = [];
result_data["date"] = [];
for (let i = 0; i < result.length; i++) {
const value = result[i];
await new Promise((resolve) => {
getResult(value.table_name, value.column_name, (response) => {
result_data["name"].push(value.name);
result_data["date"].push({
min_date: response.minvalue,
max_date: response.maxvalue,
name: value.name
});
resolve();
});
});
}
console.log(result_data);
});
use async.eachOfLimit if you want to apply an async function on all element of an array:
var async = require("async");
var array = [{_id: "...."},{...},{...}];
async.eachOfLimit(array, 1, function(element, index, cb){
myAsyncFunctionWithMyElement(element, function(err){
return cb(err);
});
}, function(err){
// final callback
});
The array forEach method won't work with async function (unless you do deeply evil things like redefining the prototype). This question has a nice insight of the internal.
If you don't want to rely on external libraries, an easy (and my favourite) approach is something like:
for (let i = 0; i < <your array>.length; i++ ) {
await Promise.all( <your logic> );
}
Just adapt it to your need! :)
You might want to use the for await of loop. See this blog post for details.
This, IMHO, is the most modern way to do it, and it doesn't require you to load any external dependencies, since it is built-in to the language itself. It's basically very similar to the classical for of loop.
This should work, if all lexical scope are taken to consideration. Async each is also is better option it would reduce if else blocks and manage promise for you.
alldetails.find({}, { _id: 0 })
.exec((err, result) => {
if (!err) {
let resolvedFinalArray = [];
result.map((value) => {
resolvedFinalArray.push({
name: value.name,
date: []
});
getResult(value.table_name, value.column_name, (err, response) => {
if (!err) {
resolvedFinalArray[resolvedFinalArray.indexOf(value.name)]['date'].push({
min_date: response.minvalue,
max_date: response.maxvalue,
name:value.name
});
} else {
// Send your error messsage.
// res.status(500).send(err);
}
});
});
console.log(resolvedFinalArray);
// res.send(resolvedFinalArray);
} else {
// Send your error messsage.
// res.status(500).send(err);
}
});

using express to loop through a mongoose schema function ,collecting data into an array and rendering the page finally with all the data

Nodejs +Express+Mongoose beginner's question:
*So I came across this problem where I had an array of categories (clothes category). For each category, I wanted to fetch results from the database. So I did what a normal person would do. I used a for loop, fetched the product, stored in an array, then fetched next product, stored in into the same array and so on....... (same process for men and women both ). Now i want to render my view with the data but it isn't accessible coz of the asynchronous nature of node js. I totally understand it. So what I am looking for is an alternative/solution for my problem *
My express code explains my agony better. Please have a look:
//INDEX PAGE
router.get('/', (req, res, next) => { //anonymous callback function
let allCategories;
let womenFashion = [],
menFashion = [];
Clothing.groupByBodyPart(function(err, categories) {
if (categories) {
allCategories = categories.slice(0);
for (let i = 0; i < allCategories.length; i++) { //looping
let category = allCategories[i]._id; //gives a category, eg.,footwear,bottomwear
Clothing.getCategoryWise(category, 'M', function(err, products) { //products here will be an array of objects
if (products) {
menFashion[i] = products.slice(0); //storing products into an array for future use
console.log(menFashion[i]); //accessible here one at a time Eg.,[{slippers},{shoes}]
}
});
Clothing.getCategoryWise(category, 'F', function(err, products) {
if (products) {
womenFashion[i] = products.slice(0); //same as above
console.log(womenFashion[i]); //same as above
}
});
}
}
console.log(menFashion[0]); // not accessible here, so I can't render my page
res.render('index.pug', {
allCategories,
menFashion,
womenFashion
}); //won't work as menFashion and womenFashion aren't available
});
});
Here you go with the mongoose static function:
//get categorywise products
clothingSchema.statics.getCategoryWise = function(bodyPart,gender,callback){
Clothing.aggregate([
{ $match: {'category.bodyPart': bodyPart,
'category.gender': gender
}
},
{ $group: {_id: "$category.type" }
},
{ $sort: {_id: 1 }
}
])
.exec((err,products)=>{
if(err){
return callback(err);
}else if(!products){
let err = new Error('No Product Found!');
err.status = 401;
return callback(err);
}
return callback(null,products);
});
}
Just for the record
Everything is working great, I am just having trouble rendering my page because the menFashion and womenFashion array aren't accesible outside the callback.
A thanks in advance :)
UPDATE:
I solved it myself,but still thnks guys (specially #faiz)
My solution basically includes nesting:
//INDEX PAGE
router.get('/',(req,res,next)=>{ //anonymous callback function
let allCategories;
let womenFashion = [], menFashion = [];
Clothing.groupByBodyPart(function(err,categories){
if(categories){
allCategories = categories.slice(0);
for(let i = 0; i < allCategories.length; i++){ //looping
let category = allCategories[i]._id; //gives a category, eg.,footwear,bottomwear
Clothing.getCategoryWise(category,'M',function(err,products){ //products here will be an array of objects
if(products){
menFashion.push(products);
menFashion[i] = products.slice(0); //storing products into an array for future use
//console.log(menFashion[i]); //accessible here on at a time Eg.,[{slippers},{shoes}]
}
Clothing.getCategoryWise(category,'F',function(err,products){
if(products){
womenFashion[i] = products.slice(0); //same as above
//console.log(womenFashion[i]); //same as above
}
if(i == allCategories.length-1){
console.log('men',menFashion); //everything accessible
console.log('men',menFashion); //everything accessible
res.render('index.pug',{allCategories,menFashion,womenFashion}); //tadaaaaaaaa
}
});
});
}
}
});
});
You can use something like this.
I used two Promise.all calls to make sure that you have two results that you can use. You can just as well use one Promise.all
router.get('/', (req, res, next) => { //anonymous callback function
let allCategories;
let womenFashion = [],
menFashion = [];
Clothing.groupByBodyPart(function (err, categories) {
if (categories) {
allCategories = categories.slice(0);
const menPromises = [];
const womenPromises = [];
for (let i = 0; i < allCategories.length; i++) { //looping
let category = allCategories[i]._id; //gives a category, eg.,footwear,bottomwear
menPromises.push(
new Promise((resolve, reject) => {
Clothing.getCategoryWise(category, 'M', function (err, products) { //products here will be an array of objects
if (products) {
menFashion[i] = products.slice(0); //storing products into an array for future use
resolve(menFashion[i]);
console.log(menFashion[i]); //accessible here one at a time Eg.,[{slippers},{shoes}]
}
})
})
);
womenPromises.push(
new Promise((resolve, reject) => {
Clothing.getCategoryWise(category, 'F', function (err, products) { //products here will be an array of objects
if (products) {
womenFashion[i] = products.slice(0); //storing products into an array for future use
resolve(womenFashion[i]);
console.log(womenFashion[i]); //accessible here one at a time Eg.,[{slippers},{shoes}]
}
})
})
);
}
Promise.all([Promise.all(menPromises), Promise.all(womenPromises)]).then(([menResults, womenResults]) => {
console.log(menFashion[0]); // not accessible here, so I can't render my page
res.render('index.pug', {
allCategories,
menFashion,
womenFashion
}); //won't work as menFashion and womenFashion aren't available
});
}
});
});

Resources