I'm not capturing the connect function to return in the getAllowedEmails function, when I do console.log in allowedEmails, it returns the emails correctly, but when I assign to the variable emails, it is returning empty. I think it's an async await problem but can't figured out.
static async getAllowedEmails() {
var MongoClient = require("mongodb").MongoClient;
//db url
let emails = [];
await MongoClient.connect(url, async function (err, client) {
const db = client.db("data-admin");
var myPromise = () => {
return new Promise((resolve, reject) => {
db.collection("users")
.find({})
.toArray(function (err, data) {
err ? reject(err) : resolve(data);
});
});
};
var result = await myPromise();
client.close();
let allowedEmails = [];
result.map((email) => allowedEmails.push(email.email));
console.log(allowedEmails)
emails = allowedEmails;
});
console.log(emails)
return emails;
}
Your code has couple of issues, I have fixed few and enhanced it, given below is the basic code, try to test it and if everything works then enhance it as needed:
const MongoClient = require("mongodb").MongoClient;
async function getAllowedEmails() {
let client;
try {
const allowedEmails = [];
client = await MongoClient.connect(url);
const db = client.db("data-admin");
const result = await db.collection("users").find({}).toArray();
result.map((email) => allowedEmails.push(email.email));
console.log(allowedEmails)
client.close();
return allowedEmails;
} catch (error) {
(client) && client.close();
console.log(error)
throw error;
}
}
Related
sessionFilter.js
'use strict';
const knex = require('../config/database')
class SessionFilter{
async setPermissionFilterArray(data){
return new Promise(async (resolve, reject) => {
try {
const sql_data = await knex.select(knex.raw(sql));
var all_permission = [];
sql_data.forEach(function(val) {
if (all_permission[val.event_ref] === undefined) all_permission[val.event_ref] = [];
if (all_permission[val.event_ref][val.event_slug] === undefined) all_permission[val.event_ref][val.event_slug] = [];
all_permission[val.event_ref][val.event_slug][val.event_slug_key] = {
permission:val.permission,
no_permission:val.no_permission,
sql_where_clause:val.sql_where_clause
}
});
resolve(all_permission);
} catch (error) {
reject(error.message);
}
})
}
}
module.exports = SessionFilter;
server.js
const filter = new SessionFilter();
const set_permission_filter_array = await filter.testFunction(data);
console.log(set_permission_filter_array);
return res.json(set_permission_filter_array);
when i console set_permission_filter_array variable showing data well but when i return set_permission_filter_array variable showing blank array at frontend. how can i get data? please help me.
Your function will automaticaly return a promise because of the async keyword.
This promise will be resolved at the return all_permission; statement.
class SessionFilter{
async setPermissionFilterArray(data){
try {
//don't know were this sql var comes from
const sql_data = await knex.select(knex.raw(sql));
var all_permission = [];
sql_data.forEach((val) => {
...
});
return all_permission;
} catch (error) {
throw error;
}
}
}
Then you can get the result of your promise using async await keywords like this :
const set_permission_filter_array = await setPermissionFilterArray(data);
I have searched for the solution of the error specified in title.
MongoError: server instance pool was destroyed
I believe it is because misplacement of db.close(). But I am nesting dbo.collection and unable to get the exact solution of this error.
Firstly, I am fetching data (array of ids having status 0) from database and then I am concatenating (each app-id) them one by one with URL to get desired appUrl which will be used for crawling data one by one and then crawled data is meant to be stored into another collection of mongoDB. This process will repeat for each id in the array. But my code is having error of "server instance pool gets destroyed" before storing data into collection. I am doing misplacement of db.close() but I am unable to resolve this. Please help me resolving this error
Here is my code
///* global sitehead */
const request = require('request');
const cheerio = require('cheerio');
//const response = require('response');
const fs = require('fs');
const express = require('express');
const app = express();
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/";
var dateTime = require('node-datetime');
MongoClient.connect(url, {useNewUrlParser: true}, function (err, db) {
if (err) {
throw err;
} else {
var dbo = db.db("WebCrawler");
var app_id;
var appUrl;
let arr = [];
dbo.collection("Unique_Apps").find({"Post_Status": 0}, {projection: {_id: 0, App_Id: 1}}).toArray(function (err, result)
{
// console.log(result);
if (err) {
throw err;
// console.log(err);
} else {
for (var i = 0; i < result.length; i++)
{
arr[i] = result[i];
}
arr.forEach((el) => {
app_id = el.App_Id;
//console.log(app_id);
appUrl = 'https://play.google.com/store/apps/details?id=' + app_id;
console.log(appUrl);
request(appUrl, function (error, response, html) {
if (!error && response.statusCode === 200) {
//START Crawling ###########
const $ = cheerio.load(html); //cheerio
const appTitle = $('.AHFaub');
const iconUrl = $('.T75of.sHb2Xb').attr("src");
const developedBy = $('.T32cc.UAO9ie').children().eq(0);
const category = $('.T32cc.UAO9ie').children().eq(1);
//store in database collection: "Single_App_Data_Post"
var curr_obj = {App_Id: app_id, App_Name: appTitle.text(),
Icon_Url: iconUrl, Price: "Free", Developed_By: developedBy.text(),
Category: category.text()
};
dbo.collection("Single_App_Data_Post").insertOne(curr_obj, function (err, res) {
console.log("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
if (err) {
throw err;
// console.log(err);
} else {
console.log("inserted....");
} //main else
});
dbo.collection("Unique_Apps").updateOne({App_Id: app_id}, {$set: {Post_Status: 0}}, function (err, res) {
if (err)
throw err;
console.log("1 document updated");
//dbo.close();
});
} else
{
throw error;
}
});
});
}
db.close();
});
} //else
}); //mongoClient connect db
Output
The following is a good start about how to turn callback into promises. Try to use it, execute the code block, by block, understand it and then add your updateOne/insertOne requests into it.
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const express = require('express');
const app = express();
const MongoClient = require('mongodb').MongoClient;
const dateTime = require('node-datetime');
// Class used to handle the database basic interractions
class DB {
constructor() {
this.db = false;
this.url = "mongodb://localhost:27017/";
}
// Do connect to the database
connect() {
return new Promise((resolve, reject) => {
MongoClient.connect(this.url, {
useNewUrlParser: true,
}, (err, db) => {
if (err) {
console.log('error mongodb connect');
return reject(err);
}
this.db = db;
return resolve(db);
});
});
}
disconnect() {
db.close();
this.db = false;
}
getCollection(name) {
return this.db.db(name);
}
}
// Get the data from the database
function getAppsIds(dbObj) {
return new Promise((resolve, reject) => {
const dbo = dbObj.getCollection('WebCrawler');
dbo.collection('Unique_Apps').find({
'Post_Status': 0,
}, {
projection: {
_id: 0,
App_Id: 1,
}
}).toArray(function(err, result) {
if (err) {
return reject(err);
}
return resolve(result);
});
});
}
function requestPlayStore(idApp) {
return new Promise((resolve, reject) => {
const appUrl = `https://play.google.com/store/apps/details?id=${app_id}`;
request(appUrl, function(error, response, html) {
if (error || response.statusCode !== 200) {
return reject(error);
}
return resolve({
response,
html,
});
});
});
}
// Do treat one id app at a time
function treatOneIdApp(dbObj, idApp) {
return requestPlayStore(idApp)
.then(({
response,
html,
}) => {
// Perform your requests here updateOne and insertOne ...
});
}
const dbObj = new DB();
dbObj.connect()
.then(() => getAppsIds(dbObj))
.then(rets => Promise.all(rets.map(x => treatOneIdApp(dbObj, x.App_Id))))
.then(() => dbObj.disconnect())
.catch((err) => {
console.log(err);
});
When i create new variable and assign callback function, But data cannot return from callback function. Undefined is occurring at new variable.
const nedb = require('nedb');
const user = new nedb({ filename: './builds/database/users.db', autoload: true });
var s = user.find({}, function (err,docs) {
if(docs.length == 0) {
var data = false;
} else {
var data = docs;
}
return data;
});
console.log(s);
var s is undefined! ....
You are mixing up callback and Promise which are two different way to handle asynchronous calls.
I recommend you to use of Promises because they are simpler and the present and future of javascript.
Using async/await which is the next step after Promises
const user = {
find: () => ['jon', 'kalessi', 'jorah'],
};
async function getUsers() {
return (await user.find({})) || [];
}
async function myJob() {
const users = await getUsers();
console.log(users);
// Put your next code here
}
myJob();
Full promise :
const user = {
find: () => new Promise((resolve) => resolve(['jon', 'kalessi', 'jorah'])),
};
user.find({})
.then((docs = []) => {
console.log(docs);
// Put you next code here, you can use of the variable docs
})
.catch((err) => {
console.log(err);
});
Full callback :
const user = {
find: (_, callback) => callback(false, ['jon', 'kalessi', 'jorah']),
};
user.find({}, (err, docs = []) => {
if (err) {
console.log(err);
} else {
console.log(docs);
// Put you next code here, you can use of the variable docs
}
});
I think user.find returning the promise. you can do like this.
const nedb = require('nedb');
const user = new nedb({ filename: './builds/database/users.db', autoload: true });
var s = user.find({}, function (err,docs) {
if(docs.length == 0) {
var data = false;
} else {
var data = docs;
}
return data;
});
Promise.all(s)
.then(result => {
console.log(result);
});
Otherwise you can also use await Like this:
async function abc(){
const nedb = require('nedb');
const user = new nedb({ filename: './builds/database/users.db', autoload: true });
var s = await user.find({}, function (err,docs) {
if(docs.length == 0) {
var data = false;
} else {
var data = docs;
}
return data;
});
}
because await worked with async thats why i put it into async function.
I have a problem with async function and callbacks in Node js. I need to get my friends' all posts and display it. But if i do that without setTimeout(), it returns only some part of data. How can i solve this problem without putting setTimeout? Of course it's absurd to wait for 5-6 or 10 seconds to get all data. I also tried with Promises, but again response is incomplete. Please someone can help me?
//Sending request with axios to Controller
axios.post(packages.proxy+'users/getFriendsPosts',{id: user_id},config)
.then(res => {
// Code for displaying result
})
//User Controller
router.post("/getFriendsPosts", getFriendsPosts);
//Send request body to userService.js
function getFriendsPosts(req, res, next) {
userService.getFriendsPosts(req.body, function(posts, user){
res.json({posts,user});
})
.catch(err => next(err));
}
//userService.js
module.exports = {
getFriendsPosts,
};
async function getFriendsPosts(user,callback){
var arr = [];
var array = [];
MongoClient.connect(url, async function(errr, db) {
if (errr) throw errr;
var dbo = db.db("drone-x");
//Find user
dbo.collection("users").find({_id: ObjectId(user.id)}).toArray(async function(err, result) {
if (err) throw err;
result.forEach(async function(element, index) {
if(element.friends.length != 0){
element.friends.forEach(async function(elem) {
//Find user's friends
dbo.collection("users").find({_id: ObjectId(elem.id)}).toArray(async function(error, res) {
if (error) throw error;
//push user's friends to arr
arr.push(res);
res.forEach(async function(elements) {
//Find user's friends posts
dbo.collection("posts").find({userId: elements._id.toString()}).toArray(async function(errors, results) {
if (errors) throw errors;
//push user's friends posts to array
array.push(results);
//callback results through setTimeout
setTimeout(async function(){ await callback(array, arr); db.close(); }, 2000);
});
});
});
});
}
else
{
await callback("0");
}
});
});
});
}
If i don't use setTimeout function, it just returns 2-3 data, but with setTimeout, it returns all data. And if data will be raise, then i need to increase the setTimeout time. But of course it's not good idea. Someone can help me?
You should use try catch in this code
getFriendsPosts = async (user,callback) => {
const arr = [];
const array = [];
const db = await MongoClient.connect(url);
const dbo = db.db("drone-x");
const results = await dbo.collection("users").find({_id: ObjectId(user.id)})
const resultPromise = _.map(results, async element => {
const friends = _.get(element, 'friends', [])
if(friends.length != 0) {
const friendPromise = _.map(friends, async friend => {
const ress = await dbo.collection("users").find({_id: ObjectId(friend.id)})
arr.push(ress);
const resPromise = _.map(ress, async res => {
const posts = await dbo.collection("posts").find({userId: res._id.toString()})
const postPromise = _.map(posts, post => {
array.push(post);
})
await Promise.all(postPromise)
})
await Promise.all(resPromise)
})
await Promise.all(friendPromise)
}
})
await Promise.all(resultPromise)
return { arr , array }
}
I am not recommand this way it take too much time. You should use mongoose and use aggregation for long Query.
I've been trying non-stop to work on this query for my datatables front end.
this is my config.js
var config = {
host : 'localhost',
user : 'root',
password : '',
database : 'ef45db'
}
module.exports = config;
this is the function I want to work with async (wait for the query to return the table's columns name)
async function getColumnNames()
{
try{
aColumns = [];
await connection.query('SHOW COLUMNS FROM '+sTable,
function selectCb(err, results, fields){
console.log("entro a getColumnNames");
if(err){
console.log(err);
}
for(var i in results)
{
aColumns.push(results[i]['Field']);
}
connection.end();
});
}catch (e){
console.log(e);
}
}
and this is the controller code to execute that function:
var mysql = require('mysql2');
var config = require('.././database/config');
var connection = mysql.createConnection(config);
var sIndexColumn = '*';
var sTable = 'users';
var aColumns = [];
module.exports = {
getInfo : async function(req,res,next)
{
var request = req.query;
(async () => await getColumnNames());
console.log(aColumns);
}
I'm trying to get the column's name so I can work with datatable's filtering for backend, since node is async this query was getting executed, but the value was undefined (and still is), I've read hundreds of post regarding promises, bluebird and async methods and trying to make this work, the last I've read a lot thats the best and I choosed it because the code seems cleaner. Any ideas whats happening?
For getColumnNames(), you shouldn't use await because connection.query doesn't return promise. It is a callback function.
However, we can make getColumnNames to return promise.
function getColumnNames() {
const aColumns = [];
return new Promise((resolve, reject) => {
connection.query('SHOW COLUMNS FROM ' + sTable,
function selectCb(err, results, fields) {
console.log("entro a getColumnNames");
if (err) {
console.log(err);
reject(err); // if error happens, reject
}
for (var i in results) {
aColumns.push(results[i]['Field']);
}
connection.end();
resolve(aColumns); // resolve with our database columns
});
});
}
and for your controller we can use async await since getColumnNames returns promise as in
module.exports = {
getInfo: async function (req, res, next) {
var request = req.query;
const aColumns = await getColumnNames();
console.log(aColumns);
}
}
Let me know if it works for you.