null values when fetching data from database in nodejs - node.js

am new to nodeJS and am having some problem understanding how async and await works. Here is my code am using to fetch data from my database:
const sql = require('mssql');
const config = require('./config');
var Student = require('./student');
module.exports.getAllStudents = async function getAllStudents(){
var studentArray = new Array();
console.log('dbconnection');
var conn = new sql.ConnectionPool(config.dbConfig[0]);
conn.connect()
.then(()=>{
var req = new sql.Request(conn);
req.query('SELECT * from student').then((recordset)=> {
var dataset = recordset.recordset;
for(var index = 0; index<dataset.length; index++){
var student = new Student(dataset[index].student_id , dataset[index].first_name, dataset[index].last_name, dataset[index].student_address , dataset[index].age);
studentArray.push(student);
}
conn.close();
}).catch( (error) =>{
console.log("An error has occured while executing your query ");
console.log(error);
});
})
.catch( (error)=> {
console.log("An error has occured while trying to connect to the database : ");
console.log(error);
});
return studentArray;
}
and here is the endpoint where i made a call to fetch all the students:
app.get('/getAllStudents' , function(req , res){
var studentArr = new Array();
studentArr = query.getAllStudents();
console.log('====================================');
//console.log(studentArr); //this prints undefined
//console.log('length:======' + studentArr.length);
//var json = JSON.stringify(studentArr);
res.status(200);
res.send('got all students');
})
am trying to understand how to counter this error, what needs to be done to wait for my query to get the results then prints the student array. Could anybody explain to me what needs to be done and why it is not working ?

Because it's asynchronous you should return a promise then resolve() the results
const sql = require('mssql');
const config = require('./config');
var Student = require('./student');
module.exports.getAllStudents = function getAllStudents() {
return new Promise((resolve, reject) => {
var studentArray = new Array();
console.log('dbconnection');
var conn = new sql.ConnectionPool(config.dbConfig[0]);
conn.connect()
.then(()=>{
var req = new sql.Request(conn);
req.query('SELECT * from student').then((recordset)=> {
var dataset = recordset.recordset;
for(var index = 0; index<dataset.length; index++){
var student = new Student(dataset[index].student_id , dataset[index].first_name, dataset[index].last_name, dataset[index].student_address , dataset[index].age);
studentArray.push(student);
}
resolve(studentArray);
conn.close();
}).catch( (error) =>{
console.log("An error has occured while executing your query ");
console.log(error);
reject(error);
});
})
.catch( (error)=> {
console.log("An error has occured while trying to connect to the database : ");
console.log(error);
reject(error);
});
});
}
Then add async to your function and await to query.getAllStudents()
app.get('/getAllStudents' , async function(req , res){
var studentArr = new Array();
studentArr = await query.getAllStudents();

Related

function returning nothing in mongodb query

I'm not capturing the connect function to return in the getAllowedEmails function, when I do console.log in allowedEmails, it returns the emails correctly, but when I assign to the variable emails, it is returning empty. I think it's an async await problem but can't figured out.
static async getAllowedEmails() {
var MongoClient = require("mongodb").MongoClient;
//db url
let emails = [];
await MongoClient.connect(url, async function (err, client) {
const db = client.db("data-admin");
var myPromise = () => {
return new Promise((resolve, reject) => {
db.collection("users")
.find({})
.toArray(function (err, data) {
err ? reject(err) : resolve(data);
});
});
};
var result = await myPromise();
client.close();
let allowedEmails = [];
result.map((email) => allowedEmails.push(email.email));
console.log(allowedEmails)
emails = allowedEmails;
});
console.log(emails)
return emails;
}
Your code has couple of issues, I have fixed few and enhanced it, given below is the basic code, try to test it and if everything works then enhance it as needed:
const MongoClient = require("mongodb").MongoClient;
async function getAllowedEmails() {
let client;
try {
const allowedEmails = [];
client = await MongoClient.connect(url);
const db = client.db("data-admin");
const result = await db.collection("users").find({}).toArray();
result.map((email) => allowedEmails.push(email.email));
console.log(allowedEmails)
client.close();
return allowedEmails;
} catch (error) {
(client) && client.close();
console.log(error)
throw error;
}
}

Using durable functions in azure function app with mongodb

I have a mongoDB which contains the time where in my durable function should run (e.g. 8:00, 9:01, 10:20). Now I have my Orchestrator code below, nothing works inside the mongoClient.connect. Why???
const df = require("durable-functions");
const moment = require("moment");
const mongoClient = require("mongodb").MongoClient;
module.exports = df.orchestrator(function*(context) {
context.log("Getting time schedules in DB");
var timeSched = [];
var dbName = <dbName>;
var collectionName = <collectionName>;
var query = {id: "1"};
try{
mongoClient.connect(<mongoDB_connection_string>,{useNewUrlParser: true, authSource: dbName}, function (err, client) {
//Anything inside this does not log or work
if(err){
context.log(`Error occurred while connecting to DB ${err}`)
return context.done();
}else{
context.log('MongoClient connected to DB');
}
var collection = client.db(dbName).collection(collectionName);
collection.find(query).toArray(function(err, result) {
if (err) throw err;
for(let i = 0; i < result.length; i++){
timeSched.push(result[i].time); //e.g.8:00
}
client.close();
//This should log [8:00,9:01,10:01] but it does not log
context.log(timeSched);
context.done();
});
});
//This logs 0
context.log(timeSched.length);
for (let j = 0; j < timeSched.length; j++) {
const deadline = moment.utc(context.df.currentUtcDateTime).add(3, 'minutes');
yield context.df.createTimer(deadline.toDate());
yield context.df.callActivity("ActivityFunction",timeSched[j]);
}
context.done();
}catch(e){
context.log(`Error ${e}`);
context.done();
}
});
Try the code below to check if you can connect to DB first. Use console.log instead of context.log.
const df = require("durable-functions");
const mongoClient = require("mongodb").MongoClient;
module.exports = df.orchestrator(function*(context) {
var mongoClient = require("mongodb").MongoClient;
mongoClient.connect(
"mongodb://tonytest:78jst6Mh****.documents.azure.com:10255/?ssl=true",
function(err, client) {
if (err) {
console.log(`Error occurred while connecting to DB ${err}`);
return context.done();
} else {
console.log("MongoClient connected to DB");
}
client.close();
}
);
});
Try with console.log(timeSched); to output timeSched. Besides, when you execute console.log(timeSched.length);, timeSched hasn't been granted value. That's why you got 0;

Error: server instance pool gets destroyed in nested db collection functions

I have searched for the solution of the error specified in title.
MongoError: server instance pool was destroyed
I believe it is because misplacement of db.close(). But I am nesting dbo.collection and unable to get the exact solution of this error.
Firstly, I am fetching data (array of ids having status 0) from database and then I am concatenating (each app-id) them one by one with URL to get desired appUrl which will be used for crawling data one by one and then crawled data is meant to be stored into another collection of mongoDB. This process will repeat for each id in the array. But my code is having error of "server instance pool gets destroyed" before storing data into collection. I am doing misplacement of db.close() but I am unable to resolve this. Please help me resolving this error
Here is my code
///* global sitehead */
const request = require('request');
const cheerio = require('cheerio');
//const response = require('response');
const fs = require('fs');
const express = require('express');
const app = express();
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/";
var dateTime = require('node-datetime');
MongoClient.connect(url, {useNewUrlParser: true}, function (err, db) {
if (err) {
throw err;
} else {
var dbo = db.db("WebCrawler");
var app_id;
var appUrl;
let arr = [];
dbo.collection("Unique_Apps").find({"Post_Status": 0}, {projection: {_id: 0, App_Id: 1}}).toArray(function (err, result)
{
// console.log(result);
if (err) {
throw err;
// console.log(err);
} else {
for (var i = 0; i < result.length; i++)
{
arr[i] = result[i];
}
arr.forEach((el) => {
app_id = el.App_Id;
//console.log(app_id);
appUrl = 'https://play.google.com/store/apps/details?id=' + app_id;
console.log(appUrl);
request(appUrl, function (error, response, html) {
if (!error && response.statusCode === 200) {
//START Crawling ###########
const $ = cheerio.load(html); //cheerio
const appTitle = $('.AHFaub');
const iconUrl = $('.T75of.sHb2Xb').attr("src");
const developedBy = $('.T32cc.UAO9ie').children().eq(0);
const category = $('.T32cc.UAO9ie').children().eq(1);
//store in database collection: "Single_App_Data_Post"
var curr_obj = {App_Id: app_id, App_Name: appTitle.text(),
Icon_Url: iconUrl, Price: "Free", Developed_By: developedBy.text(),
Category: category.text()
};
dbo.collection("Single_App_Data_Post").insertOne(curr_obj, function (err, res) {
console.log("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
if (err) {
throw err;
// console.log(err);
} else {
console.log("inserted....");
} //main else
});
dbo.collection("Unique_Apps").updateOne({App_Id: app_id}, {$set: {Post_Status: 0}}, function (err, res) {
if (err)
throw err;
console.log("1 document updated");
//dbo.close();
});
} else
{
throw error;
}
});
});
}
db.close();
});
} //else
}); //mongoClient connect db
Output
The following is a good start about how to turn callback into promises. Try to use it, execute the code block, by block, understand it and then add your updateOne/insertOne requests into it.
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const express = require('express');
const app = express();
const MongoClient = require('mongodb').MongoClient;
const dateTime = require('node-datetime');
// Class used to handle the database basic interractions
class DB {
constructor() {
this.db = false;
this.url = "mongodb://localhost:27017/";
}
// Do connect to the database
connect() {
return new Promise((resolve, reject) => {
MongoClient.connect(this.url, {
useNewUrlParser: true,
}, (err, db) => {
if (err) {
console.log('error mongodb connect');
return reject(err);
}
this.db = db;
return resolve(db);
});
});
}
disconnect() {
db.close();
this.db = false;
}
getCollection(name) {
return this.db.db(name);
}
}
// Get the data from the database
function getAppsIds(dbObj) {
return new Promise((resolve, reject) => {
const dbo = dbObj.getCollection('WebCrawler');
dbo.collection('Unique_Apps').find({
'Post_Status': 0,
}, {
projection: {
_id: 0,
App_Id: 1,
}
}).toArray(function(err, result) {
if (err) {
return reject(err);
}
return resolve(result);
});
});
}
function requestPlayStore(idApp) {
return new Promise((resolve, reject) => {
const appUrl = `https://play.google.com/store/apps/details?id=${app_id}`;
request(appUrl, function(error, response, html) {
if (error || response.statusCode !== 200) {
return reject(error);
}
return resolve({
response,
html,
});
});
});
}
// Do treat one id app at a time
function treatOneIdApp(dbObj, idApp) {
return requestPlayStore(idApp)
.then(({
response,
html,
}) => {
// Perform your requests here updateOne and insertOne ...
});
}
const dbObj = new DB();
dbObj.connect()
.then(() => getAppsIds(dbObj))
.then(rets => Promise.all(rets.map(x => treatOneIdApp(dbObj, x.App_Id))))
.then(() => dbObj.disconnect())
.catch((err) => {
console.log(err);
});

Node js Printing info from JSON file using a function from another file V2.0

This is a continuation from another question I asked earlier Node.js Printing info from JSON file using a function from another JS file
In my previous question I had problems in calling a function from my data-service.js file that printed all the items in my JSON array, and had it resolved, but now I'm struggling in doing something similar in printing only the employees from my JSON array that I specify through the url. For example http://localhost:8080/employeesstatus=5 would print only the employee with a status of 5 however nothing is getting printed
SERVER.JS
var HTTP_PORT = process.env.PORT || 8080;
var express = require('express');
var data = require('./data-service');
var fs = require('fs');
var app = express();
var object = require('./data-service');
console.log("Express http server listening on 8080");
//app.get('/employees', function(req,res){
// return object.getAllEmployees()
// .then((response) => res.send(response))
//}); //QUESION FROM PREVIOUS POST WHICH WAS RESOLVED
app.get('/employees:?status=:value', function(req,res){
return object.getEmployeesByStatus(req.params.value)
.then((response) => res.send(response));
});
DATA SERVICE.JS
var employees = [];
var departments = [];
var error = 0;
var fs = require("fs");
function initialize(){
employees = fs.readFileSync("./data/employees.json", 'utf8', function(err, data){
if(err){
error = 1;
}
employees = JSON.parse(data);
});
departments = fs.readFileSync("./data/department.json", 'utf8',
function(err, data){
if(err){
error = 1;
}
departments = JSON.parse(data);
});
}
function check() {
return new Promise(function(resolve,reject){
if (error === 0){
resolve("Success");
}
else if(error === 1){
reject("unable to read file");
}
})
};
var getAllEmployees = function(){
return check().then(function(x){
console.log(x);
console.log(employees);
return employees;
}).catch(function(x){
console.log("No results returned");
});
}
var getEmployeesByStatus = function (status){
return check().then(function(x){
var employees2 = JSON.parse(employees);
for (var i = 0; i<employees2.length; i++){
if (employees2[i].status == status){
return console.log(employees2[i]);
}
}
}).catch(function(){
console.log("no results returned");
})
}
module.exports.getAllEmployees = getAllEmployees;
module.exports.getEmployeesByStatus = getEmployeesByStatus;
The 2 functions in question
app.get('/employees:?status=:value', function(req,res){
return object.getEmployeesByStatus(req.params.value)
.then((response) => res.send(response));
});
var getEmployeesByStatus = function (status){
return check().then(function(x){
var employees2 = JSON.parse(employees);
for (var i = 0; i<employees2.length; i++){
if (employees2[i].status == status){
return employees2[i];
}
}
}).catch(function(){
console.log("no results returned");
})
}
1) You should replace /employees route with the following
app.get('/employees/status=:value', function(req,res){
return object.getEmployeesByStatus(req.params.value)
.then((response) => res.send(response));
});
You are able to access using http://localhost:8080/employees/status=5
2) Return employees2[i] instead of console.log(employees2[i]).

NodeJS With Mongo: Callback Is Not A Function error

Im currently working on a NodeJS/Mongo project where I need to pull all the documents from a collection. I currently have the following code written:
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server,
ReplSetServers = require('mongodb').ReplSetServers,
ObjectID = require('mongodb').ObjectID,
Binary = require('mongodb').Binary,
GridStore = require('mongodb').GridStore,
Grid = require('mongodb').Grid,
Code = require('mongodb').Code,
assert = require('assert');
var server = new Server('[server]', 27017);
var authDB = new Db('admin', server);
var DB1250 = new Db('1250', server);
var findDocuments = function (callback) {
authDB.authenticate("Username", "Password");
DB1250.open(function (error, db) {
if (error) {
console.log(error);
}
else {
console.log("successfully accessed: ", db);
callback;
var cursor = db.collection('Patients').find();
console.log('cursor ', cursor);
cursor.forEach(function (error, document) {
if (error) {
console.log('Document does not exist. Error: ', error);
}
else {
console.log('Document: ', document);
}
});
}
});
};
findDocuments(function (data) {
});
I am able to authenticate/connect to the server, connect to the DB, and connect to the collection. When I enter the forEach loop to iterate through all the documents, I keep getting the error "Callback is not a function". Can you guys see what I am doing wrong?
I believe the cursor you have there has not resolved into an array, so forEach is not a valid method. You might be looking for eachAsync, which will wait for the query to return before iterating.
Alternatively, you can wait for the query promise to resolve, ie:
cursor.then(docs => docs.forEach(callback));
which I personally find a little clearer.
Heres the solution I came up with after using Mongoose:
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server,
ReplSetServers = require('mongodb').ReplSetServers,
ObjectID = require('mongodb').ObjectID,
Binary = require('mongodb').Binary,
GridStore = require('mongodb').GridStore,
Grid = require('mongodb').Grid,
Code = require('mongodb').Code,
Mongoose = require('mongoose');
assert = require('assert');
var findDocuments = function (callback) {
var options = { server: { socketOptions: { keepAlive: 1000 } } };
var connectionString = 'mongodb://username:password#server:27017/admin';
// Connected handler
Mongoose.connect(connectionString, function (err) {
var db = Mongoose.connection.useDb('db');
var collection = db.collection("collection");
collection.find().stream()
.on('data', function (document) {
console.log(document);
})
.on('error', function (err) {
// handle error
console.log(err);
})
.on('end', function () {
// final callback
});
});
// Error handler
Mongoose.connection.on('error', function (err) {
console.log(err);
});
// Reconnect when closed
Mongoose.connection.on('disconnected', function () {
self.connectToDatabase();
});
};
findDocuments(function () {
db.close();
});
cursor.forEach() is an asynchronous operation but returns nothing. So you shouldn't do db.close() or likewise below the line somethere synchronously.
Besides; I am not sure which version Node driver you are refering to but in the recent ones like v3+ cursor.forEach() doesn't take an error first type callback like the one that you use in your code. It will take an "iterator" and an "end" callback such as;
cursor.forEach(doc => console.log(doc),
err => err ? console.log(err)
: db.close())
So the above code will just work fine, iterating and processing the documents one by one as they appear without waiting up until all of them have been retrieved to the memory.

Resources