Using durable functions in azure function app with mongodb - node.js

I have a mongoDB which contains the time where in my durable function should run (e.g. 8:00, 9:01, 10:20). Now I have my Orchestrator code below, nothing works inside the mongoClient.connect. Why???
const df = require("durable-functions");
const moment = require("moment");
const mongoClient = require("mongodb").MongoClient;
module.exports = df.orchestrator(function*(context) {
context.log("Getting time schedules in DB");
var timeSched = [];
var dbName = <dbName>;
var collectionName = <collectionName>;
var query = {id: "1"};
try{
mongoClient.connect(<mongoDB_connection_string>,{useNewUrlParser: true, authSource: dbName}, function (err, client) {
//Anything inside this does not log or work
if(err){
context.log(`Error occurred while connecting to DB ${err}`)
return context.done();
}else{
context.log('MongoClient connected to DB');
}
var collection = client.db(dbName).collection(collectionName);
collection.find(query).toArray(function(err, result) {
if (err) throw err;
for(let i = 0; i < result.length; i++){
timeSched.push(result[i].time); //e.g.8:00
}
client.close();
//This should log [8:00,9:01,10:01] but it does not log
context.log(timeSched);
context.done();
});
});
//This logs 0
context.log(timeSched.length);
for (let j = 0; j < timeSched.length; j++) {
const deadline = moment.utc(context.df.currentUtcDateTime).add(3, 'minutes');
yield context.df.createTimer(deadline.toDate());
yield context.df.callActivity("ActivityFunction",timeSched[j]);
}
context.done();
}catch(e){
context.log(`Error ${e}`);
context.done();
}
});

Try the code below to check if you can connect to DB first. Use console.log instead of context.log.
const df = require("durable-functions");
const mongoClient = require("mongodb").MongoClient;
module.exports = df.orchestrator(function*(context) {
var mongoClient = require("mongodb").MongoClient;
mongoClient.connect(
"mongodb://tonytest:78jst6Mh****.documents.azure.com:10255/?ssl=true",
function(err, client) {
if (err) {
console.log(`Error occurred while connecting to DB ${err}`);
return context.done();
} else {
console.log("MongoClient connected to DB");
}
client.close();
}
);
});
Try with console.log(timeSched); to output timeSched. Besides, when you execute console.log(timeSched.length);, timeSched hasn't been granted value. That's why you got 0;

Related

Error: server instance pool gets destroyed in nested db collection functions

I have searched for the solution of the error specified in title.
MongoError: server instance pool was destroyed
I believe it is because misplacement of db.close(). But I am nesting dbo.collection and unable to get the exact solution of this error.
Firstly, I am fetching data (array of ids having status 0) from database and then I am concatenating (each app-id) them one by one with URL to get desired appUrl which will be used for crawling data one by one and then crawled data is meant to be stored into another collection of mongoDB. This process will repeat for each id in the array. But my code is having error of "server instance pool gets destroyed" before storing data into collection. I am doing misplacement of db.close() but I am unable to resolve this. Please help me resolving this error
Here is my code
///* global sitehead */
const request = require('request');
const cheerio = require('cheerio');
//const response = require('response');
const fs = require('fs');
const express = require('express');
const app = express();
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/";
var dateTime = require('node-datetime');
MongoClient.connect(url, {useNewUrlParser: true}, function (err, db) {
if (err) {
throw err;
} else {
var dbo = db.db("WebCrawler");
var app_id;
var appUrl;
let arr = [];
dbo.collection("Unique_Apps").find({"Post_Status": 0}, {projection: {_id: 0, App_Id: 1}}).toArray(function (err, result)
{
// console.log(result);
if (err) {
throw err;
// console.log(err);
} else {
for (var i = 0; i < result.length; i++)
{
arr[i] = result[i];
}
arr.forEach((el) => {
app_id = el.App_Id;
//console.log(app_id);
appUrl = 'https://play.google.com/store/apps/details?id=' + app_id;
console.log(appUrl);
request(appUrl, function (error, response, html) {
if (!error && response.statusCode === 200) {
//START Crawling ###########
const $ = cheerio.load(html); //cheerio
const appTitle = $('.AHFaub');
const iconUrl = $('.T75of.sHb2Xb').attr("src");
const developedBy = $('.T32cc.UAO9ie').children().eq(0);
const category = $('.T32cc.UAO9ie').children().eq(1);
//store in database collection: "Single_App_Data_Post"
var curr_obj = {App_Id: app_id, App_Name: appTitle.text(),
Icon_Url: iconUrl, Price: "Free", Developed_By: developedBy.text(),
Category: category.text()
};
dbo.collection("Single_App_Data_Post").insertOne(curr_obj, function (err, res) {
console.log("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
if (err) {
throw err;
// console.log(err);
} else {
console.log("inserted....");
} //main else
});
dbo.collection("Unique_Apps").updateOne({App_Id: app_id}, {$set: {Post_Status: 0}}, function (err, res) {
if (err)
throw err;
console.log("1 document updated");
//dbo.close();
});
} else
{
throw error;
}
});
});
}
db.close();
});
} //else
}); //mongoClient connect db
Output
The following is a good start about how to turn callback into promises. Try to use it, execute the code block, by block, understand it and then add your updateOne/insertOne requests into it.
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const express = require('express');
const app = express();
const MongoClient = require('mongodb').MongoClient;
const dateTime = require('node-datetime');
// Class used to handle the database basic interractions
class DB {
constructor() {
this.db = false;
this.url = "mongodb://localhost:27017/";
}
// Do connect to the database
connect() {
return new Promise((resolve, reject) => {
MongoClient.connect(this.url, {
useNewUrlParser: true,
}, (err, db) => {
if (err) {
console.log('error mongodb connect');
return reject(err);
}
this.db = db;
return resolve(db);
});
});
}
disconnect() {
db.close();
this.db = false;
}
getCollection(name) {
return this.db.db(name);
}
}
// Get the data from the database
function getAppsIds(dbObj) {
return new Promise((resolve, reject) => {
const dbo = dbObj.getCollection('WebCrawler');
dbo.collection('Unique_Apps').find({
'Post_Status': 0,
}, {
projection: {
_id: 0,
App_Id: 1,
}
}).toArray(function(err, result) {
if (err) {
return reject(err);
}
return resolve(result);
});
});
}
function requestPlayStore(idApp) {
return new Promise((resolve, reject) => {
const appUrl = `https://play.google.com/store/apps/details?id=${app_id}`;
request(appUrl, function(error, response, html) {
if (error || response.statusCode !== 200) {
return reject(error);
}
return resolve({
response,
html,
});
});
});
}
// Do treat one id app at a time
function treatOneIdApp(dbObj, idApp) {
return requestPlayStore(idApp)
.then(({
response,
html,
}) => {
// Perform your requests here updateOne and insertOne ...
});
}
const dbObj = new DB();
dbObj.connect()
.then(() => getAppsIds(dbObj))
.then(rets => Promise.all(rets.map(x => treatOneIdApp(dbObj, x.App_Id))))
.then(() => dbObj.disconnect())
.catch((err) => {
console.log(err);
});

null values when fetching data from database in nodejs

am new to nodeJS and am having some problem understanding how async and await works. Here is my code am using to fetch data from my database:
const sql = require('mssql');
const config = require('./config');
var Student = require('./student');
module.exports.getAllStudents = async function getAllStudents(){
var studentArray = new Array();
console.log('dbconnection');
var conn = new sql.ConnectionPool(config.dbConfig[0]);
conn.connect()
.then(()=>{
var req = new sql.Request(conn);
req.query('SELECT * from student').then((recordset)=> {
var dataset = recordset.recordset;
for(var index = 0; index<dataset.length; index++){
var student = new Student(dataset[index].student_id , dataset[index].first_name, dataset[index].last_name, dataset[index].student_address , dataset[index].age);
studentArray.push(student);
}
conn.close();
}).catch( (error) =>{
console.log("An error has occured while executing your query ");
console.log(error);
});
})
.catch( (error)=> {
console.log("An error has occured while trying to connect to the database : ");
console.log(error);
});
return studentArray;
}
and here is the endpoint where i made a call to fetch all the students:
app.get('/getAllStudents' , function(req , res){
var studentArr = new Array();
studentArr = query.getAllStudents();
console.log('====================================');
//console.log(studentArr); //this prints undefined
//console.log('length:======' + studentArr.length);
//var json = JSON.stringify(studentArr);
res.status(200);
res.send('got all students');
})
am trying to understand how to counter this error, what needs to be done to wait for my query to get the results then prints the student array. Could anybody explain to me what needs to be done and why it is not working ?
Because it's asynchronous you should return a promise then resolve() the results
const sql = require('mssql');
const config = require('./config');
var Student = require('./student');
module.exports.getAllStudents = function getAllStudents() {
return new Promise((resolve, reject) => {
var studentArray = new Array();
console.log('dbconnection');
var conn = new sql.ConnectionPool(config.dbConfig[0]);
conn.connect()
.then(()=>{
var req = new sql.Request(conn);
req.query('SELECT * from student').then((recordset)=> {
var dataset = recordset.recordset;
for(var index = 0; index<dataset.length; index++){
var student = new Student(dataset[index].student_id , dataset[index].first_name, dataset[index].last_name, dataset[index].student_address , dataset[index].age);
studentArray.push(student);
}
resolve(studentArray);
conn.close();
}).catch( (error) =>{
console.log("An error has occured while executing your query ");
console.log(error);
reject(error);
});
})
.catch( (error)=> {
console.log("An error has occured while trying to connect to the database : ");
console.log(error);
reject(error);
});
});
}
Then add async to your function and await to query.getAllStudents()
app.get('/getAllStudents' , async function(req , res){
var studentArr = new Array();
studentArr = await query.getAllStudents();

nodejs + Mongodb: Inserting into two collections in sequence repeats last value in second

I am using the following to insert into MongoDB.
var tagData = JSON.parse(data);
var allTags = tagData.tags;
for (var j = 0; j < allTags.length; j++) {
var p = allTags[j].tagId.toString();
for (var k = 0; k < loggerParams.length; k++) {
var q = Object.keys(loggerParams[k]).toString();
if (p === q) {
// Prepare raw data tag
var tagRawDoc = {};
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
database.addDocument('tagraw', tagRawDoc, function (err) {
if (err) {
log.info(util.format('Error adding document to tagrawdatas. %s', err.message));
throw err;
} else {
// Prepare history tag
var historyTagDoc = {};
historyTagDoc.tagNameAlias = tagRawDoc.tagNameAlias;
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
database.addDocument('taghistory', historyTagDoc, function (err) {
if (err) {
log.info(util.format('Error adding document to tagrawdatas. %s', err.message));
throw err;
}
});
}
});
// Match found; exit loop
break;
}
}
}
The loggerParms is a simple JSON document read from file else-where. It allows for look-up in this code to build the document to be inserted. There will be 12 values in the allTags array. These 12 values are inserted successfully into the tagraw collection. However, in taghistory collection, the values from the last (or most recent) entry made into tagraw collection is repeated 12 times. Why does this happen?
The database.addDocument is shown below. It is a part of this article I am trying to replicate.
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var logger = require('../../util/logger');
var util = require('util');
function DB() {
this.db = "empty";
this.log = logger().getLogger('mongoMange-DB');
}
DB.prototype.connect = function(uri, callback) {
this.log.info(util.format('About to connect to DB'));
if (this.db != "empty") {
callback();
this.log.info('Already connected to database.');
} else {
var _this = this;
MongoClient.connect(uri, function(err, database) {
if (err) {
_this.log.info(util.format('Error connecting to DB: %s', err.message));
callback(err);
} else {
_this.db = database;
_this.log.info(util.format('Connected to database.'));
callback();
}
})
}
}
DB.prototype.close = function(callback) {
log.info('Closing database');
this.db.close();
this.log.info('Closed database');
callback();
}
DB.prototype.addDocument = function(coll, doc, callback) {
var collection = this.db.collection(coll);
var _this = this;
collection.insertOne(doc, function(err, result) {
if (err) {
_this.log.info(util.format('Error inserting document: %s', err.message));
callback(err.message);
} else {
_this.log.info(util.format('Inserted document into %s collection.', coll));
callback();
}
});
};
module.exports = DB;
That's because you are mixing a/multiple synchronous for and asynchronous code with database.addDocument which cause issues with function scope in nodejs.
A simple example of this kind of thing:
for(var i = 0; i < 10; i++){
setTimeout(() => console.log(i), 0);
}
You should use a package like async to handle flow control when iterating arrays/object asynchronously.
Simple example of your code refactored to use async:
var async = require('async');
var tagData = JSON.parse(data);
var allTags = tagData.tags;
async.each(allTags, function(tag, done){
var p = tag.tagId.toString();
var loggerParam = loggerParams.find(function(loggerParam){
var q = Object.keys(loggerParam).toString();
return p === q;
});
var tagRawDoc = {};
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
return database.addDocument('tagraw', tagRawDoc, function (err){
if (err) return done(err);
// Prepare history tag
var historyTagDoc = {};
historyTagDoc.tagNameAlias = tagRawDoc.tagNameAlias;
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
return database.addDocument('taghistory', historyTagDoc, done);
});
}, (err) => {
if(err) throw err;
console.log('All done');
});

NodeJS With Mongo: Callback Is Not A Function error

Im currently working on a NodeJS/Mongo project where I need to pull all the documents from a collection. I currently have the following code written:
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server,
ReplSetServers = require('mongodb').ReplSetServers,
ObjectID = require('mongodb').ObjectID,
Binary = require('mongodb').Binary,
GridStore = require('mongodb').GridStore,
Grid = require('mongodb').Grid,
Code = require('mongodb').Code,
assert = require('assert');
var server = new Server('[server]', 27017);
var authDB = new Db('admin', server);
var DB1250 = new Db('1250', server);
var findDocuments = function (callback) {
authDB.authenticate("Username", "Password");
DB1250.open(function (error, db) {
if (error) {
console.log(error);
}
else {
console.log("successfully accessed: ", db);
callback;
var cursor = db.collection('Patients').find();
console.log('cursor ', cursor);
cursor.forEach(function (error, document) {
if (error) {
console.log('Document does not exist. Error: ', error);
}
else {
console.log('Document: ', document);
}
});
}
});
};
findDocuments(function (data) {
});
I am able to authenticate/connect to the server, connect to the DB, and connect to the collection. When I enter the forEach loop to iterate through all the documents, I keep getting the error "Callback is not a function". Can you guys see what I am doing wrong?
I believe the cursor you have there has not resolved into an array, so forEach is not a valid method. You might be looking for eachAsync, which will wait for the query to return before iterating.
Alternatively, you can wait for the query promise to resolve, ie:
cursor.then(docs => docs.forEach(callback));
which I personally find a little clearer.
Heres the solution I came up with after using Mongoose:
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server,
ReplSetServers = require('mongodb').ReplSetServers,
ObjectID = require('mongodb').ObjectID,
Binary = require('mongodb').Binary,
GridStore = require('mongodb').GridStore,
Grid = require('mongodb').Grid,
Code = require('mongodb').Code,
Mongoose = require('mongoose');
assert = require('assert');
var findDocuments = function (callback) {
var options = { server: { socketOptions: { keepAlive: 1000 } } };
var connectionString = 'mongodb://username:password#server:27017/admin';
// Connected handler
Mongoose.connect(connectionString, function (err) {
var db = Mongoose.connection.useDb('db');
var collection = db.collection("collection");
collection.find().stream()
.on('data', function (document) {
console.log(document);
})
.on('error', function (err) {
// handle error
console.log(err);
})
.on('end', function () {
// final callback
});
});
// Error handler
Mongoose.connection.on('error', function (err) {
console.log(err);
});
// Reconnect when closed
Mongoose.connection.on('disconnected', function () {
self.connectToDatabase();
});
};
findDocuments(function () {
db.close();
});
cursor.forEach() is an asynchronous operation but returns nothing. So you shouldn't do db.close() or likewise below the line somethere synchronously.
Besides; I am not sure which version Node driver you are refering to but in the recent ones like v3+ cursor.forEach() doesn't take an error first type callback like the one that you use in your code. It will take an "iterator" and an "end" callback such as;
cursor.forEach(doc => console.log(doc),
err => err ? console.log(err)
: db.close())
So the above code will just work fine, iterating and processing the documents one by one as they appear without waiting up until all of them have been retrieved to the memory.

How can I execute db.copyDatabase through NodeJS's MongoDB native driver?

I do have a shell script that invokes
mongo --eval "db.copyDatabase('somedatabase', 'somedatabase_duplicate', 'sourcehost')"
to copy a database.
Currently I am stuck with doing the same from within a Node.JS application. Calling
mongoCommand = `db.copyDatabase("somedatabase", "somedatabase_duplicate", "localhost")`;
db.command(mongoCommand, function(commandErr, data) {
if(!commandErr) {
log.info(data);
} else {
log.error(commandErr.errmsg);
}
});
Always resulsts in a "no such command" error message.
Edit for clarification: Using db.admin().command() results in the same problem and using the command suggested in enter link description here, too.
What's the correct way to call this command or, alternatively, to clone a database from Node.JS?
Well, you are trying to copy database which is administration operation so have to do with admin account. Again, to copy database command is copydb.
try running this command in shell, db.copyDatabase and you'll see source of command.
try:
var assert = require('assert');
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost:27017/test';
MongoClient.connect(url, function(err, db) {
if (err) {
console.log(err);
}
else {
var mongoCommand = { copydb: 1, fromhost: "localhost", fromdb: "test", todb: "test_dup" };
var admin = db.admin();
admin.command(mongoCommand, function(commandErr, data) {
if (!commandErr) {
console.log(data);
} else {
console.log(commandErr.errmsg);
}
db.close();
});
}
});
//core modules
const assert = require('assert')
const MongoClient = require('mongodb').MongoClient;
const moment = require('moment');
const mongo = require('mongodb')
//custom modules
let { ip, port, database } = require('./dbUpgradeConfig')
const url = `mongodb://${ip}:${port}`
let todayDate = moment().format('DD/MM/YYYY HH:mm')
console.log(todayDate)
const myDate = new Date()
console.log(myDate)
var d = Date(Date.now());
// Converting the number of millisecond in date string
a = d.toString()
// Options for mongoDB
const mongoOptions = { useNewUrlParser: true }
let db
//TODO: handle reconnect
let connect = () => {
return new Promise((resolve, reject) => {
if (db) resolve()
else {
mongo.connect(url, mongoOptions, (err, client) => {
if (err) reject(err)
else {
db = client.db(database)
resolve()
}
})
}
})
}
/**
* #description create duplicate database from current database in mongodb
*/
let CloneDb = () => {
return new Promise((resolve, reject) => {
connect()
.then(() => {
console.log(db)
let mongoCommand = { copydb: 1, fromhost: "localhost", fromdb: "db_name", todb: "db_name_duplicate" }
let adminDB = db.admin()
adminDB.command(mongoCommand, function (commandErr, data) {
if (!commandErr) {
console.log(data)
} else {
console.log(commandErr.errmsg)
}
});
})
})
}
CloneDb().then(data => {
// debugger;
console.log("The clone db", data)
})

Resources