Node JS - function not executed - node.js

I have a module which i wrote myself. It contains 3 functions which i call as chained promises. The last one does not execute and i cannot figure out why.
The module geodata.js
require('use-strict');
var Promise = require('promise');
var NodeGeocoder = require('node-geocoder');
var mongoose = require('mongoose');
var User = require('../models/user');
var options = {
provider: 'google',
httpAdapter: 'https',
apiKey: 'AIzaSyA4v81WbNOMeRL7p911Mxr6PBZnidX0cIM',
formatter: null
};
module.exports = {
//*******************************************************
// Find user and return his adress (street + town)
//
//*******************************************************
findUser: function(username) {
return new Promise(function(resolve, reject) {
User.findOne({
'username': username
}, function(err, doc) {
if (!err) {
resolve(doc);
} else {
reject(err);
}
});
});
},
//*******************************************************
// Fetch geodata (latitude + longitude) to users record in
// the user collection.
//*******************************************************
fetchGeoData: function(userObj) {
return new Promise(function(resolve, reject) {
var geocoder = NodeGeocoder(options);
var adress = userObj.street + ' ' + userObj.town;
geocoder.geocode(adress, function(err, res) {
if (!err) {
var res2 = {
'username': userObj.username
}
console.log(res);
resolve([res, res2]);
} else {
reject(err);
}
});
});
},
//*******************************************************
// Fetch geodata (latitude + longitude) to users record in
// the user collection.
//*******************************************************
addGeoData: function(message) {
return new Promise(function(resolve, reject) {
User.findOne({
'username': message.username
}, function(err, doc) {
console.log(doc);
if (err) {
console.log(err);
reject(err);
}
if (!doc) {
console.log('User not found.');
reject('User not found.');
}
doc.longitude = String(message.longitude);
doc.latitude = String(message.latitude);
doc.save();
resolve(doc);
});
});
},
fixJSON: function(inpJSON) {
var strung = JSON.stringify(inpJSON);
obj = strung.substring(1, strung.length - 1);
return (JSON.parse(obj));
}
};
.. and here is the calling script:
require('use-strict');
var mongoose = require('mongoose');
var dbConfig = require('./db');
var geodata = require('./lib/geodata.js');
// Connect to DB
mongoose.connect(dbConfig.url, {
auth: {
authdb: "admin"
}
});
mongoose.set('debug', true);
geodata.findUser('jimmy').then(function(result) {
console.log('findUser() done');
return geodata.fetchGeoData(result);
}).then(function(result) {
console.log('fetchGeoData() done');
var res1Fixed = geodata.fixJSON(result[0]);
var params = {
username: result[1].username,
longitude: res1Fixed.longitude,
latitude: res1Fixed.latitude
}
console.log(params);
return geodata.addGeoData(params);
}).then(function(result) {
console.log('addGeoData() done');
});
mongoose.connection.close();
the execution stops after this line:
console.log('fetchGeoData() done');
and i can't figure out why?
Regards
Jimmy

If you are referring to mongoose.connection.close(); that is actually being run before your console.log('addGeoData() done');
Notice that mongoose.connection.close() is synchronous whereas the mongoose.findUser is asynchronous.

Related

Error: server instance pool gets destroyed in nested db collection functions

I have searched for the solution of the error specified in title.
MongoError: server instance pool was destroyed
I believe it is because misplacement of db.close(). But I am nesting dbo.collection and unable to get the exact solution of this error.
Firstly, I am fetching data (array of ids having status 0) from database and then I am concatenating (each app-id) them one by one with URL to get desired appUrl which will be used for crawling data one by one and then crawled data is meant to be stored into another collection of mongoDB. This process will repeat for each id in the array. But my code is having error of "server instance pool gets destroyed" before storing data into collection. I am doing misplacement of db.close() but I am unable to resolve this. Please help me resolving this error
Here is my code
///* global sitehead */
const request = require('request');
const cheerio = require('cheerio');
//const response = require('response');
const fs = require('fs');
const express = require('express');
const app = express();
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/";
var dateTime = require('node-datetime');
MongoClient.connect(url, {useNewUrlParser: true}, function (err, db) {
if (err) {
throw err;
} else {
var dbo = db.db("WebCrawler");
var app_id;
var appUrl;
let arr = [];
dbo.collection("Unique_Apps").find({"Post_Status": 0}, {projection: {_id: 0, App_Id: 1}}).toArray(function (err, result)
{
// console.log(result);
if (err) {
throw err;
// console.log(err);
} else {
for (var i = 0; i < result.length; i++)
{
arr[i] = result[i];
}
arr.forEach((el) => {
app_id = el.App_Id;
//console.log(app_id);
appUrl = 'https://play.google.com/store/apps/details?id=' + app_id;
console.log(appUrl);
request(appUrl, function (error, response, html) {
if (!error && response.statusCode === 200) {
//START Crawling ###########
const $ = cheerio.load(html); //cheerio
const appTitle = $('.AHFaub');
const iconUrl = $('.T75of.sHb2Xb').attr("src");
const developedBy = $('.T32cc.UAO9ie').children().eq(0);
const category = $('.T32cc.UAO9ie').children().eq(1);
//store in database collection: "Single_App_Data_Post"
var curr_obj = {App_Id: app_id, App_Name: appTitle.text(),
Icon_Url: iconUrl, Price: "Free", Developed_By: developedBy.text(),
Category: category.text()
};
dbo.collection("Single_App_Data_Post").insertOne(curr_obj, function (err, res) {
console.log("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
if (err) {
throw err;
// console.log(err);
} else {
console.log("inserted....");
} //main else
});
dbo.collection("Unique_Apps").updateOne({App_Id: app_id}, {$set: {Post_Status: 0}}, function (err, res) {
if (err)
throw err;
console.log("1 document updated");
//dbo.close();
});
} else
{
throw error;
}
});
});
}
db.close();
});
} //else
}); //mongoClient connect db
Output
The following is a good start about how to turn callback into promises. Try to use it, execute the code block, by block, understand it and then add your updateOne/insertOne requests into it.
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const express = require('express');
const app = express();
const MongoClient = require('mongodb').MongoClient;
const dateTime = require('node-datetime');
// Class used to handle the database basic interractions
class DB {
constructor() {
this.db = false;
this.url = "mongodb://localhost:27017/";
}
// Do connect to the database
connect() {
return new Promise((resolve, reject) => {
MongoClient.connect(this.url, {
useNewUrlParser: true,
}, (err, db) => {
if (err) {
console.log('error mongodb connect');
return reject(err);
}
this.db = db;
return resolve(db);
});
});
}
disconnect() {
db.close();
this.db = false;
}
getCollection(name) {
return this.db.db(name);
}
}
// Get the data from the database
function getAppsIds(dbObj) {
return new Promise((resolve, reject) => {
const dbo = dbObj.getCollection('WebCrawler');
dbo.collection('Unique_Apps').find({
'Post_Status': 0,
}, {
projection: {
_id: 0,
App_Id: 1,
}
}).toArray(function(err, result) {
if (err) {
return reject(err);
}
return resolve(result);
});
});
}
function requestPlayStore(idApp) {
return new Promise((resolve, reject) => {
const appUrl = `https://play.google.com/store/apps/details?id=${app_id}`;
request(appUrl, function(error, response, html) {
if (error || response.statusCode !== 200) {
return reject(error);
}
return resolve({
response,
html,
});
});
});
}
// Do treat one id app at a time
function treatOneIdApp(dbObj, idApp) {
return requestPlayStore(idApp)
.then(({
response,
html,
}) => {
// Perform your requests here updateOne and insertOne ...
});
}
const dbObj = new DB();
dbObj.connect()
.then(() => getAppsIds(dbObj))
.then(rets => Promise.all(rets.map(x => treatOneIdApp(dbObj, x.App_Id))))
.then(() => dbObj.disconnect())
.catch((err) => {
console.log(err);
});

Unable to retrive data and push inside loop in node js

I am trying to retrieve attendance list along with user details.
I am using caminte.js(http://www.camintejs.com/) Cross-db ORM for database interaction.
Here is my code sample of model function "attendanceList".
exports.attendanceList = function (req, callback) {
var query = req.query;
var searchfilters = {};
if(!req.user){
callback({ code:400, status:'error', message: 'Invalid Request', data:{}});
}else{
searchfilters["vendor_id"] = parseInt(req.user._id);
}
if(query.location && parseString(query.location) != '') {
searchfilters["location"] = parseString(query.location);
}
if (query.device_details && parseString(query.device_details) != '') {
searchfilters["device_details"] = parseString(query.device_details);
}
if(query.created_on) {
searchfilters["created_on"] = query.created_on;
}
if(query.status) {
searchfilters["status"] = { regex: new RegExp(query.status.toLowerCase(), "i") };
}
var SkipRecord = 0;
var PageSize = 10;
var LimitRecord = PageSize;
var PageIndex = 1;
if(query.pagesize) {
PageSize = parseInt(query.pagesize);
}
if(query.pageindex) {
PageIndex = parseInt(query.pageindex);
}
if (PageIndex > 1) {
SkipRecord = (PageIndex - 1) * PageSize;
}
LimitRecord = PageSize;
var SortRecord = "created_on";
if(query.sortby && query.sorttype) {
var sortingBy = query.sortby;
var sortingType = 'ASC';
if(typeof query.sorttype !== 'undefined') {
sortingType = query.sorttype;
}
SortRecord = sortingBy + ' ' + sortingType;
}
Attendance.find({ where: searchfilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, async function (err, result) {
if(err){
callback({ code:400, status:'error', message:'Unable to connect server', errors:err });
} else {
await result.map(function(row, i){
User.findById(parseInt(row.user_id), function(err, data){
if(err){
console.log(err);
} else {
result[i]['userDetails'] = data;
}
});
});
await Attendance.count({ where: searchfilters }, function (err, count) {
callback({ code:200, status:'success', message:'OK', total:count, data:result });
});
}
});
};
I am getting only attendance list without user details. How do I force to push user details into attendance list? Any Help!!
Thank You
This behavior is asynchronous. When you're making request to DB, your code keeps running, while task to get data comes to task queue.
To keep things simple, you need to use promises while handling asynchronous jobs.
Rewrite your code from this:
Attendance.find({ where: searchfilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, async function (err, result) {
if(err){
callback({ code:400, status:'error', message:'Unable to connect server', errors:err });
} else {
await result.map(function(row, i){
User.findById(parseInt(row.user_id), function(err, data){
if(err){
console.log(err);
} else {
result[i]['userDetails'] = data;
}
});
});
await Attendance.count({ where: searchfilters }, function (err, count) {
callback({ code:200, status:'success', message:'OK', total:count, data:result });
});
}
});
To this:
const findAttendanceFirst = (searchFilters, SortRecord, LimitRecord, SkipRecord) => {
return new Promise((resolve, reject) => {
Attendance.find({ where: searchFilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, (err, result) => {
if(err) return reject(err);
resolve(result);
});
});
}
const findUserByIdForUserDetails = (userId) => {
return new Promise((resolve, reject) => {
User.findById(parseInt(userId), function(err, data){
if(err) return reject(err);
resolve(data);
})
});
}
const getAttendanceCount = (searchFilters) => {
return new Promise((resolve, reject) => {
Attendance.count({ where: searchFilters }, (err, count) => {
if(err) return reject(err);
resolve(count);
});
})
}
So, now we can use this separate functions to make async behavior looks like sync.
try {
const data = await findAttendanceFirst(searchFilters, SortRecord, LimitRecord, SkipRecord);
for(let userData of data){
try {
userData.userDetails = await findUserByIdForUserDetails(userData.user_id);
} catch(e) {
// Some error happened, so no user details.
// you can set here null or nothing to userDetails.
}
}
let count;
try {
count = await getAttendanceCount(searchFilters);
} catch(e){
// Same as before.
}
const callBackData = { code:200, status:'success', message:'OK', total:count, data:result };
// And here you can do whatever you want with callback data. Send to client etc.
} catch(e) {
}
NB: I've not tested this code, it will be easier for yu to play with your actual data and use Promises and async/await
Just remember that each request to db is asynchronous, and you need to make your code wait for this data.

Backbone and Express: concatinating (duplicating) routes on res.redirect

I have an action where I need to update MongoDB entry including _id field, which requires deleting old entry and making a new one, here is server side:
exports.update = function(req, res, next){
var outcome = [];
outcome.previousId = req.params.id;
outcome.newId = req.body.name;
var getPreviousRecord = function(callback) {
req.app.db.models.AccountGroup
.findOne({ _id: req.params.id })
.lean()
.exec(function(err, accountGroups) {
if (err) {
return callback(err, null);
}
outcome.accountGroups = accountGroups;
return callback(null, 'done');
});
};
var makeNewRecord = function(callback) {
var permissions = outcome.accountGroups.permissions;
var fieldsToSet = {
_id: outcome.newId.toLowerCase(),
name: outcome.newId,
permissions: permissions
};
req.app.db.models.AccountGroup
.create(fieldsToSet, function(err, record) {
if (err) {
return callback(err, null);
}
outcome.record = record;
return callback(null, 'done');
});
};
var deletePreviousRecord = function() {
req.app.db.models.AccountGroup
.findByIdAndRemove(outcome.previousId)
.exec(function(err) {
if (err) {
return next(err);
}
res.redirect('admin/account-groups/' + outcome.newId + '/');
});
};
var asyncFinally = function(err) {
if (err) {
return next(err);
}
};
require('async').series([getPreviousRecord, makeNewRecord, deletePreviousRecord], asyncFinally);
};
It works fine, but I can't make this work normally on the front-end, it returns me both old route and a new route, for example:
PUT /admin/account-groups/customers22/admin/account-groups/Customers2233/ 404 213.749 ms - 31
where customers22 is old _id and customers2233 is new _id. If I navigate from another page to new entry it gets route normally.
On client side:
(function() {
'use strict';
app = app || {};
app.Details = Backbone.Model.extend({
idAttribute: '_id',
defaults: {
success: false,
errors: [],
errfor: {},
name: ''
},
url: function() {
return '/admin/account-groups/'+ app.mainView.model.id +'/';
},
parse: function(response) {
if (response.accountGroup) {
app.mainView.model.set(response.accountGroup);
delete response.accountGroup;
}
return response;
}
});
app.DetailsView = Backbone.View.extend({
el: '#details',
events: {
'click .btn-update': 'update'
},
template: Handlebars.compile( $('#tmpl-details').html() ),
initialize: function() {
this.model = new app.Details();
this.syncUp();
this.listenTo(app.mainView.model, 'change', this.syncUp);
this.listenTo(this.model, 'sync', this.render);
this.render();
},
syncUp: function() {
this.model.set({
_id: app.mainView.model.id,
name: app.mainView.model.get('name')
});
},
render: function() {
this.$el.html(this.template( this.model.attributes ));
for (var key in this.model.attributes) {
if (this.model.attributes.hasOwnProperty(key)) {
this.$el.find('[name="'+ key +'"]').val(this.model.attributes[key]);
}
}
},
update: function() {
this.model.save({
name: this.$el.find('[name="name"]').val()
});
}
});
app.MainView = Backbone.View.extend({
el: '.page .container',
initialize: function() {
app.mainView = this;
this.model = new app.AccountGroup( JSON.parse( unescape($('#data-record').html()) ) );
// ...
app.detailsView = new app.DetailsView();
}
});
$(document).ready(function() {
app.mainView = new app.MainView();
});
}());
It probably requires to trigger both model.save and model.destroy or prevent URL being used. Any advice on how to do it is appreciated, thank you.
Edit
Just a typo mistake here that is not related to the question, recklessly checking routes, see as cancelled
I believe the problem is here:
res.redirect('admin/account-groups/' + outcome.newId + '/');
That's a relative path so it'll be appended onto the current URL. I suspect you want something like this:
res.redirect('/admin/account-groups/' + outcome.newId + '/');

node js mongo db dependencies (doc not being found)

I have the following code:
var method = PushLoop.prototype;
var agent = require('./_header')
var request = require('request');
var User = require('../models/user_model.js');
var Message = require('../models/message_model.js');
var async = require('async')
function PushLoop() {};
method.startPushLoop = function() {
getUserList()
function getUserList() {
User.find({}, function(err, users) {
if (err) throw err;
if (users.length > 0) {
getUserMessages(users)
} else {
setTimeout(getUserList, 3000)
}
});
}
function getUserMessages(users) {
// console.log("getUserMessages")
async.eachSeries(users, function (user, callback) {
var params = {
email: user.email,
pwd: user.password,
token: user.device_token
}
messageRequest(params)
callback();
}, function (err) {
if (err) {
console.log(err)
setTimeout(getUserList, 3000)
}
});
}
function messageRequest(params) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ params.email +"&api_password="+ params.pwd +"&method=getSMS&type=1&limit=5"
request(url, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
if (responseObject["status"] == "success") {
async.eachSeries(messages, function(message, callback){
console.log(params.token)
saveMessage(message, params.token)
callback();
}, function(err) {
if (err) {
console.log(err)
}
// setTimeout(getUserList, 3000)
})
} else {
// setTimeout(getUserList, 3000)
}
} else {
console.log(err)
// setTimeout(getUserList, 3000)
}
});
setTimeout(getUserList, 3000)
}
function saveMessage(message, token) {
// { $and: [ { price: { $ne: 1.99 } }, { price: { $exists: true } }
// Message.find({ $and: [{ message_id: message.id}, {device_token: token}]}, function (err, doc){
Message.findOne({message_id: message.id}, function (err, doc){
if (!doc) {
console.log('emtpy today')
var m = new Message({
message_id: message.id,
did: message.did,
contact: message.contact,
message: message.message,
date: message.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: token
});
m.save(function(e) {
if (e) {
console.log(e)
} else {
agent.createMessage()
.device(token)
.alert(message.message)
.set('contact', message.contact)
.set('did', message.did)
.set('id', message.id)
.set('date', message.date)
.set('message', message.message)
.send();
}
});
}
}) //.limit(1);
}
};
module.exports = PushLoop;
Which actually works perfectly fine in my development environment - However in production (i'm using Openshift) the mongo documents get saved in an endless loop so it looks like the (if (!doc)) condition always return true therefore the document gets created each time. Not sure if this could be a mongoose issue - I also tried the "find" method instead of "findOne". My dev env has node 0.12.7 and Openshift has 0.10.x - this could be the issue, and i'm still investigating - but if anybody can spot an error I cannot see in my logic/code please let me know
thanks!
I solved this issue by using a "series" like pattern and using the shift method on the users array. The mongoose upsert findOneOrCreate is good however if there is a found document, the document is returned, if one isn't found and therefore created, it's also returned. Therefore I could not distinguish between the newly insert doc vs. a found doc, so used the same findOne function which returns null if no doc is found I just create it and send the push notification. Still abit ugly, and I know I could have used promises or the async lib, might refactor in the future. This works for now
function PushLoop() {};
var results = [];
method.go = function() {
var userArr = [];
startLoop()
function startLoop() {
User.find({},function(err, users) {
if (err) throw err;
users.forEach(function(u) {
userArr.push(u)
})
function async(arg, callback) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ arg.email +"&api_password="+ arg.password +"&method=getSMS&type=1&limit=5"
request.get(url, {timeout: 30000}, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
var status = responseObject.status
if (status === "success") {
messages.forEach(function(m) {
var message = new Message({
message_id: m.id,
did: m.did,
contact: m.contact,
message: m.message,
date: m.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: arg.device_token
});
var query = { $and : [{message_id: m.id}, {device_token: arg.device_token}] }
var query1 = { message_id: m.id }
Message.findOne(query).lean().exec(function (err, doc){
if (!doc || doc == null) {
message.save(function(e) {
console.log("message saved")
if (e) {
console.log("there is an error")
console.log(e)
} else {
console.log(message.device_token)
var messageStringCleaned = message.message.toString().replace(/\\/g,"");
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : messageStringCleaned
}
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = messageStringCleaned;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
}
})
}
});
});
}
else {
console.log(err)
}
}
});
setTimeout(function() {
callback(arg + "testing 12");
}, 1000);
}
// Final task (same in all the examples)
function series(item) {
if(item) {
async( item, function(result) {
results.push(result);
return series(userArr.shift());
});
} else {
return final();
}
}
function final() {
console.log('Done');
startLoop();
}
series(userArr.shift())
});
}
}
module.exports = PushLoop;

How to make tedious Async in Meteor

I am using this module tedious to connect. I am having issues when I try to populate a collection with the data from MSSQL.
My code thus far:
http://pastebin.com/q4ByRCbW
Meteor.startup(function () {
var Request = Meteor.require('tedious').Request;
var Connection = Meteor.require('tedious').Connection;
var config = {
userName: 'xxxxx',
password: 'xxxx',
server: '197.xxx.xxx.xxx',
// If you're on Windows Azure, you will need this:
options: {
encrypt: true,
debug: {
packet: true,
data: true,
payload: true,
token: false,
log: true
}
}
};
var connection = new Connection(config);
var asnycWrapFunc = Async.wrap(connection.execSql);
var rettarr = [];
function executeStatement() {
Fiber(function(){
request = new Request("select * from AccountSummary", function(err, rowCount) {
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
});
request.on('row', function(columns) {
aaary = []; cnting = 0;
columns.forEach(function(column) {
console.log(column.value);
aaary.push(column.value);
});
if (AccountSummary.find().count() === 0){
AccountSummary.insert({ID:aaary[0], ClientNo:aaary[1], ClientName:aaary[2]});
}
});
//rettarr.push(aaary);
}).run();
asnycWrapFunc(request);
//return rettarr;
}
connection.on('connect', function(err) {
// If no error, then good to go...
var res = executeStatement();
// aaary = res[0];
console.log(res);
errr = err;
});
});
I have found that you have to use Future if you want to you a package like Tedious.
This mini tutorial has the answer

Resources