I have been learning about q promises and tried to build up some mock APIs to implement its functionality,While doing so I came across the following error,
Enterprise.forEach is not a function
My API code is as follows,
var mongoose = require('mongoose');
var Enterprise = mongoose.model('Enterprise_gpy');
var q = require('q');
var displayEnterprise = function(req, res) {
function displayEnterpriseName() {
var deferred = q.defer();
Enterprise.forEach(function(err, doc) {
if (err) {
console.log('Error Finding Files');
deferred.reject(err);
} else {
var name = Enterprise.enterprise_name;
deferred.resolve({
name: name
});
}
return deferred.promise;
});
}
function displayEnterpriseEmail() {
var deferred = q.defer();
Enterprise.forEach(function(err, doc) {
if (err) {
console.log('Error Finding Files');
deferred.reject(err);
} else {
var email = Enterprise.enterprise_email;
deferred.resolve({
email: email
});
}
return deferred.promise;
});
}
q.all([
displayEnterpriseName(),
displayEnterpriseEmail()
])
.then(function(success) {
console.log(500, success);
})
.fail(function(err) {
console.log(200, err);
});
}
module.exports = {
displayEnterprise: displayEnterprise
}
In your code Enterprise is a mongoose schema so when you try to do loop using forEach then got
Enterprise.forEach is not a function
you can use forEach after Enterprise.find(). so use
Enterprise.find({}, function(err, docs) {
if (err) {
console.log('Error Finding Files');
deferred.reject(err);
} else {
var names = [];
docs.forEach (function(doc) {
var name = doc.enterprise_name;
names.push(name);// pushed in names array
//.....
});
deferred.resolve({
names: names
}); // return all names
}
});
instead of
Enterprise.find().forEach
and should use
var name = doc.enterprise_name; instead of var name = Enterprise.enterprise_name;
and
var email = doc.enterprise_email; instead of var email = Enterprise.enterprise_email;
forEach only works for arrays, and you're using it on a mongoose model.
try this instead:
Enterprise.find().exec(function(err, docs) {
docs.forEach(function(doc) {
// do something with all the documents
}
// do something outside the loop
})
Related
I have searched for the solution of the error specified in title.
MongoError: server instance pool was destroyed
I believe it is because misplacement of db.close(). But I am nesting dbo.collection and unable to get the exact solution of this error.
Firstly, I am fetching data (array of ids having status 0) from database and then I am concatenating (each app-id) them one by one with URL to get desired appUrl which will be used for crawling data one by one and then crawled data is meant to be stored into another collection of mongoDB. This process will repeat for each id in the array. But my code is having error of "server instance pool gets destroyed" before storing data into collection. I am doing misplacement of db.close() but I am unable to resolve this. Please help me resolving this error
Here is my code
///* global sitehead */
const request = require('request');
const cheerio = require('cheerio');
//const response = require('response');
const fs = require('fs');
const express = require('express');
const app = express();
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/";
var dateTime = require('node-datetime');
MongoClient.connect(url, {useNewUrlParser: true}, function (err, db) {
if (err) {
throw err;
} else {
var dbo = db.db("WebCrawler");
var app_id;
var appUrl;
let arr = [];
dbo.collection("Unique_Apps").find({"Post_Status": 0}, {projection: {_id: 0, App_Id: 1}}).toArray(function (err, result)
{
// console.log(result);
if (err) {
throw err;
// console.log(err);
} else {
for (var i = 0; i < result.length; i++)
{
arr[i] = result[i];
}
arr.forEach((el) => {
app_id = el.App_Id;
//console.log(app_id);
appUrl = 'https://play.google.com/store/apps/details?id=' + app_id;
console.log(appUrl);
request(appUrl, function (error, response, html) {
if (!error && response.statusCode === 200) {
//START Crawling ###########
const $ = cheerio.load(html); //cheerio
const appTitle = $('.AHFaub');
const iconUrl = $('.T75of.sHb2Xb').attr("src");
const developedBy = $('.T32cc.UAO9ie').children().eq(0);
const category = $('.T32cc.UAO9ie').children().eq(1);
//store in database collection: "Single_App_Data_Post"
var curr_obj = {App_Id: app_id, App_Name: appTitle.text(),
Icon_Url: iconUrl, Price: "Free", Developed_By: developedBy.text(),
Category: category.text()
};
dbo.collection("Single_App_Data_Post").insertOne(curr_obj, function (err, res) {
console.log("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
if (err) {
throw err;
// console.log(err);
} else {
console.log("inserted....");
} //main else
});
dbo.collection("Unique_Apps").updateOne({App_Id: app_id}, {$set: {Post_Status: 0}}, function (err, res) {
if (err)
throw err;
console.log("1 document updated");
//dbo.close();
});
} else
{
throw error;
}
});
});
}
db.close();
});
} //else
}); //mongoClient connect db
Output
The following is a good start about how to turn callback into promises. Try to use it, execute the code block, by block, understand it and then add your updateOne/insertOne requests into it.
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const express = require('express');
const app = express();
const MongoClient = require('mongodb').MongoClient;
const dateTime = require('node-datetime');
// Class used to handle the database basic interractions
class DB {
constructor() {
this.db = false;
this.url = "mongodb://localhost:27017/";
}
// Do connect to the database
connect() {
return new Promise((resolve, reject) => {
MongoClient.connect(this.url, {
useNewUrlParser: true,
}, (err, db) => {
if (err) {
console.log('error mongodb connect');
return reject(err);
}
this.db = db;
return resolve(db);
});
});
}
disconnect() {
db.close();
this.db = false;
}
getCollection(name) {
return this.db.db(name);
}
}
// Get the data from the database
function getAppsIds(dbObj) {
return new Promise((resolve, reject) => {
const dbo = dbObj.getCollection('WebCrawler');
dbo.collection('Unique_Apps').find({
'Post_Status': 0,
}, {
projection: {
_id: 0,
App_Id: 1,
}
}).toArray(function(err, result) {
if (err) {
return reject(err);
}
return resolve(result);
});
});
}
function requestPlayStore(idApp) {
return new Promise((resolve, reject) => {
const appUrl = `https://play.google.com/store/apps/details?id=${app_id}`;
request(appUrl, function(error, response, html) {
if (error || response.statusCode !== 200) {
return reject(error);
}
return resolve({
response,
html,
});
});
});
}
// Do treat one id app at a time
function treatOneIdApp(dbObj, idApp) {
return requestPlayStore(idApp)
.then(({
response,
html,
}) => {
// Perform your requests here updateOne and insertOne ...
});
}
const dbObj = new DB();
dbObj.connect()
.then(() => getAppsIds(dbObj))
.then(rets => Promise.all(rets.map(x => treatOneIdApp(dbObj, x.App_Id))))
.then(() => dbObj.disconnect())
.catch((err) => {
console.log(err);
});
I am using the following to insert into MongoDB.
var tagData = JSON.parse(data);
var allTags = tagData.tags;
for (var j = 0; j < allTags.length; j++) {
var p = allTags[j].tagId.toString();
for (var k = 0; k < loggerParams.length; k++) {
var q = Object.keys(loggerParams[k]).toString();
if (p === q) {
// Prepare raw data tag
var tagRawDoc = {};
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
database.addDocument('tagraw', tagRawDoc, function (err) {
if (err) {
log.info(util.format('Error adding document to tagrawdatas. %s', err.message));
throw err;
} else {
// Prepare history tag
var historyTagDoc = {};
historyTagDoc.tagNameAlias = tagRawDoc.tagNameAlias;
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
database.addDocument('taghistory', historyTagDoc, function (err) {
if (err) {
log.info(util.format('Error adding document to tagrawdatas. %s', err.message));
throw err;
}
});
}
});
// Match found; exit loop
break;
}
}
}
The loggerParms is a simple JSON document read from file else-where. It allows for look-up in this code to build the document to be inserted. There will be 12 values in the allTags array. These 12 values are inserted successfully into the tagraw collection. However, in taghistory collection, the values from the last (or most recent) entry made into tagraw collection is repeated 12 times. Why does this happen?
The database.addDocument is shown below. It is a part of this article I am trying to replicate.
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var logger = require('../../util/logger');
var util = require('util');
function DB() {
this.db = "empty";
this.log = logger().getLogger('mongoMange-DB');
}
DB.prototype.connect = function(uri, callback) {
this.log.info(util.format('About to connect to DB'));
if (this.db != "empty") {
callback();
this.log.info('Already connected to database.');
} else {
var _this = this;
MongoClient.connect(uri, function(err, database) {
if (err) {
_this.log.info(util.format('Error connecting to DB: %s', err.message));
callback(err);
} else {
_this.db = database;
_this.log.info(util.format('Connected to database.'));
callback();
}
})
}
}
DB.prototype.close = function(callback) {
log.info('Closing database');
this.db.close();
this.log.info('Closed database');
callback();
}
DB.prototype.addDocument = function(coll, doc, callback) {
var collection = this.db.collection(coll);
var _this = this;
collection.insertOne(doc, function(err, result) {
if (err) {
_this.log.info(util.format('Error inserting document: %s', err.message));
callback(err.message);
} else {
_this.log.info(util.format('Inserted document into %s collection.', coll));
callback();
}
});
};
module.exports = DB;
That's because you are mixing a/multiple synchronous for and asynchronous code with database.addDocument which cause issues with function scope in nodejs.
A simple example of this kind of thing:
for(var i = 0; i < 10; i++){
setTimeout(() => console.log(i), 0);
}
You should use a package like async to handle flow control when iterating arrays/object asynchronously.
Simple example of your code refactored to use async:
var async = require('async');
var tagData = JSON.parse(data);
var allTags = tagData.tags;
async.each(allTags, function(tag, done){
var p = tag.tagId.toString();
var loggerParam = loggerParams.find(function(loggerParam){
var q = Object.keys(loggerParam).toString();
return p === q;
});
var tagRawDoc = {};
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
return database.addDocument('tagraw', tagRawDoc, function (err){
if (err) return done(err);
// Prepare history tag
var historyTagDoc = {};
historyTagDoc.tagNameAlias = tagRawDoc.tagNameAlias;
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
return database.addDocument('taghistory', historyTagDoc, done);
});
}, (err) => {
if(err) throw err;
console.log('All done');
});
I've never had to do this before. I've created myModule.GetRecord() and I can see the recordset successfully has the expected record.
However, ret in router.get() is undefined.
I believe because I need to catch the returned value in a callback.
I don't know how to define this module/function for a callback (if that is what I am appropriately looking to do) or how to call this function for a callback.
I've tried a few different things i.e. the typical GetRecord(ret, function () { ... }); But didn't see anything that appeared to work. And I read a bunch on the line but didn't find what I believe I'm looking for.
I really don't care to much about how I get there, but all I'm really trying to do is have mm.GetRecord()'s returned value in some usable form in the router.get()
--- myModulefile.js ---
'use strict';
module.exports = {
GetRecord: function (id) {
var sql = require('mssql');
sql.connect({ user: 'sa', ... database: 'name' }, function (err) {
if (err) { console.log(err); return null; }
var cmd = new sql.Request();
cmd.query('select * from Records where id = ' + id, function (err, rs) {
if (err) { console.log(err); return null; }
if (rs.recordset.length > 0) {
console.log('rs[0]', rs.recordset[0]);
return rs.recordset[0];
} else {
return null;
}
});
});
}
};
--- myRouter.js ----
const express = require('express');
const router = express.Router();
const mm = require('../myModule');
router.get('/:id', function (req, res) {
var id = req.params.id;
var ret = mm.GetRecord(id)
console.log('ret', ret);
if (ret == null)
ret = JSON.stringify({ ID: -1, f1: 'unknown' });
res.send(ret);
});
module.exports = router;
Of course I find the answer after having placed the question on here.
GetRecord() has to be defined with a parameter that recieves the callback passed to it. And the callback paramater's variable fn is used in place of return
GetRecord: function (id, fn) {
var sql = require('mssql');
sql.connect({ user: 'sa', ... database: 'name' }, function (err) {
if (err) { console.log(err); fn(null); }
var cmd = new sql.Request();
cmd.query('select * from Records where id = ' + id, function (err, rs) {
if (err) { console.log(err); fn(null); }
if (rs.recordset.length > 0) {
console.log('rs[0]', rs.recordset[0]);
fn(rs.recordset[0]);
} else {
fn(null);
}
});
});
}
and
GetRecord(id, function(ret) {
console.log('ret', ret);
});
I have a module which i wrote myself. It contains 3 functions which i call as chained promises. The last one does not execute and i cannot figure out why.
The module geodata.js
require('use-strict');
var Promise = require('promise');
var NodeGeocoder = require('node-geocoder');
var mongoose = require('mongoose');
var User = require('../models/user');
var options = {
provider: 'google',
httpAdapter: 'https',
apiKey: 'AIzaSyA4v81WbNOMeRL7p911Mxr6PBZnidX0cIM',
formatter: null
};
module.exports = {
//*******************************************************
// Find user and return his adress (street + town)
//
//*******************************************************
findUser: function(username) {
return new Promise(function(resolve, reject) {
User.findOne({
'username': username
}, function(err, doc) {
if (!err) {
resolve(doc);
} else {
reject(err);
}
});
});
},
//*******************************************************
// Fetch geodata (latitude + longitude) to users record in
// the user collection.
//*******************************************************
fetchGeoData: function(userObj) {
return new Promise(function(resolve, reject) {
var geocoder = NodeGeocoder(options);
var adress = userObj.street + ' ' + userObj.town;
geocoder.geocode(adress, function(err, res) {
if (!err) {
var res2 = {
'username': userObj.username
}
console.log(res);
resolve([res, res2]);
} else {
reject(err);
}
});
});
},
//*******************************************************
// Fetch geodata (latitude + longitude) to users record in
// the user collection.
//*******************************************************
addGeoData: function(message) {
return new Promise(function(resolve, reject) {
User.findOne({
'username': message.username
}, function(err, doc) {
console.log(doc);
if (err) {
console.log(err);
reject(err);
}
if (!doc) {
console.log('User not found.');
reject('User not found.');
}
doc.longitude = String(message.longitude);
doc.latitude = String(message.latitude);
doc.save();
resolve(doc);
});
});
},
fixJSON: function(inpJSON) {
var strung = JSON.stringify(inpJSON);
obj = strung.substring(1, strung.length - 1);
return (JSON.parse(obj));
}
};
.. and here is the calling script:
require('use-strict');
var mongoose = require('mongoose');
var dbConfig = require('./db');
var geodata = require('./lib/geodata.js');
// Connect to DB
mongoose.connect(dbConfig.url, {
auth: {
authdb: "admin"
}
});
mongoose.set('debug', true);
geodata.findUser('jimmy').then(function(result) {
console.log('findUser() done');
return geodata.fetchGeoData(result);
}).then(function(result) {
console.log('fetchGeoData() done');
var res1Fixed = geodata.fixJSON(result[0]);
var params = {
username: result[1].username,
longitude: res1Fixed.longitude,
latitude: res1Fixed.latitude
}
console.log(params);
return geodata.addGeoData(params);
}).then(function(result) {
console.log('addGeoData() done');
});
mongoose.connection.close();
the execution stops after this line:
console.log('fetchGeoData() done');
and i can't figure out why?
Regards
Jimmy
If you are referring to mongoose.connection.close(); that is actually being run before your console.log('addGeoData() done');
Notice that mongoose.connection.close() is synchronous whereas the mongoose.findUser is asynchronous.
I have the following code:
var method = PushLoop.prototype;
var agent = require('./_header')
var request = require('request');
var User = require('../models/user_model.js');
var Message = require('../models/message_model.js');
var async = require('async')
function PushLoop() {};
method.startPushLoop = function() {
getUserList()
function getUserList() {
User.find({}, function(err, users) {
if (err) throw err;
if (users.length > 0) {
getUserMessages(users)
} else {
setTimeout(getUserList, 3000)
}
});
}
function getUserMessages(users) {
// console.log("getUserMessages")
async.eachSeries(users, function (user, callback) {
var params = {
email: user.email,
pwd: user.password,
token: user.device_token
}
messageRequest(params)
callback();
}, function (err) {
if (err) {
console.log(err)
setTimeout(getUserList, 3000)
}
});
}
function messageRequest(params) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ params.email +"&api_password="+ params.pwd +"&method=getSMS&type=1&limit=5"
request(url, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
if (responseObject["status"] == "success") {
async.eachSeries(messages, function(message, callback){
console.log(params.token)
saveMessage(message, params.token)
callback();
}, function(err) {
if (err) {
console.log(err)
}
// setTimeout(getUserList, 3000)
})
} else {
// setTimeout(getUserList, 3000)
}
} else {
console.log(err)
// setTimeout(getUserList, 3000)
}
});
setTimeout(getUserList, 3000)
}
function saveMessage(message, token) {
// { $and: [ { price: { $ne: 1.99 } }, { price: { $exists: true } }
// Message.find({ $and: [{ message_id: message.id}, {device_token: token}]}, function (err, doc){
Message.findOne({message_id: message.id}, function (err, doc){
if (!doc) {
console.log('emtpy today')
var m = new Message({
message_id: message.id,
did: message.did,
contact: message.contact,
message: message.message,
date: message.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: token
});
m.save(function(e) {
if (e) {
console.log(e)
} else {
agent.createMessage()
.device(token)
.alert(message.message)
.set('contact', message.contact)
.set('did', message.did)
.set('id', message.id)
.set('date', message.date)
.set('message', message.message)
.send();
}
});
}
}) //.limit(1);
}
};
module.exports = PushLoop;
Which actually works perfectly fine in my development environment - However in production (i'm using Openshift) the mongo documents get saved in an endless loop so it looks like the (if (!doc)) condition always return true therefore the document gets created each time. Not sure if this could be a mongoose issue - I also tried the "find" method instead of "findOne". My dev env has node 0.12.7 and Openshift has 0.10.x - this could be the issue, and i'm still investigating - but if anybody can spot an error I cannot see in my logic/code please let me know
thanks!
I solved this issue by using a "series" like pattern and using the shift method on the users array. The mongoose upsert findOneOrCreate is good however if there is a found document, the document is returned, if one isn't found and therefore created, it's also returned. Therefore I could not distinguish between the newly insert doc vs. a found doc, so used the same findOne function which returns null if no doc is found I just create it and send the push notification. Still abit ugly, and I know I could have used promises or the async lib, might refactor in the future. This works for now
function PushLoop() {};
var results = [];
method.go = function() {
var userArr = [];
startLoop()
function startLoop() {
User.find({},function(err, users) {
if (err) throw err;
users.forEach(function(u) {
userArr.push(u)
})
function async(arg, callback) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ arg.email +"&api_password="+ arg.password +"&method=getSMS&type=1&limit=5"
request.get(url, {timeout: 30000}, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
var status = responseObject.status
if (status === "success") {
messages.forEach(function(m) {
var message = new Message({
message_id: m.id,
did: m.did,
contact: m.contact,
message: m.message,
date: m.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: arg.device_token
});
var query = { $and : [{message_id: m.id}, {device_token: arg.device_token}] }
var query1 = { message_id: m.id }
Message.findOne(query).lean().exec(function (err, doc){
if (!doc || doc == null) {
message.save(function(e) {
console.log("message saved")
if (e) {
console.log("there is an error")
console.log(e)
} else {
console.log(message.device_token)
var messageStringCleaned = message.message.toString().replace(/\\/g,"");
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : messageStringCleaned
}
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = messageStringCleaned;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
}
})
}
});
});
}
else {
console.log(err)
}
}
});
setTimeout(function() {
callback(arg + "testing 12");
}, 1000);
}
// Final task (same in all the examples)
function series(item) {
if(item) {
async( item, function(result) {
results.push(result);
return series(userArr.shift());
});
} else {
return final();
}
}
function final() {
console.log('Done');
startLoop();
}
series(userArr.shift())
});
}
}
module.exports = PushLoop;