I have following code in my nodejs app:
db.open(function(err, db) {
while (item = stream.read())
{
var parsedDate = parseDateFromString(item.title);
var movieName = parseNameFromString(item.title);
var channelName = parseChannelFromString(item.title);
//var doc = { "name": movieName, "channel": channelName, "showed": parsedDate, "showDate": parsedDate.format(), "run_id": new Date().toString()};
var doc = { "name": movieName, "channel": channelName, "showed": parsedDate, "run_id": new Date().toString()};
db.collection('programs', function(err, collection) {
collection.find({'name' : doc.name, "channel" : doc.channel}).count(function(countErr, count) {
console.log('count!');
if(count == 0)
insertDocument(collection, doc);
else
console.log('Found match from db with name ' + matchDoc.name);
});
});
} // End of while
}); // End of Open
Problem is that count! is never reported. So collection.count is not calling its callback. I checked that collection.find is called multiple times. What could cause that problem?
Related
I am trying to create Dynamic JSON from database but due to async behaviour of node js I am not able to do that.
exports.wsGetDirectoryTree = function(request,response){
pool.getConnection(function(err, connection) {
if (err) {
console.log("Error while connecting to DB : " + err);
response.send(error);
return;
} else {
// Org List
objDb.wsGetOrganisationTree(connection, function(callback) {
if(callback){
var data= JSON.stringify(callback);
var jsonObject = JSON.parse(data);
var count = Object.keys(jsonObject).length;
var finalResponse = '';
var org = '';
var gst = '';
var url = '';
// Org List
for(var i=0;i<count;i++){
var temp = JSON.stringify(jsonObject[i]);
var tempData = JSON.parse(temp);
(function(i){
// {"Apple"
org = '{'+'\"'+tempData.organisation_name+'\":';
console.log("org -> "+org);
// Guest list
objDb.wsGetGuestTree(tempData.organisation_id,connection, function(callback) {
if(callback){
var data= JSON.stringify(callback);
var jsonObject = JSON.parse(data);
var count = Object.keys(jsonObject).length;
// Guest list
for(var j=0;j<count;j++){
var temp = JSON.stringify(jsonObject[j]);
var tempData = JSON.parse(temp);
//{"Jhon":
gst = '{'+'\"'+tempData.guest_name+'\":';
console.log("gst = "+gst);
finalResponse = org + gst;
// Url list
objDb.wsGetUrlTree(tempData.guest_id,connection, function(callback) {
if(callback){
var data= JSON.stringify(callback);
finalResponse = finalResponse + data;
console.log("finalResponse = "+finalResponse);
return;
} else {
return;
}
});
}
data = data.replace('[','');
data = data.replace(']','');
return;
} else {
return;
}
});
})(i); //function for counting
} //loop end
response.send(data);
return;
} else {
response.send(invalidData);
return;
}
});
}
connection.release();
});
};
I am expecting below output -
{
"Organisation01": {
"Guest01": {
"Images": ["IMG-20180117-WA0004.jpg", "IMG-20180117-WA0004.jpg"],
"Video": ["IMG-20180117-WA0004.jpg", "IMG-20180117-WA0004.jpg"]
},
"Guest02": {
"Images": ["IMG-20180117-WA0004.jpg", "IMG-20180117-WA0004.jpg"]
}
}, "Organisation02": {
"Guest01": {
"Images": ["IMG-20180117-WA0004.jpg", "IMG-20180117-WA0004.jpg"]
} }, "Organisation03": {
"Guest01": {
"Images": ["IMG-20180117-WA0004.jpg", "IMG-20180117-WA0004.jpg"]
} }, "Organisation04": {
"Guest01": {
"Images": ["IMG-20180117-WA0004.jpg", "IMG-20180117-WA0004.jpg"]
} } }
Since your for loop over count actually requires to make async calls objDb.wsGetGuestTree() , it would be advisable to use async module.
Please check :
https://www.npmjs.com/package/async
https://caolan.github.io/async/docs.html#eachSeries
I've a program that does the below.
Look into a DynamoDB table.
Get the data from the table.
Save the variables in session
After the process, print the values in console.
My code is as below.
intentHandlers['GetMYBusinessInfo'] = function (request, session, response, slots) {
console.log('entered ext bloxk');
if (!session.attributes.userName) {
console.log('eneterd the user entered the block');
var userName = 'jim';
isUserRegistered(userName.toLowerCase(), function (res, err) {
if (err) {
response.fail(err);
console.log(err);
}
else if (!res) {
response.shouldEndSession = true;
}
else {
console.log(res);
var countRes = JSON.stringify(res.Count);
var unpUserRegion = JSON.stringify(res.Items[0].Region);
var unpUserCity = JSON.stringify(res.Items[0].State);
var userRegion = JSON.parse(unpUserRegion);
var userCity = JSON.parse(unpUserCity);
session.attributes.city = userCity;
session.attributes.region = userRegion;
console.log("parsed " + countRes + "\t region is " + userRegion);
session.attributes.userName = true;
}
});
}
console.log(`session values after authentication are user city is ${session.attributes.city}`);
}
The method to check if the value is in DynamoDb or not.
function isUserRegistered(userName, callback) {
var params = {
TableName: "usersTable",
FilterExpression: "#nme = :nme",
ExpressionAttributeNames: {
"#nme": "Name",
},
ExpressionAttributeValues: {
":nme": userName
}
};
var count = 0;
docClient.scan(params, function (err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
callback(false, err);
} else {
console.log("Scan succeeded." + data.Items.length);
if (data.Items.length === 0) {
callback(false);
}
else {
data.Items.forEach(function (itemData) {
console.log("Item :", ++count, JSON.stringify(itemData));
});
callback(data);
}
}
});
}
when I run this, the output that I get is:
session values after authentication are user city is undefined
Scan succeeded.1
Item : 1
{
"ID": "3",
"State": "wisconsin",
"Region": "midwest",
"Name": "jim"
}
{ Items: [ { ID: '3', State: 'wisconsin', Region: 'midwest', Name: 'jim' } ],
Count: 1,
ScannedCount: 1 }
parsed 1 region is midwest
Here I know that Node js being Non-blockable process, the above output is correct, but I want to get the value of city printed in session values after authentication are user city is {HereTheCityComes} instead of session values after authentication are user city is undefined.
I'm sure that placing the console.log(session values after authentication are user city is ${session.attributes.city}); in the last else block(place where the data is returned).
But I need this type of functionality(Get data as shown in my current scenario), as there is some other things to be done after checking if the user is available in database.
please let me know where am I going wrong and how can I fix this.
You can't synchronously expect async result.
What you can do here is solve your problem with promises.
Here is a solution:
intentHandlers['GetMYBusinessInfo'] = function(request, session, response, slots) {
console.log('entered ext bloxk');
var userPromise = Promise.resolve();
if (!session.attributes.userName) {
console.log('eneterd the user entered the block');
var userName = 'jim';
userPromise = new Promise(function (resolve, reject) {
isUserRegistered(userName.toLowerCase(), function (res, err) {
if (err) {
response.fail(err);
reject(err);
}
var countRes = JSON.stringify(res.Count);
var unpUserRegion = JSON.stringify(res.Items[0].Region);
var unpUserCity = JSON.stringify(res.Items[0].State);
var userRegion = JSON.parse(unpUserRegion);
var userCity = JSON.parse(unpUserCity);
session.attributes.city = userCity;
session.attributes.region = userRegion;
console.log("parsed " + countRes + "\t region is " + userRegion);
resolve(res);
});
});
}
userPromise.then(function () {
console.log(`session values after authentication are user city is ${session.attributes.city}`);
});
}
If you are not using ES6, then just install bluebird and use var Promise = require('bluebird')
I'm trying to traverse a collection and update an array for each document.
What am I doing wrong below?
var MongoClient = require('mongodb').MongoClient;
var removeLowestHWScore = function(scores) {
var lowestHWID = -1;
for(var i=0;i<scores.length; i++) {
if (scores[i].type === 'homework') {
if (lowestHWID === -1) {
lowestHWID = i;
} else if (scores[i].score < scores[lowestHWID].score) {
lowestHWID = i;
}
}
}
scores.splice(lowestHWID);
return scores;
};
var callback = function(err, r) {
console.log('updated record');
};
var updateScore = function(err, doc) {
var updatedScores = removeLowestHWScore(doc.scores);
collection.updateOne({_id:doc._id},
{$set: {scores: updatedScores }},
callback);
};
MongoClient.connect('mongodb://localhost:27017/school', function(err, db) {
if(err) throw err;
var collection = db.collection('students');
var cursor = collection.find({});
var next = cursor.next();
while (next) {
next(updateScore);
next = cursor.next();
}
db.close();
});
error
/Users/harrymoreno/programming/mongodb/mongo101ForNode/node_modules/mongodb
/lib/mongodb/mongo_client.js:475
throw err
^
TypeError: Object #<Cursor> has no method 'next'
at /Users/harrymoreno/programming/mongodb/mongo101ForNode/week03/app.js:35:21
at /Users/harrymoreno/programming/mongodb/mongo101ForNode/node_modules/mongodb/lib/mongodb/mongo_client.js:472:11
at process._tickCallback (node.js:419:13)
sample student
{
"_id" : 137,
"name" : "Tamika Schildgen",
"scores" : [
{
"type" : "exam",
"score" : 4.433956226109692
},
{
"type" : "quiz",
"score" : 65.50313785402548
},
{
"type" : "homework",
"score" : 89.5950384993947
}
]
}
UPDATED - v.2
According to the information provided in your last remark abut mongodb package version, I've changed to solution to the one you've improved for specific version compliance (using the 1.4.x node.js mongodb driver) :
var MongoClient = require('mongodb').MongoClient;
var cursor = null,
collection = null,
dbSrc = null;
var removeLowestHWScore = function(scores) {
var lowestHWID = -1;
for(var i=0;i<scores.length; i++) {
if (scores[i].type === 'homework') {
if (lowestHWID === -1) {
lowestHWID = i;
} else if (scores[i].score < scores[lowestHWID].score) {
lowestHWID = i;
}
}
}
// scores.splice(lowestHWID);
if (lowestHWID >= 0)
scores.splice(lowestHWID, 1);
return scores;
};
var callback = function(err, r) {
if (err) throw err;
console.log('updated record');
cursor.nextObject(updateScore);
};
var updateScore = function(err, doc) {
if (err) throw err;
if (doc === null)
return dbSrc.close();
var updatedScores = removeLowestHWScore(doc.scores);
collection.update({_id:doc._id},
{$set: {scores: updatedScores }},
callback);
};
MongoClient.connect('mongodb://localhost:27017/school', function(err, db) {
if(err) throw err;
dbSrc = db;
collection = db.collection('students');
cursor = collection.find({});
cursor.nextObject(updateScore);
});
There is a small code i have written below and issue is explained within comments. Problem is with the bold callback function while creating the collection. There is an error while creating the collection and the message should be displayed as soon as the main function for creating the collections is ending but message appears randomly as seen in the output below:
It is called unexpected on line marked with question mark. I am running this js file on node.js and mongoDB environment.
Thanks.
var mongo = require("mongodb");
var Db = mongo.Db;
var mongoC = mongo.MongoClient;
var assert = require('assert');
mongoC.connect("mongodb://localhost:27017/social", {
native_parser: true
}, function (err, db) {
if (!err) {
console.log("We are Connected!\n");
//creating the Collection
db.createCollection("node", {
strict: true
}, function (err, coll) {
if (!err) {
console.log("*********Collection Created Succesfully*********");
console.log("Collection created Successfully\n" + JSON.stringify(coll) + "\n-------------------\n");
}
else{
console.log("Cannot create Collection because " + err);
}
});
//Collection created now
console.log("*********************************inserting documents in the selected collection***********************");
var coll = db.collection('node');
var doc1 = {"name":"doc1","age":26};
var manydocs = [{"name":"doc2","age":45},{"name":"doc3","age":19}];
//coll.insert(doc1,{w:1},function(err,result){if(err){console.log("Error while inserting doc1 " + err);}else{console.log(result);}});
//coll.insert(manydocs,{w:1},function(err,result){if(err){console.log("Error while inserting manydocs " + err);}});
console.log("***************************documents are now updated successfully***********************************");
console.log("*******************Now let us update the documents*******************");
var query = {"name": "doc1"};
var update= {$set : {"age":86}};
//coll.update(query,update,function(err,result){if(!err){console.log(result + "'s age has been successfully update to " + result);}});
console.log("***************Docments updated");
console.log("*******************Querying the items**************************");
coll.find().each(function(err,myDoc){console.dir(myDoc);console.dir("hey");});
//coll.findOne(query,function(err,result){});
var stream = coll.find(query).stream();
stream.on("data",function(item){console.log(item._id);});
stream.on("end",function(){});
}
else {
console.log("Cannot connect because : " + err);
}
});
Below is the output.
We are Connected!
*********************************inserting documents in the selected collection***********************
***************************documents are now updated successfully***********************************
*******************Now let us update the documents*******************
***************Docments updated
*******************Querying the items**************************
Cannot create Collection because Error: Collection node already exists. Currently in strict mode.
You should work on node collection inside the db.createCollection's callback:
UPDATE: run this code:
var mongo = require("mongodb");
var Db = mongo.Db;
var mongoC = mongo.MongoClient;
var assert = require('assert');
mongoC.connect("mongodb://localhost:27017/social", {
native_parser: true
}, function (err, db) {
if (!err) {
console.log("We are Connected!\n");
//creating the Collection
db.createCollection("node", {
strict: true
}, function (err, coll) {
if (!err) {
console.log("*********Collection Created Succesfully*********");
console.log("Collection created Successfully\n" + JSON.stringify(coll) + "\n-------------------\n");
//Collection created now
console.log("*********************************inserting documents in the selected collection***********************");
var doc1 = {
"name": "doc1",
"age": 26
};
var manydocs = [{
"name": "doc2",
"age": 45
}, {
"name": "doc3",
"age": 19
}];
//coll.insert(doc1,{w:1},function(err,result){if(err){console.log("Error while inserting doc1 " + err);}else{console.log(result);}});
//coll.insert(manydocs,{w:1},function(err,result){if(err){console.log("Error while inserting manydocs " + err);}});
console.log("***************************documents are now updated successfully***********************************");
console.log("*******************Now let us update the documents*******************");
var query = {
"name": "doc1"
};
var update = {
$set: {
"age": 86
}
};
//coll.update(query,update,function(err,result){if(!err){console.log(result + "'s age has been successfully update to " + result);}});
console.log("***************Docments updated");
console.log("*******************Querying the items**************************");
coll.find().each(function (err, myDoc) {
console.dir(myDoc);
console.dir("hey");
});
//coll.findOne(query,function(err,result){});
var stream = coll.find(query).stream();
stream.on("data", function (item) {
console.log(item._id);
});
stream.on("end", function () {});
} else {
console.log("Cannot create Collection because " + err);
}
});
} else {
console.log("Cannot connect because : " + err);
}
});
I am in the market for a new vehicle. Instead of repeatedly searching the dealerships websites, I thought this would be an interesting and fun opportunity to learn a little node and mongodb so I'm scraping my local dealerships' websites to grab the makes and models that I am interested in.
The problem that I am running into is that node won't terminate after my final callback has run through.
var cheerio = require('cheerio');
var request = require('request');
var db = require('mongodb');
var S = require('string');
var log = require('console').log;
var async = require('async');
var links = [];
var website = 'http://www.yahoo.com';
async.series(
[
function(){
log('starting');
db.connect('mongodb://127.0.0.1:27017/test',
function(err, base){
if(err) throw err;
db = base;
});
},
request(website, start)
],
function(){
log('closing DB');
db.close();
});
function start(err,resp,body){
var $ = cheerio.load(body);
var numPages = 2;
$('.gbps').each(function(i,elem) {
links.push('http://www.yahoo.com');
});
var pageURLS = [];
for (var i = 2; i<=numPages; i++){
//create URLs for additional pages
pageURLS[i-2] = website;
}
var pages = 1;
log('getting page URLs');
pageURLS.forEach(function(url, index, array){
request(url, function(error,response,bodies) {
pages++;
var $ = cheerio.load(bodies);
$('.tab').each(function(i,elem) {
links.push('http://www.yahoo.com');
});
if (pages == numPages){
getDetailInfo();
};
});
});
}
function getDetailInfo(){
log(links.length);
links.forEach(function(link, index, array){
request(link, doStuff);
});
}
function doStuff(err, response, body){
if(err){
log(err);
}
parseDetailResponse(err,response,body, addToDB);
}
function parseDetailResponse(err,resp,body,callback){
log('parsing');
var $ = cheerio.load(body);
var specs = $('.specifications').children().map(function(i, elem){
var key = 'key';
var value = 'value';
var ret = {};
ret [ 'name' ] = key;
ret [ 'value' ] = value;
return ret;
});
var makeAndModel = 'makeAndModel';
callback(['picture url', 'vehicle description', 100, specs, makeAndModel]);
}
function getMakeAndModel(stuff){
var $ = cheerio.load(stuff);
temp = $('.gbps').map(function(i, elem){
var ret = {};
switch(i){
case 0:
ret['name'] = 'year';
ret['value'] = $(this).text();
break;
case 1:
ret['name'] = 'make';
ret['value'] = $(this).text();
break;
case 2:
ret['name'] = 'model';
ret['value'] = $(this).text();
break;
case 3:
ret['name'] = 'ignore';
ret['value'] = $(this).text();
break;
default:
ret['name'] = 'ignore';
ret['value'] = 'ignore';
}
return ret;
});
return temp;
}
function addToDB(arr){
log('adding to DB');
pic = arr[0];
description = arr[1];
price = arr[2];
specs = arr[3];
makeAndModel = arr[4];
var obj = {};
for (var i = specs.length - 1; i >= 0; i--) {
obj [specs[i].name] = specs[i].value;
};
for (var i = makeAndModel.length - 1; i >= 0; i--){
obj [makeAndModel[i].name] = makeAndModel[i].value;
};
db.collection('carsTest').update(
{VIN: obj.VIN},
{
$set: {
VIN: obj.VIN,
make: obj.make,
model: obj.model,
year: obj.year,
price: price,
engine: obj.Engine,
interior: obj.Interior,
exterior: obj.Exterior,
'model code': obj['Model Code'],
'stock number': S(obj['Stock Number']).toInt(),
transmission: obj.Transmission,
mileage: obj.Mileage ? obj.Mileage : 0,
description: description,
picture: pic,
}
},
{upsert: true, safe: true},
function(err,result){
if(err){
throw err;
}
});
log('finished with this one!');
}
I've omitted and changed a fair amount as a proof here without a lot of error checking or anything but even this will add the document but won't quit. Node just sits there, waiting for something to happen and it never calls the final callback to close the db and exit.
> db.carsTest.find().pretty()
{
"_id" : ObjectId("52139aa7c9b7a39e0f1eb61d"),
"VIN" : null,
"description" : "vehicle description",
"engine" : null,
"exterior" : null,
"interior" : null,
"make" : null,
"mileage" : 0,
"model" : null,
"model code" : null,
"picture" : "picture url",
"price" : 100,
"stock number" : NaN,
"transmission" : null,
"year" : null
}
I think that you misunderstand how async.series works.
Your functions in async.series don't take callback as an argument and they don't call it. And that request(...) stuff is probably not a function at all. That's probably why it breaks async loop. Try this:
async.series(
[
function(callback) { // <--- missing callback
log('starting');
db.connect('mongodb://127.0.0.1:27017/test',
function(err, base){
if(err) throw err;
db = base;
callback(); // <--- missing callback
});
},
function(callback) { // <--- missing function with callback
request(website, function(err,resp,body) {
start(err, resp, body, callback);
})
}
],
function(){
log('closing DB');
db.close();
}
);
Note that I've added callback argument when calling start. Thus you will have to refactor your code havily so that every function accepts callback which can be called at the end when you know that all jobs are done. For example you can add async.parallel inside start and this function may look like this:
function start(err, resp, body, callback) {
// some stuff happens here
var jobs = []
pageURLS.forEach(function(url, index, array){
jobs.push(function(clb) {
request(url, function(error,response,bodies) {
// some stuff
clb(); // <--- this refers to the local callback for the job
});
});
});
async.parallel(jobs, function() {
// all jobs are done, let's finilize everything
callback();
});
};