Node.js MongoDB insertion takes too much time - node.js

I have a certain loop where I'm inserting documents in a MongoDB collection.But only the last document is inserted,may be,the loop is too fast for the insertion.
How can I make the loop wait for the completion of the insertion?
Here is my code:
mongo.connect(url, function(err, db) {
if (err) throw err;
var dbo = db.db("mydb");
for (var i = 0; i < totalEmployee; i++) {
var some = array_data[index].split(":");
var thing = some[1].split("\"");
var EID = thing[1];
var Info = [];
index++;
for (var j = 0; j < 2; j++) {
var some = array_data[index].split(":");
var thing = some[1].split("\"");
var first = thing[1];
thing = some[2].split("\"");
var second = thing[0];
Info[j] = first + ":" + second;
index++;
}
console.log(Info);
var unique = date + EID;
dbo.collection("Attendance").find({_id: unique}).toArray(function (err, result) {
if (!result.length) {
dbo.collection("Attendance").insertOne({
_id: unique,
Date: date,
EID: EID,
InTime: Info[0],
OutTime: Info[1]
},function(err, res) {
if (err) throw err;
console.log("1 document inserted");
inserted=false;
}
);
}
else {
console.log(unique+date+EID+Info[0]);
var myquery = {_id: unique};
var newvalues = {$set: {Date: date, EID: EID, InTime: Info[0], OutTime: Info[1]}};
dbo.collection("Attendance").updateOne(myquery, newvalues, function (err, res) {
if (err) throw err;
console.log("1 document updated");
inserted=false;
});
}
}
}
});

Related

Modify the value of a variable outside callback with the callback inside loop

I am new to Nodejs and I am facing with a problem: Modify the value of a variable outside callback with the callback inside a loop.
I am coding online-judgle project, this is my function to check output of a program with answer from database. I created result object to store amount of correct testcase.
function compareResult(fileName, problem, timeLimit, callback) {
const cp = require('child_process');
const exePath = 'submit\\' + fileName + '.exe';
const child = cp.spawn(exePath, ['--from=markdown', '--to=html'], {timeout: timeLimit});
MongoClient.connect(uri, function(err, db) {
if (err) throw err;
var dbo = db.db(dbName);
var query = { id_problem: problem, is_eg: "false" };
var proj = { projection: {input: 1, output: 1} };
dbo.collection("sample").find(query, proj).toArray(function(err, arr) {
if (err) throw err;
if (arr != null) {
var result = {
correct: 0,
total: arr.length
};
for (const json of arr) {
const answer = json['output'];
child.stdin.write(json['input']);
child.stdout.on('data', function(data) {
if (data == answer) {
result.correct += 1; // I want to modify result object here.
}
});
child.stdin.end();
};
console.log(result);
callback(result);
}
});
});
I want to modify result object in that place. How will I do it?
function compareResult(fileName, problem, timeLimit, callback) {
const cp = require('child_process');
const exePath = 'submit\\' + fileName + '.exe';
const child = cp.spawn(exePath, ['--from=markdown', '--to=html'], {timeout: timeLimit});
MongoClient.connect(uri, function(err, db) {
if (err) throw err;
var dbo = db.db(dbName);
var query = { id_problem: problem, is_eg: "false" };
var proj = { projection: {input: 1, output: 1} };
dbo.collection("sample").find(query, proj).toArray(function(err, arr) {
if (err) throw err;
if (arr != null) {
var result = {
correct: 0,
total: arr.length
};
for (const json of arr) {
const answer = json['output'];
child.stdin.write(json['input']);
child.stdout.on('data', function(data) {
if (data == answer) {
result.correct += 1;
}
// Decrement total here to track how many 'data' events have been emitted
result.total--;
if (result.total === 0) {
// All 'data' events have been emitted, so call the callback function
callback(result);
}
});
child.stdin.end();
};
}
});
});
}

display 2 two table records in nodejs

I have tow MongoDB collection.
1) users
2) reviews.
at the first collection, I have stored username and id. and 2nd table I have stored user_id and comments and star rating.
I want to display on listing page users and his reviews.
I have written below code but it is not working.
var getTopSellers = function () {
var defer = Q.defer();
User.find({ isRegistered: true }).sort({ updatedAt: -1 }).limit(10).exec(function (err, sellers) {
if (!err) {
if (sellers && sellers.length > 0) {
for (var i = 0; i < sellers.length; i++) {
var sellerDetails = {};
var tempObj = {};
try {
tempObj.reviews = getreviews(sellers[i]._id);
sellerArr.push(tempObj);
} catch (e) {
// console.log("catch error:-", e);
}
}
out = U.getJson(C.SUCCESS_CODE, C.SUCCESS, sellerArr);
defer.resolve(out);
} else {
out = U.getJson(C.KNOWN_ERROR_CODE, 'No data found');
defer.reject(out);
}
} else {
console.log("Error:-", err);
out = U.getJson(C.ERROR_CODE, C.INETRNAL_SERVER_ERROR, b, err);
defer.reject(out);
}
})
return defer.promise;
};
var getreviews = function (user_id) {
var defer = Q.defer();
Review.find({ user_type: user_id }).sort({ updatedAt: -1 }).limit(10).exec(function (err, reviews) {
if (!err) {
if (reviews && reviews.length > 0) {
out = U.getJson(C.SUCCESS_CODE, C.SUCCESS, reviews);
defer.resolve(out);
} else {
out = U.getJson(C.KNOWN_ERROR_CODE, 'No data found');
defer.reject(out);
}
} else {
console.log("Error:-", err);
out = U.getJson(C.ERROR_CODE, C.INETRNAL_SERVER_ERROR, b, err);
defer.reject(out);
}
})
return defer.promise;
};
Please suggest

nodejs + Mongodb: Inserting into two collections in sequence repeats last value in second

I am using the following to insert into MongoDB.
var tagData = JSON.parse(data);
var allTags = tagData.tags;
for (var j = 0; j < allTags.length; j++) {
var p = allTags[j].tagId.toString();
for (var k = 0; k < loggerParams.length; k++) {
var q = Object.keys(loggerParams[k]).toString();
if (p === q) {
// Prepare raw data tag
var tagRawDoc = {};
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
database.addDocument('tagraw', tagRawDoc, function (err) {
if (err) {
log.info(util.format('Error adding document to tagrawdatas. %s', err.message));
throw err;
} else {
// Prepare history tag
var historyTagDoc = {};
historyTagDoc.tagNameAlias = tagRawDoc.tagNameAlias;
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
database.addDocument('taghistory', historyTagDoc, function (err) {
if (err) {
log.info(util.format('Error adding document to tagrawdatas. %s', err.message));
throw err;
}
});
}
});
// Match found; exit loop
break;
}
}
}
The loggerParms is a simple JSON document read from file else-where. It allows for look-up in this code to build the document to be inserted. There will be 12 values in the allTags array. These 12 values are inserted successfully into the tagraw collection. However, in taghistory collection, the values from the last (or most recent) entry made into tagraw collection is repeated 12 times. Why does this happen?
The database.addDocument is shown below. It is a part of this article I am trying to replicate.
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var logger = require('../../util/logger');
var util = require('util');
function DB() {
this.db = "empty";
this.log = logger().getLogger('mongoMange-DB');
}
DB.prototype.connect = function(uri, callback) {
this.log.info(util.format('About to connect to DB'));
if (this.db != "empty") {
callback();
this.log.info('Already connected to database.');
} else {
var _this = this;
MongoClient.connect(uri, function(err, database) {
if (err) {
_this.log.info(util.format('Error connecting to DB: %s', err.message));
callback(err);
} else {
_this.db = database;
_this.log.info(util.format('Connected to database.'));
callback();
}
})
}
}
DB.prototype.close = function(callback) {
log.info('Closing database');
this.db.close();
this.log.info('Closed database');
callback();
}
DB.prototype.addDocument = function(coll, doc, callback) {
var collection = this.db.collection(coll);
var _this = this;
collection.insertOne(doc, function(err, result) {
if (err) {
_this.log.info(util.format('Error inserting document: %s', err.message));
callback(err.message);
} else {
_this.log.info(util.format('Inserted document into %s collection.', coll));
callback();
}
});
};
module.exports = DB;
That's because you are mixing a/multiple synchronous for and asynchronous code with database.addDocument which cause issues with function scope in nodejs.
A simple example of this kind of thing:
for(var i = 0; i < 10; i++){
setTimeout(() => console.log(i), 0);
}
You should use a package like async to handle flow control when iterating arrays/object asynchronously.
Simple example of your code refactored to use async:
var async = require('async');
var tagData = JSON.parse(data);
var allTags = tagData.tags;
async.each(allTags, function(tag, done){
var p = tag.tagId.toString();
var loggerParam = loggerParams.find(function(loggerParam){
var q = Object.keys(loggerParam).toString();
return p === q;
});
var tagRawDoc = {};
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
return database.addDocument('tagraw', tagRawDoc, function (err){
if (err) return done(err);
// Prepare history tag
var historyTagDoc = {};
historyTagDoc.tagNameAlias = tagRawDoc.tagNameAlias;
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
return database.addDocument('taghistory', historyTagDoc, done);
});
}, (err) => {
if(err) throw err;
console.log('All done');
});

cursor has no method next

I'm trying to traverse a collection and update an array for each document.
What am I doing wrong below?
var MongoClient = require('mongodb').MongoClient;
var removeLowestHWScore = function(scores) {
var lowestHWID = -1;
for(var i=0;i<scores.length; i++) {
if (scores[i].type === 'homework') {
if (lowestHWID === -1) {
lowestHWID = i;
} else if (scores[i].score < scores[lowestHWID].score) {
lowestHWID = i;
}
}
}
scores.splice(lowestHWID);
return scores;
};
var callback = function(err, r) {
console.log('updated record');
};
var updateScore = function(err, doc) {
var updatedScores = removeLowestHWScore(doc.scores);
collection.updateOne({_id:doc._id},
{$set: {scores: updatedScores }},
callback);
};
MongoClient.connect('mongodb://localhost:27017/school', function(err, db) {
if(err) throw err;
var collection = db.collection('students');
var cursor = collection.find({});
var next = cursor.next();
while (next) {
next(updateScore);
next = cursor.next();
}
db.close();
});
error
/Users/harrymoreno/programming/mongodb/mongo101ForNode/node_modules/mongodb
/lib/mongodb/mongo_client.js:475
throw err
^
TypeError: Object #<Cursor> has no method 'next'
at /Users/harrymoreno/programming/mongodb/mongo101ForNode/week03/app.js:35:21
at /Users/harrymoreno/programming/mongodb/mongo101ForNode/node_modules/mongodb/lib/mongodb/mongo_client.js:472:11
at process._tickCallback (node.js:419:13)
sample student
{
"_id" : 137,
"name" : "Tamika Schildgen",
"scores" : [
{
"type" : "exam",
"score" : 4.433956226109692
},
{
"type" : "quiz",
"score" : 65.50313785402548
},
{
"type" : "homework",
"score" : 89.5950384993947
}
]
}
UPDATED - v.2
According to the information provided in your last remark abut mongodb package version, I've changed to solution to the one you've improved for specific version compliance (using the 1.4.x node.js mongodb driver) :
var MongoClient = require('mongodb').MongoClient;
var cursor = null,
collection = null,
dbSrc = null;
var removeLowestHWScore = function(scores) {
var lowestHWID = -1;
for(var i=0;i<scores.length; i++) {
if (scores[i].type === 'homework') {
if (lowestHWID === -1) {
lowestHWID = i;
} else if (scores[i].score < scores[lowestHWID].score) {
lowestHWID = i;
}
}
}
// scores.splice(lowestHWID);
if (lowestHWID >= 0)
scores.splice(lowestHWID, 1);
return scores;
};
var callback = function(err, r) {
if (err) throw err;
console.log('updated record');
cursor.nextObject(updateScore);
};
var updateScore = function(err, doc) {
if (err) throw err;
if (doc === null)
return dbSrc.close();
var updatedScores = removeLowestHWScore(doc.scores);
collection.update({_id:doc._id},
{$set: {scores: updatedScores }},
callback);
};
MongoClient.connect('mongodb://localhost:27017/school', function(err, db) {
if(err) throw err;
dbSrc = db;
collection = db.collection('students');
cursor = collection.find({});
cursor.nextObject(updateScore);
});

Nodejs MongoDB updating documents with asynchronous function

I'm new to NodeJS and MongoDB. Is there a way to update a document using asynchronous functions in nodejs? I have a collection of websites with documents: _id, name and registrant. I want to insert a value in registrar using an asynchronous function. The function is a system call, and I just parse the output that I want. Running the code, the async function logs the information I want but does not store in the database. Any suggestions on how I can solve this? Thanks.
/*
* { _id: 53448014b15c693931000002,
* name: 'google.com',
* registrant: 'defaultval' } */
var MongoClient = require('mongodb').MongoClient;
var ObjectID = require('mongodb').ObjectID;
var id = '53448014b15c693931000002';
var domain = 'google.com';
MongoClient.connect('mongodb://127.0.0.1:27017/mydb', function(err, db) {
if (err) throw err;
var collection = db.collection('websites');
collection.findAndModify({_id:new ObjectID(id)},
{},
{$set: {registrant: test(domain, function (output) {
var t = output.split("\n");
for (var i = 0; i < t.length; ++i) {
if (t[i].indexOf("Registrant Organization:") != -1) {
console.log(t[i].substring(t[i].indexOf(":") + 2, t[i].length));//prints correct value, need this to store in registrant doc
return t[i].substring(t[i].indexOf(":") + 2, t[i].length);
}
}
})}},
{},
function(err, object) {
if (err) console.log(err.message);
else {
console.log(object);
}
db.close();
});
});
var test = function(domain, cb) {
var sys = require('sys');
var exec = require('child_process').exec;
var child = exec('whois ' + domain, function(error, stdout, stderr) {
cb(stdout);
});
}
Start one process on core:
var cluster = require('cluster')
, numCPUs = require('os').cpus().length
, windows = require('os').platform() == 'win32';
if(cluster.isMaster) {
// Fork workers.
for (var i = 0; i < numCPUs; i++) {
cluster.fork();
}
cluster.on('exit', function(worker, code, signal) {
console.log('worker ' + worker.pid + ' died');
});
return;
}
Create function to be async:
function mongodbDriverQuery(callback) {
collection.findOne({ id: getRandomNumber()}, function(err, world) {
callback(err, world);
});
}
http.createServer(function (req, res) {
// JSON response object
var hello = {message: "Hello, World!"};
var helloStr = "Hello, World!";
var path = url.parse(req.url).pathname;
// mysql on windows is not supported
if (windows && (path.substr(0, 3) == '/my' || path == '/update')) {
path = '/doesntexist';
}
switch (path) {
case '/mongodbdriver':
// Database Test
var values = url.parse(req.url, true);
var queries = values.query.queries || 1;
var queryFunctions = new Array(queries);
for (var i = 0; i < queries; i += 1) {
queryFunctions[i] = mongodbDriverQuery;
}
res.writeHead(200, {'Content-Type': 'application/json; charset=UTF-8'});
// and run it
async.parallel(queryFunctions, function(err, results) {
if (queries == 1) {
results = results[0];
}
res.end(JSON.stringify(results));
});
break;

Resources