hello all I am new to node js and mongo db so I am facing a problem which is written above want to match an id from an array and get all the record of matching id , but thats not working i have already tried for loop but that is taking too much time so i am looking for some kind of query by which I pass id and get the matching results i have below function also where i can pass id and get the matching result but i don't know how to call that .
exports.getDistrictFromId = function(req, res, next) {
var distId = req.params.id;
districtslib.getDistricts({_id: utils.toObjectId(distId)}, function(err, district) {
if (err) {
return next(err);
}
req.store.district = district;
next();
});
};
Here is my code
exports.getCompleteTeachersList = function(req, res, next) {
var query = req.store.get('query');
var teacherQuery = {enabled: true};
var searchQuery = '';
if (!_.isUndefined(query)) {
// schoolQuery = {'name':new RegExp(query.replace(/([.?*+^$[\]\\(){}|-])/g, "\\$1"),'i')};
//{ $text: { $search: "amit hinduja"} }
//teacherQuery = {enabled: true,$or:[{firstName:new RegExp(query.replace(/([.?*+^$[\]\\(){}|-])/g, "\\$1"),'i')},{lastName:new RegExp(query.replace(/([.?*+^$[\]\\(){}|-])/g, "\\$1"),'i')}]};
if(query.trim() != '') {
teacherQuery = {enabled: true,$text: { $search: query} };
searchQuery = query;
}
}
teacherslib.getTeachers(teacherQuery, function(err, teachers) {
if (err) {
return next(err);
}
var schools = req.store.get('schools');
for(var i = 0; i < teachers.length; i++) {
teachers[i].schoolName = "";
for(var j = 0; j < schools.length; j++) {
if (teachers[i].schoolId.toString() === schools[j]._id.toString()) {
teachers[i].schoolName = schools[j].name;
teachers[i].distId = "";
var districts = req.store.get('districts');
console.log(schools[j].distId);
// i want to get the array of matching district id `schools[j].distId` from the district array from `var districts = req.store.get('districts');` this line
break;
}
}
}
req.store.set('searchQuery', searchQuery);
req.store.set('teachers', teachers);
//req.store.set('districts', districts);
next();
});
};
Collection structure is like this
1) distid is coming in schools collection
using distid get all the matching record from district
2)district array has countyid and from that county id has to get data from the county collection
Instead of looping inside a loop, i would suggest you look into the $lookup operator of the aggregation framework. Here you can perform the lookup server side.
Related
I was writing a nest code, tried using async.waterfall or async.series but I am getting random results every time I refresh. it seems to be because of the queries of the first 2 functions randomly finishing.
first query was sorting to committed_date DESC. but when I add the 2 sub queries. the sort gets distorted.
Step1: Loop landingpages
Step1.1 - fetch details1 repositories
Step1.2 - fetch details2 versions
Step2: build array
db.collection('landingpages').find({is_deleted:{$ne:1}}).sort({committed_date:-1}).limit(10).toArray(function(err, db_results) {
var data_array = [];
var x=1;
if(db_results.length == 0) {
return_data.lps = data_array;
parallel_done();
}else{
async.each(db_results, function (db_results1, cb) {
async.watefall(
[
function(callback) {
//if this is removed or passed as callback(null, ""); the sort is fixed from committed - 1
var data_repo = {};
db.collection('repositories').find({repository_id: repository_id}).toArray(function(err, db_results1) {
if(db_results1.length == 0){
var data_repo = {};
callback(null, data_repo);
}else{
var data_repo = db_results1[0];
callback(null, data_repo);
}
});
},
function(callback) {
//if this is removed or passed as callback(null, ""); the sort is fixed from committed - 1
var data_version = {};
db.collection('versions').find({landingpage_id: landingpage_id}).sort({_id:-1}).limit(1).toArray(function(err, db_results1) {
if(db_results1.length == 0){
var data_version = {};
callback(null, data_version);
}else{
var data_version = db_results1[0];
callback(null, data_version);
}
});
}
],
function (err, data_repo,data_version) {
var document = {
"x": x++,
"landingpage_id": db_results1.landingpage_id,
"repository_id": db_results1.repository_id,
"version_id": data_version.version_id,
"val": db_results1,
"data_repo": data_repo,
"data_version": data_version,
};
data_array.push(document);
if(data_array.length == db_results.length) {
return_data.lps = data_array;
}
}
);
});
}
});
I Have to Insert about 10,00000 documents in mongodb using nodejs.
I'm generating these documents using a for loop storing them into an array before finally inserting them into mongodb.
var codeArray = new Array();
for (var i = 0; i<1000000; i++){
var token = strNpm.generate();
var now = moment().format('YYYYMMDD hhmmss');
var doc1 = {id:token,
Discount_strId:"pending",
Promotion_strCode:token,
Promotion_strStatus:"I",
Promotion_dtmGeneratedDate:now,
User_strLogin:"test",
Promotion_strMode:"S",
Promotion_dtmValidFrom:"pending",
Promotion_dtmValidTill:"pending",
LastModified_dtmStamp:now
};
codeArray.push(doc1);
db.collection('ClPromoCodeMaster').insert(codeArray, function (err, result) {
if (err){
console.log(err);
}else{
console.log('Inserted Records - ', result.ops.length);
}
});
The problem I'm facing is mongo has an inserting limit of 16mb, so I can't insert the entire array at once.
Please suggest most optimum solutions.
The main problem is in the request size and not the document size, but it amounts to the same limitation. Bulk operations and the async library with async.whilst will handle this:
var bulk = db.collection('ClPromoCodeMaster').initializeOrderedBulkOp(),
i = 0;
async.whilst(
function() { return i < 1000000; },
function(callback) {
var token = strNpm.generate();
var now = moment().format('YYYYMMDD hhmmss');
var doc = {
id:token,
Discount_strId:"pending",
Promotion_strCode:token,
Promotion_strStatus:"I",
Promotion_dtmGeneratedDate:now,
User_strLogin:"test",
Promotion_strMode:"S",
Promotion_dtmValidFrom:"pending",
Promotion_dtmValidTill:"pending",
LastModified_dtmStamp:now
};
bulk.insert(doc);
i++;
// Drain every 1000
if ( i % 1000 == 0 ) {
bulk.execute(function(err,response){
bulk = db.collection('ClPromoCodeMaster').initializeOrderedBulkOp();
callback(err);
});
} else {
callback();
}
},
function(err) {
if (err) throw err;
console.log("done");
}
);
I should note that regardless there is an internal limit on bulk operations to 1000 operations per batch. You can submit in larger sizes, but the driver is just going to break these up and still submit in batches of 1000.
The 1000 is a good number to stay at though, since it is already in line with how the request will be handled, as well as being a reasonable number of things to hold in memory before draining the request queue and sending to the server.
For inserting millions of record at a time, Create node.js child process fork with MongoDb bulk api.
Child Process Creation:(index.js)
const {fork} = require("child_process");
let counter = 1;
function createProcess(data){
const worker = fork("./dbOperation");
worker.send(data);
worker.on("message", (msg) => {
console.log("Worker Message :",counter, msg);
counter++;
})
}
function bulkSaveUser(records) {
const singleBatchCount = 10000; // Save 10,000 records per hit
const noOfProcess = Math.ceil(records/singleBatchCount);
let data = {};
console.log("No of Process :", noOfProcess);
for(let index = 1; index <= noOfProcess; index++) {
data.startCount = (index == 1) ? index : (((index - 1) * singleBatchCount) + 1);
data.endCount = index * singleBatchCount;
createProcess(data);
}
}
bulkSaveUser(1500000);
DB Operation (dbOperation.js)
const MongoClient = require('mongodb').MongoClient;
// Collection Name
const collectionName = "";
// DB Connection String
const connString = "";
process.on("message", (msg) => {
console.log("Initialize Child Process", msg)
const {startCount, endCount} = msg;
inputStudents(startCount, endCount);
});
function initConnection() {
return new Promise(function(r, e) {
MongoClient.connect(connString, function(err, db) {
if (err) e(err)
r(db);
});
});
}
function inputStudents(startCount, endCount) {
let bulkData = [];
for(let index = startCount; index <= endCount; index++ ){
var types = ['exam', 'quiz', 'homework', 'homework'];
let scores = []
// and each class has 4 grades
for (j = 0; j < 4; j++) {
scores.push({'type':types[j],'score':Math.random()*100});
}
// there are 500 different classes that they can take
class_id = Math.floor(Math.random()*501); // get a class id between 0 and 500
record = {'student_id':index, 'scores':scores, 'class_id':class_id};
bulkData.push({ insertOne : { "document" : record } })
}
initConnection()
.then((db) => {
const studentDb = db.db("student");
const collection = studentDb.collection(colName)
console.log("Bulk Data :", bulkData.length);
collection.bulkWrite(bulkData, function(err, res) {
if (err) throw err;
//console.log("Connected Successfully",res);
process.send("Saved Successfully");
db.close();
});
})
.catch((err) => { console.log("Err :", err) });
}
Sample project to insert millions of record in mongodb using child process fork
This is very strange to me. If put the array onlyIds in the aggregation query of my db I get no results. If I however put the content of onlyIds that I get printed from line 5 which looks like:
["52e953942a13df5be22cf792","52e953942a13df5be22cf793","52e953942a13df5be22cf797"...]
Then it works. But not if I use the variable.
This function:
var onlyIds = [];
for (var i = 0; i < users.length; i++) {
onlyIds.push(users[i]._id);
}
console.log("ids: " + JSON.stringify(onlyIds)); <---------- not empty
db.collection('posts', function(err, collection) {
collection.aggregate([
{$match: {user_id: {$in: onlyIds}}}, <------- not working
{$match: {created:{$gte: 0}}},
{$sort:{"created": -1}},
{$skip: req.body.skip},
{$limit: req.body.limit}
],
function(err, posts) {
var errorNo, content, message;
if (err) {
errorNo = resSend.errorDB;
message = JSON.stringify(err);
} else {
errorNo = resSend.errorNo;
content = posts;
message = "";
--> console.log(JSON.stringify(posts));
}
resSend.sendResponse(res, resSend.errorNo, content, message);
});
});
So in short, why does this work:
{$match: {user_id: {$in: ["52e953942a13df5be22cf792","52e953942a13df5be22cf793","52e953942a13df5be22cf797"...]}}}
and this doesn't:
{$match: {user_id: {$in: onlyIds}}}
And the line that does not work, works perfectly in another function. Any ideas or enlightenments?
EDIT:
Switching to find and using the below answer like this:
collection.find({'user_id': {$in: onlyIdsX}}).toArray(function(err, posts)
does not work either.
ANSWER:
As the selected answer indicates below is when the variable you search for is an ObjectId or a string. For anyone else, make sure that the variable in the db is the same type as the one you try to match it with. In my case both were supposed to be strings, but one's in "onlyIds" were ObjectIds.
Try following code to modify your loop:
var ids = ["52e953942a13df5be22cf792","52e953942a13df5be22cf793","52e953942a13df5be22cf797"];
var obj_ids = [];
for (var i = 0; i < users.length; i++) {
obj_ids.push(new ObjectID(users[i]._id.toString()));
var obj_ids.push(users[i]._id); // <== This will not work if your DB has _id : ObjectID("xyz") [i.e. you are not overiding defaults]
}
And you should include var ObjectID = require('mongodb').ObjectID; into your code.
You should use .toArray(function(err,.. (Not in your case since you used aggregation framework).This will also cause issue if you are not using findOne() (For more info on this here is the link)
Following is the example which spots the issue (in comments) & working Code:
var mongo = require('mongodb'),
Server = mongo.Server,
Db = mongo.Db,
ObjectID = require('mongodb').ObjectID;
var BSON = require('mongodb').BSONPure;
var server = new Server('localhost', 27017, {
auto_reconnect: true
});
var MongoClient = require('mongodb').MongoClient
//let id = your _id, smth like '6dg27sh2sdhsdhs72hsdfs2sfs'...
var users = ["52e953942a13df5be22cf792","52cbd028e9f43a090ca0c1af","52e953942a13df5be22cf797"];
var obj_ids = [];
for (var i = 0; i < users.length; i++) {
obj_ids.push(new ObjectID(users[i].toString()));
//obj_ids.push(users[i]._id); // <== This will not work if your DB has _id : ObjectID("xyz") [i.e. you are not overiding defaults]
}
MongoClient.connect('mongodb://127.0.0.1:27017/YourDBName', function(err, db) {
console.log('err' + err);
db.collection('posts', function(error, collection) {
//collection.find({_id:{$in: users}}),function(err, docs) { //This will not work
collection.find({_id:{$in: obj_ids}}).toArray(function(err, docs) {
console.log("Printing docs from Array. count " + docs.length);
docs.forEach(function(doc) {
console.log("Doc from Array ");
console.dir(doc);
});
});
});
});
I pass a json variable to a module but I can't do the update of my collection, always I have an error in the updating.
var gestion = function(myJSON) {
var dburl = 'localhost/mongoapp';
var collection = ['clientes'];
var db = require('mongojs').connect(dburl, collection );
function cliente(nombre, estado, nuevo){
this.nombre = nombre;
this.estado = estado;
this.nuevo = nuevo;
}
var cliente1 = new cliente(myJSON.nombre myJSON.estado, myJSON.nuevo);
if (cliente1.estado == "desconectado"){
db.clientes.update(cliente1.nombre, {$set: {estado: "desconectado", nuevo: "no"}}, function(err) {
if (err) console.log("error "+cliente1.nombre);
else console.log("OK");
});
}
}
return 0;
}
I also tried to remove my db and create one more time and I'm sure that my object exist in my db.
The signature you should be using is
update(query, update, callback)
but you're passing a string for query, which doesn't mean anything to Mongo. You may want to look at the docs for an overview, but for this specific instance, it looks like you're trying to find the document where nombre is equal to the string at cliente1.nombre. The query for this is a dictionary { nombre: cliente1.nombre }, so that line should be
db.clientes.update({nombre: cliente1.nombre}, {$set: {estado: "desconectado", nuevo: "no"}}, function(err) {
hi am new to nodeJs i have to query a table from the result of the query, i have to query another table. i tried the code as follows but it returns only null.
action(function getpositions(req){
var namearray = [];
NHLPlayerStatsDaily.find ({"created_at": {$gt: new Date(y+"-"+m+"-"+d)}}, function(err,position){
if(!err)
{
for (i=0;i< position.length; i++)
{
var obj = JSON.stringify(position[i]);
var pos = JSON.parse(obj);
var p = pos["player_stats_daily"]["content"]["team_sport_content"]["league_content"]["season_content"]["team_content"]["team"]["id"]
NHLTeam.find({"sdi_team_id": p}, "first_name nick_name short_name sport_id", function(err, team){
if (!err){
var obj = JSON.stringify(team);
var pos = JSON.parse(obj);
namearray.push(team)
}
})
}
return send(namearray);
}
})
})
if i just push "p" it shows the result and when am query "NHLTeam" in separate function it also shows the result. when querying a collection with in collection it return null. how to query a collection within a collection in mongoose. thanks in advance.
This is not a query problem, it is callback issue. The send(namearray) is called before any of the NHLTeam queries in the loop are completed (remember the result of these queries is passed to callback asynchronously).
What you can do is this (basically tracking when all callbacks are completed):
NHLPlayerStatsDaily.find ({"created_at": {$gt: new Date(y+"-"+m+"-"+d)}}, function(err,position){
if(!err)
{
var total = position.length;
for (i=0;i< position.length; i++)
{
var obj = JSON.stringify(position[i]);
var pos = JSON.parse(obj);
var p = pos["player_stats_daily"]["content"]["team_sport_content"]["league_content"]["season_content"]["team_content"]["team"]["id"]
NHLTeam.find({"sdi_team_id": p}, "first_name nick_name short_name sport_id", function(err, team){
total--;
if (!err){
var obj = JSON.stringify(team);
var pos = JSON.parse(obj);
namearray.push(team);
}
if(total == 0)
send(namearray);
})
}
}
})