I need to perform complicated aggregation on Mongo DB collection. I also need to go through the results twice. After first pass, I execute rewind() on a cursor. Then I try to perform second pass. This is where I get the error. What is strange that if I replace each(), with couple of next(). Then everything works as expected.
Is it a bug in each(), which I should submit to MongoDB bugtracker? Or it is some sort of my error?
Much simplified sample, but still reproduce the error:
var MongoClient = require('mongodb').MongoClient;
var ObjectId = require('mongodb').ObjectId;
MongoClient.connect('mongodb://localhost:27017/test', function(err, db) {
// Create a collection
let collection = db.collection('aggregation_each_example');
// collection.drop();
// Some docs for insertion
function createData(callback) {
let docs = [
{
"oid" : new ObjectId("59883f2e7d8c6325200b81e4"),
"tod" : new Date("2017-08-07T10:21:34.877Z")
},
{
"veryImportanta" : "newOne",
"oid" : new ObjectId("59883f2e7d8c6325200b81e4")
}
];
// Insert the docs
collection.insertMany(docs, callback);
}
function getData(callback) {
return function(err) {
if (err) {
console.error(err);
} else {
let cursor = collection.aggregate([
{
"$match": {
"oid": new ObjectId("59883f2e7d8c6325200b81e4"),
"tod": {
"$exists": 0
}
}
}
], {cursor: {batchSize: 1}});
let count = 0;
cursor.each(function(err, doc) {
if(doc) {
console.log(doc);
count++;
} else {
console.log(cursor.isClosed());
cursor.rewind();
console.log(cursor.isClosed());
callback(count, cursor);
}
});
}
}
}
createData(getData(function(count, cursor) {
console.log("Count: "+ count);
console.log("Cursor is closed: " + cursor.isClosed());
cursor.next(function(err, doc) {
if (err) console.log(err);
else console.log(doc);
// db.dropDatabase();
db.close();
});
}));
});
Output:
{ _id: 598851ad48a1841c18b50bcf,
veryImportanta: 'newOne',
oid: 59883f2e7d8c6325200b81e4 }
true
false
Count: 1
Cursor is closed: false
{ MongoError: Cursor is closed
at Function.MongoError.create (error.js:31:11)
at nextObject (node_modules\mongodb\lib\cursor.js:669:112)
at AggregationCursor.Cursor.next (node_modules\mongodb\lib\cursor.js:269:12)
at error.js:61:12
at error.js:50:13
at handleCallback (node_modules\mongodb\lib\utils.js:120:56)
at node_modules\mongodb\lib\cursor.js:748:16
at handleCallback (node_modules\mongodb\lib\utils.js:120:56)
at node_modules\mongodb\lib\cursor.js:682:5
at handleCallback (node_modules\mongodb-core\lib\cursor.js:171:5) name: 'MongoError', message: 'Cursor is closed', driver: true }
Environment:
nodejs: v6.6.0
mongodb: 2.2.30
os: windows 10
mongodb engine: 3.4
Related
I am new to mongoDb, i am trying to update the fields for Each Records for around 10 to 15k records. When i am trying to update the records the query blocks whole database, the Running query will allow me to do any read or write operations till the Query Execution completes, is there anyway to improve the performance for this kind of Queries.
Here is My Code:
var ExisitingData=[{"isActive" : true,
"barcode" : "8908001921015",
"mrp" : 2000,
},
{"isActive" : true,
"barcode" : "7808001921019",
"mrp" : 1000,
}
....15k]
var updatedRsult=[];
async.forEach(ExisistingData, function (item, innerCallback) {
exports.populateData(item, function (err, populatedResult) {
if (err) {
innerCallback(err);
}
if(populatedResult==true)
totalRecordsUpdated++;
else
totalIgnoredRecords++;
innerCallback();
});
}, function (err) {
console.log("FinalDone");
var h1={}
h1['totalRecordsUpdated'] = totalRecordsUpdated;
h1['totalIgnoredRecords'] = totalIgnoredRecords;
updatedResult.push(h1);
updateCheck(null, updatedResult);
});
exports.populateData=function(item, mainCallback) {
var updated = false;
async.parallel([
function (callback1) {
Test.update({
$and: [
{'barcode': item['barcode']},
{'mrp': {$lt: parseInt(item['mrp'])}}
]
}, {$set: {'mrp': parseInt(item['mrp'])}}, function (err, result) {
if (err) {
console.log(err);
callback1();
}
else {
if (result['nModified'] == 1) {
console.log("Its Updated");
console.log(item);
updated=true;
callback1()
}
else {
callback1()
}
}
});
}
], function done(err) {
mainCallback(null,updated);
});
};
I have this function, which search for a document with a given ID (mID), check if another field exist (u) and if not, add the given id (uID).
The code doesn't throw any error, but doesn't update the document either, however, building the query myself from the same field (the two console.log) and using them in the mongo shell does work.
The result of the query when used with node-mongodb-native is { ok: 1, nModified: 0, n: 0 }
'use strict';
const MongoClient = require('mongodb').MongoClient,
async = require('async'),
uuid = require('node-uuid'),
winston = require('winston'),
logger = new (winston.Logger)({transports: [new winston.transports.Console()]});
let Link = {};
function link(mId, base, uId, callback) {
let filter = {'mId': mId},
update = {'$set': {}};
filter[base] = {'$exists': false};
update['$set'][base] = uId;
logger.info('link update', {filter: filter, update: update});
console.log('db.Link.findOne(' + require('util').inspect(filter) + ');');
console.log('db.Link.findOneAndUpdate(' + require('util').inspect(filter) + ', ' + require('util').inspect(update) + ', {upsert: false, new: true});');
Link.collection('link')
.findOneAndUpdate(
filter,
update,
{
upsert: false,
returnOriginal: false
}
).then((result) => {
logger.info('link update ok', {result: result});
callback();
}).catch((error) => {
logger.error('link update error', {error: error});
callback(new Error(4299));
});
}
async.waterfall([
(callback) => {
MongoClient.connect('mongodb://127.0.0.1/Link').then((db) => {
logger.info('Connected to Link');
Link = db;
callback(null);
}).catch((error) => {
logger.error('init Link', {error: error});
callback(error);
});
},
(callback) => {
let mId = uuid.v4();
logger.info('create');
Link.collection('Link')
.insertOne({'mId': mId})
.then((error) => {
logger.info('create ok')
callback(null, mId);
}).catch((error) => {
logger.error('create error', {error: error});
callback(new Error(4299));
});
},
(mId, callback) => {
link(mId, 'link', uuid.v4(), callback);
}
], (error) => {
if(error) {
logger.error('End', {error, error});
}
logger.info('End');
Link.close();
});
I also tried with update and updateOne functions, but same result: command work, not the code.
Can anybody explain why a command that work from the shell couldn't work when made from the driver, and why Mongo report it found a document, but don't update it?
node v6.9.1, node-mongodb-native v2.2.11
Edit:
Base document:
{
"_id" : ObjectId("58332c30224fe3273c7b1ba6"),
"mId" : "37286c83-7d81-484d-b62a-310f690cac97"
}
Updated document:
{
"_id" : ObjectId("58332c30224fe3273c7b1ba6"),
"mId" : "37286c83-7d81-484d-b62a-310f690cac97",
"test" : "f7bb9386-eedd-43fe-890a-348cb3a97ed3"
}
Logger output:
info: Connected to Link
info: create
info: create ok
info: link update mId=f8ba93da-3b6d-43f7-9f90-4e345ba04131, $exists=false, link=f882d44d-60a3-4701-b5df-ba493c3b249b
db.Link.findOne({ mId: 'f8ba93da-3b6d-43f7-9f90-4e345ba04131',
link: { '$exists': false } });
db.Link.findOneAndUpdate({ mId: 'f8ba93da-3b6d-43f7-9f90-4e345ba04131',
link: { '$exists': false } }, { '$set': { link: 'f882d44d-60a3-4701-b5df-ba493c3b249b' } }, {upsert: false, new: true});
info: link update ok updatedExisting=false, n=0, value=null, ok=1
info: End
The init function connect to mongoDB, the create function insert into it a new document with a random mId that is passed to the link function.
The uId is also created randomly and is also a UUID.
While it should be equivalent, the command printed in the console logs:
db.Link.findOneAndUpdate({ mId: 'f8ba93da-3b6d-43f7-9f90-4e345ba04131',
link: { '$exists': false } }, { '$set': { link: 'f882d44d-60a3-4701-b5df-ba493c3b249b' } }, {upsert: false});
do update the document
Well, you're using wrong Collection Name
When you insert one you use uppercase Link
Link.collection('Link') <<<<<<<<<<<<<<<<<<<<
.insertOne({'mId': mId})
When you try to update you are using lowercase link
Link.collection('link') <<<<<<<<<<<<<<<<<
.findOneAndUpdate(
I want to have exactly auto-increment field like relational or objective databases, so i need an integer _id field with automatically set field value, value should be one more last record _id value like this:
data:
{_id:1,name"foo"}
{_id:2,name"bar"}
remove last record:
{_id:1,name"foo"}
add new record:
{_id:1,name"foo"}
{_id:3,name"newbar"}
I added a function to my datastore and calculate maximum of _id and plus 1 max(_id)+1 and set as field value, but there is problem here:
When we use auto-increment field in relational databases, it works like i said and after you remove last record it reserved a deleted record number and new inserted records continue increment but in my way its says the _id of removed record for new record.
My code is:
var Datastore = require('nedb'),
localDb = new Datastore({
filename: __dirname + '/dbFilePath.db',
autoload: true
});
localDb.getMax = function(fieldName, onFind){
db.find({}).sort({_id:-1}).limit(1).exec(function (err, docs) {onFind && onFind(err, docs['_id']);});
return localDb;
}
localDb.insertAutoId = function(data, onAdd){
var newIndex = 0;
localDb.getMax(function (err, maxValue) {
newIndex = maxValue+1;
if(!data["_id"])
data["_id"] = newIndex;
localDb.insert(data, function (err, newDoc) {
onAdd && onAdd(err, newDoc);
});
});
return localDb;
}
An improved answer for nedb would be:
db.getAutoincrementId = function (cb) {
this.update(
{ _id: '__autoid__' },
{ $inc: { seq: 1 } },
{ upsert: true, returnUpdatedDocs: true },
function (err, affected, autoid) {
cb && cb(err, autoid.seq);
}
);
return this;
};
Which is equivalent to the mongodb way:
db.getAutoincrementId = function (cb) {
this.findAndModify({
query: { _id: '__autoid__' },
update: { $inc: { seq: 1 } },
new: true
}
function (err, autoid) {
cb && cb(err, autoid.seq);
}
);
return this;
};
You can store the last value of the index in the database. Something like this:
var Datastore = require('nedb');
var db = new Datastore({
filename: __dirname + '/dbFilePath.db',
autoload: true
});
// Initialize the initial index value
// (if it already exists in the database, it is not overwritten)
db.insert({_id: '__autoid__', value: -1});
db.getAutoId = function(onFind) {
db.findOne( { _id: '__autoid__' }, function(err, doc) {
if (err) {
onFind && onFind(err)
} else {
// Update and returns the index value
db.update({ _id: '__autoid__'}, { $set: {value: ++doc.value} }, {},
function(err, count) {
onFind && onFind(err, doc.value);
});
}
});
return db;
}
I do not know if it will be useful for you anymore I use a database to store the next ids, inspired in the mysql system. Who always reserves the next id.
So I created a function that verifies if there is an id to the db, if it does not, it add with the value "1", and when it updates it looks for and if it exists and it performs the sequence.
This gave me full control over my ids.
The schema would be:
{
name: nameDb,
nextId: itemID
}
If you want you can create functions for updating documents, versioning, etc.
example:
db.autoincrement = new Datastore({filename: 'data/autoincrement.db', autoload: true});
function getUniqueId(nameDb, cb) {
db.autoincrement.findOne({name: nameDb}, function (err, doc) {
if (err) {
throw err;
} else {
if (doc) {
const itemID = doc.nextId + 1;
db.autoincrement.update({name: nameDb}, {
name: nameDb,
nextId: itemID
}, {}, function (err, numReplaced) {
db.autoincrement.persistence.compactDatafile();
if (err) {
throw err;
} else {
// console.log(numReplaced);
}
cb(doc.nextId);
});
} else {
const data = {
name: nameDb,
nextId: 2
};
db.autoincrement.insert(data, function (err, newDoc) {
if (err) {
throw err;
} else {
// console.log(newDoc);
}
cb(1);
});
}
}
});
}
insert new document example:
function insert(req, cb) {
getUniqueId("testdb", function (uniqueId) {
data.itemId = uniqueId;
db.testdb.insert(data, function (err, newDoc) {
if (err) {
cb({error: '1', message: 'error#2'});
throw err;
}
cb({error: '0', message: 'Item add'});
});
});
}
I am using MongoDb Driver for NodeJS.
I am facing issues in using aggregation.
The error is
{"name":"MongoError","message":"unknown group operator
'_id'","ok":0,"errmsg":"unknown group operator '_id'","code":15952}
for the below script :
MongoClient.connect(url, function (err, db) {
if (err)
{
console.log('Unable to connect to the mongoDB server. Error:', err);
return;
}
var collName = "order";
var whereParas = {};
var groupParas = {"_id":null,total:{$sum:"$Value"}};
var havingParas = {};
db.collection(collName).aggregate(
[
{ $match: whereParas },
{
$group: { groupParas}
},
{ $match: havingParas }
]).toArray(function (err,result) {
console.log("err");
console.log(err);
console.log("result");
console.log(result);
});
});
The data used is
Desired Output is Sum of Values.
In sql, I would have written :
Select Sum(Value) From order
The group pipeline should be { $group: groupParas } instead of { $group: { groupParas } } hence the error you are getting as Mongo is trying to interpret the nested document win the object as the _id group operator.
I'm writing a webservice using Node.js.
This webservice makes calls to a MongoDB and MSSQL.
For MSSQL i've use npm mssql library, for mongo i use the native npm mongodb library.
I use Q as my promise library.
I've a memory leak issue running a find over a MongoDB collection. I simply need to get elements from a connection. And eventually update the status of elements i get.
See my sample code below.
var Q = require('q');
var connection = require('..\connection.js'); //the connection module open a connection that can be used with pools.
function list(req, res) {
return Q.Promise(function(resolve, reject, notify) {
var collection = null;
var result = [];
var cursor = null;
Q.fcall(function(){}).then(function() {
collection = connection.collection(collectionName);
})
.then(function() {
cursor = collection.find({ fieldstatus : 0 });
return Q.Promise(function(resolve, reject, notify) {
Q.allSettled(cursor.each(function(err, item){
return Q.fcall(function(){
try {
if(item != null) {
result.push({
field1 : item.field1,
field2 : item.field2,
fieldstatus : item.fieldstatus
});
collection.update({_id: item._id}, {$set: {fieldstatus : 1}});
}
resolve(result);
} catch (err){
reject(err);
}
})
.fin(function() {
cursor.close();
});
}));
});
})
.then(function(ret) {
resolve(ret);
})
.fail(function(err) {
reject([ err.toString() ]);
})
.fin(function() {
result = null;
cursor = null;
collection = null;
});
})
.fail(function(err) {
throw([ err.toString() ]);
});
}
}
UPDATED : Answer 1
The code below seems to works without leak issuess.
var Q = require('q');
var connection = require('..\connection.js'); //the connection module open a connection that can be used with pools.
function list(req, res) {
return Q.Promise(function(resolve, reject, notify) {
var collection = null;
var result = [];
var cursor = null;
Q.fcall(function(){}).then(function() {
collection = connection.collection(collectionName);
})
.then(function() {
return Q.npost(
collection,
"find",
[
{ fieldstatus : 0 }
]
).then(function(ret){
return Q.npost(ret, "toArray").then(function(item){
return item;
});
})
.then(function(ret){
var result = [];
ret.forEach(function (item) {
result.push({
field1 : item.field1,
field2 : item.field2,
fieldstatus : item.fieldstatus
});
collection.update({_id: item._id}, {$set: {fieldstatus : 1}});
});
return result;
})
})
.then(function(ret) {
resolve(ret);
})
.fail(function(err) {
reject([ err.toString() ]);
})
.fin(function() {
collection = null;
});
})
.fail(function(err) {
throw([ err.toString() ]);
});
}
}