Related
I am new to Mongodb
Here is what i have tried. however this method worked for me but its way slow.
Is it possible to design single query to get all this data. instead of querying one by one.
var Schema = mongoose.Schema;// Schema
var dataModelSchema = new Schema({
data: String,
});
var fdevices = mongoose.model('fdevices', dataModelSchema);
var DPMPow = fdevices.findOne({ "Data": { $regex: /DPMPow_P01/i } }).sort({
field: 'asc', _id: -1 }); //Active Power
var DPMAmp = fdevices.findOne({ "Data": { $regex: /DPMAmp_P01/i } }).sort({
field: 'asc', _id: -1 });//Total Current
var DPMkWh = fdevices.findOne({ "Data": { $regex: /DPMkWh_P01/i } }).sort({
field: 'asc', _id: -1 });//Consumption
var MxPeak = fdevices.findOne({ "Data": { $regex: /MxPeak_P01/i } }).sort({
field: 'asc', _id: -1 });//Max Demand Peak
var MPPeak = fdevices.findOne({ "Data": { $regex: /MPPeak_P01/i } }).sort({
field: 'asc', _id: -1 });//Max Demand Partial
async.series([
function (callback) {
DPMPow.exec(function (err, res) {
callback(null, res);
})
},
function (callback) {
DPMAmp.exec(function (err, res) {
callback(null, res);
});
},
function (callback) {
DPMkWh.exec(function (err, res) {
callback(null, res);
})
},
function (callback) {
MxPeak.exec(function (err, res) {
callback(null, res);
})
}, function (callback) {
MPPeak.exec(function (err, res) {
callback(null, res);
})
//chiller1
}
], function (err, results) {
console.log(results);
});
this is how my data looks like in database. in collection got only one field which name as 'Data' this field receives multiple sensors data in every one hour.
{ _id: 5b2118c4ae33249711a498a6,
Data: '13-06-2018 20:14:46 DPMPow_P01_04_ 699.15 ',
__v: 0 },
{ _id: 5b2118c4ae35439711a498a6,
Data: '13-06-2018 20:14:46 MPPeak_P01_04_ 699.15 ',
__v: 0 },
{ _id: 5b23428c4ae32989711a498a6,
Data: '13-06-2018 20:14:46 DPMAmp_P01_04_ 699.15 ',
__v: 0 },
{ _id: 5b21234c4ae32989711a498a6,
Data: '13-06-2018 20:14:46 DPMkWh_P01_04_ 699.15 ',
__v: 0 },
{ _id: 5b2118c4ae34989711a598a6,
Data: '13-06-2018 20:14:46 MxPeak_P01_04_ 699.15 ',
__v: 0 }
You can try below RegEx in find query
let data = fdevices.find({ "Data": { $regex: /(DPMPow)|(MPPeak)|(DPMAmp)|(DPMkWh)|
(MxPeak)/gm } }).sort({ field: 'asc', _id: -1 });
I'm using Mongoose (MongoDB in node.js), and after reading this answer:
Replace value in array
I have another question:
Is it possible to do in the same sentence: push element into array or replace if this element is existing in the array?
Maybe something like this? (The example doesn't work)
Model.findByIdAndUpdate(id,
{
$pull: {"readers": {user: req.user.id}},
$push:{"readers":{user: req.user.id, someData: data}}
},{multi:true},callback)
Message error:
errmsg: 'exception: Cannot update \'readers\' and \'readers\' at the same time
Reference:
https://stackoverflow.com/a/15975515/4467741
Thank you!
Multiple operations on the same property path are simply not allowed in a single request, with the main reason being that the operations themselves have "no particular order" in the way the engine assigns them as the document is updated, and therefore there is a conflict that should be reported as an error.
So the basic abstraction on this is that you have "two" update operations to perform, being one to "replace" the element where it exists, and the other to "push" the new element where it does not exist.
The best way to implement this is using "Bulk" operations, which whilst still "technically" is "two" update operations, it is however just a "single" request and response, no matter which condition was met:
var bulk = Model.collection.initializeOrderedBulkOp();
bulk.find({ "_id": id, "readers.user": req.user.id }).updateOne({
"$set": { "readers.$.someData": data } }
});
bulk.find({ "_id": id, "readers.user": { "$ne": req.user.id } }).updateOne({
"$push": { "readers": { "user": req.user.id, "someData": data } }
});
bulk.execute(function(err,result) {
// deal with result here
});
If you really "need" the updated object in result, then this truly becomes a "possible" multiple request following the logic where the array element was not found:
Model.findOneAndUpdate(
{ "_id": id, "readers.user": req.user.id },
{ "$set": { "readers.$.someData": data } },
{ "new": true },
function(err,doc) {
if (err) // handle error;
if (!doc) {
Model.findOneAndUpdate(
{ "_id": id, "readers.user": { "$ne": req.user.id } },
{ "$push": { "readers":{ "user": req.user.id, "someData": data } } },
{ "new": true },
function(err,doc) {
// or return here when the first did not match
}
);
} else {
// was updated on first try, respond
}
}
);
And again using you preferred method of not nesting callbacks with either something like async or nested promise results of some description, to avoid the basic indent creep that is inherrent to one action being dependant on the result of another.
Basically probably a lot more efficient to perform the updates in "Bulk" and then "fetch" the data afterwards if you really need it.
Complete Listing
var async = require('async'),
mongoose = require('mongoose')
Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/test');
var userSchema = new Schema({
name: String
});
var dataSchema = new Schema({
user: { type: Schema.Types.ObjectId, ref: 'User' },
someData: String
},{ "_id": false });
var testSchema = new Schema({
name: String,
readers: [dataSchema]
});
var User = mongoose.model( 'User', userSchema ),
Test = mongoose.model( 'Test', testSchema );
var userId = null,
id = null;
async.series(
[
// Clean models
function(callback) {
async.each([User,Test],function(model,callback) {
model.remove({},callback);
},callback);
},
// Create a user
function(callback) {
User.create({ name: 'bill' },function(err,user) {
userId = user._id;
callback(err);
});
},
function(callback) {
Test.create({ name: 'Topic' },function(err,topic) {
id = topic._id;
console.log("initial state:");
console.log(topic);
callback(err);
});
},
// 1st insert array 2nd update match 1 modified
function(callback) {
var bulk = Test.collection.initializeOrderedBulkOp();
bulk.find({ "_id": id, "readers.user": userId }).updateOne({
"$set": { "readers.$.someData": 1 }
});
bulk.find({ "_id": id, "readers.user": { "$ne": userId }}).updateOne({
"$push": { "readers": { "user": userId, "someData": 1 } }
});
bulk.execute(function(err,result) {
if (err) callback(err);
console.log("update 1:");
console.log(JSON.stringify( result, undefined, 2));
Test.findById(id,function(err,doc) {
console.log(doc);
callback(err);
});
});
},
// 2nd replace array 1st update match 1 modified
function(callback) {
var bulk = Test.collection.initializeOrderedBulkOp();
bulk.find({ "_id": id, "readers.user": userId }).updateOne({
"$set": { "readers.$.someData": 2 }
});
bulk.find({ "_id": id, "readers.user": { "$ne": userId }}).updateOne({
"$push": { "readers": { "user": userId, "someData": 2 } }
});
bulk.execute(function(err,result) {
if (err) callback(err);
console.log("update 2:");
console.log(JSON.stringify( result, undefined, 2));
Test.findById(id,function(err,doc) {
console.log(doc);
callback(err);
});
});
},
// clear array
function(callback) {
Test.findByIdAndUpdate(id,
{ "$pull": { "readers": {} } },
{ "new": true },
function(err,doc) {
console.log('cleared:');
console.log(doc);
callback(err);
}
);
},
// cascade 1 inner condition called on no array match
function(callback) {
console.log('update 3:');
Test.findOneAndUpdate(
{ "_id": id, "readers.user": userId },
{ "$set": { "readers.$.someData": 1 } },
{ "new": true },
function(err,doc) {
if (err) callback(err);
if (!doc) {
console.log('went inner');
Test.findOneAndUpdate(
{ "_id": id, "readers.user": { "$ne": userId } },
{ "$push": { "readers": { "user": userId, "someData": 1 } } },
{ "new": true },
function(err,doc) {
console.log(doc)
callback(err);
}
);
} else {
console.log(doc);
callback(err);
}
}
);
},
// cascade 2 outer condition met on array match
function(callback) {
console.log('update 3:');
Test.findOneAndUpdate(
{ "_id": id, "readers.user": userId },
{ "$set": { "readers.$.someData": 2 } },
{ "new": true },
function(err,doc) {
if (err) callback(err);
if (!doc) {
console.log('went inner');
Test.findOneAndUpdate(
{ "_id": id, "readers.user": { "$ne": userId } },
{ "$push": { "readers": { "user": userId, "someData": 2 } } },
{ "new": true },
function(err,doc) {
console.log(doc)
callback(err);
}
);
} else {
console.log(doc);
callback(err);
}
}
);
}
],
function(err) {
if (err) throw err;
mongoose.disconnect();
}
);
Output:
initial state:
{ __v: 0,
name: 'Topic',
_id: 55f60adc1beeff6b0a175e98,
readers: [] }
update 1:
{
"ok": 1,
"writeErrors": [],
"writeConcernErrors": [],
"insertedIds": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 1,
"nModified": 1,
"nRemoved": 0,
"upserted": []
}
{ _id: 55f60adc1beeff6b0a175e98,
name: 'Topic',
__v: 0,
readers: [ { user: 55f60adc1beeff6b0a175e97, someData: '1' } ] }
update 2:
{
"ok": 1,
"writeErrors": [],
"writeConcernErrors": [],
"insertedIds": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 1,
"nModified": 1,
"nRemoved": 0,
"upserted": []
}
{ _id: 55f60adc1beeff6b0a175e98,
name: 'Topic',
__v: 0,
readers: [ { user: 55f60adc1beeff6b0a175e97, someData: '2' } ] }
cleared:
{ _id: 55f60adc1beeff6b0a175e98,
name: 'Topic',
__v: 0,
readers: [] }
update 3:
went inner
{ _id: 55f60adc1beeff6b0a175e98,
name: 'Topic',
__v: 0,
readers: [ { someData: '1', user: 55f60adc1beeff6b0a175e97 } ] }
update 3:
{ _id: 55f60adc1beeff6b0a175e98,
name: 'Topic',
__v: 0,
readers: [ { someData: '2', user: 55f60adc1beeff6b0a175e97 } ] }
I use Nodejs, Hapijs and Mongoose.
I 've a schema and model as follows.
var schema = {
name: {
type: String,
required: true
},
lectures: {}
};
var mongooseSchema = new mongoose.Schema(schema, {
collection: "Users"
});
mongoose.model("Users", mongooseSchema);
For some reason, I need to keep "lectures"
as mixed type.
While saving/creating a document I create a nested property lectures.physics.topic[] where topic is an array.
Now, I'm trying to add/push a new object to "lectures.physics.topic" using $addToSet or $push.
userModel.findByIdAndUpdateAsync(user._id, {
$addToSet: {
"lectures.physics.topic": {
"name": "Fluid Mechanics",
"day": "Monday",
"faculty": "Nancy Wagner"
}
}
});
But the document is simply not getting updated. I tried using $push too. Nothing worked. What could be the problem?
I tried to another approach using mongoclient , to update the db directly .It works please find the below code which works
db.collection("Users").update({
"_id": user._id
}, {
$addToSet: {
"lectures.physics.topic": {
"name": "Fluid Mechanics",
"day": "Monday",
"faculty": "Nancy Wagner"
}
}
}, function(err, result) {
if (err) {
console.log("Superman!");
console.log(err);
return;
}
console.log(result);
});
I have to start the mongo client every time a request is hit.This is not a feasible solution.
Mongoose loses the ability to auto detect and save changes made on Mixed types so you need to "tell" it that the value of a Mixed type has changed by calling the .markModified(path) method of the document passing the path to the Mixed type you just changed:
doc.mixed.type = 'changed';
doc.markModified('mixed.type');
doc.save() // changes to mixed.type are now persisted
In your case, you could use findById() method to make your changes by calling the addToSet() method on the topic array and then triggering the save() method to persist the changes:
userModel.findById(user._id, function (err, doc){
var item = {
"name": "Fluid Mechanics",
"day": "Monday",
"faculty": "Nancy Wagner"
};
doc.lectures.physics.topic.addToSet(item);
doc.markModified('lectures');
doc.save() // changes to lectures are now persisted
});
I'd be calling "bug" on this. Mongoose is clearly doing the wrong thing as can be evidenced in the logging as shown later. But here is a listing that calls .findOneAndUpdate() from the native driver with the same update you are trying to do:
var async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/school');
mongoose.set('debug',true);
var userSchema = new Schema({
name: {
type: String,
required: true
},
lectures: { type: Schema.Types.Mixed }
});
var User = mongoose.model( "User", userSchema );
function logger(data) {
return JSON.stringify(data, undefined, 2);
}
async.waterfall(
[
function(callback) {
User.remove({},function(err) {
callback(err);
});
},
function(callback) {
console.log("here");
var user = new User({ "name": "bob" });
user.save(function(err,user) {
callback(err,user);
});
},
function(user,callback) {
console.log("Saved: %s", logger(user));
User.collection.findOneAndUpdate(
{ "_id": user._id },
{
"$addToSet": {
"lectures.physics.topic": {
"name": "Fluid Mechanics",
"day": "Monday",
"faculty": "Nancy Wagner"
}
}
},
{ "returnOriginal": false },
function(err,user) {
callback(err,user);
}
);
}
],
function(err,user) {
if (err) throw err;
console.log("Modified: %s", logger(user));
mongoose.disconnect();
}
);
This works perfectly with the result:
Saved: {
"__v": 0,
"name": "bob",
"_id": "55cda1f5b5ee8b870e2f53bd"
}
Modified: {
"lastErrorObject": {
"updatedExisting": true,
"n": 1
},
"value": {
"_id": "55cda1f5b5ee8b870e2f53bd",
"name": "bob",
"__v": 0,
"lectures": {
"physics": {
"topic": [
{
"name": "Fluid Mechanics",
"day": "Monday",
"faculty": "Nancy Wagner"
}
]
}
}
},
"ok": 1
}
You neeed to be careful here as native driver methods are not aware of the connection status like the mongoose methods are. So you need to be sure a connection has been made by a "mongoose" method firing earlier, or wrap your app in a connection event like so:
mongoose.connection.on("connect",function(err) {
// start app in here
});
As for the "bug", look at the logging output from this listing:
var async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/school');
mongoose.set('debug',true);
var userSchema = new Schema({
name: {
type: String,
required: true
},
lectures: { type: Schema.Types.Mixed }
});
var User = mongoose.model( "User", userSchema );
function logger(data) {
return JSON.stringify(data, undefined, 2);
}
async.waterfall(
[
function(callback) {
User.remove({},function(err) {
callback(err);
});
},
function(callback) {
console.log("here");
var user = new User({ "name": "bob" });
user.save(function(err,user) {
callback(err,user);
});
},
function(user,callback) {
console.log("Saved: %s", logger(user));
User.findByIdAndUpdate(
user._id,
{
"$addToSet": {
"lectures.physics.topic": {
"name": "Fluid Mechanics",
"day": "Monday",
"faculty": "Nancy Wagner"
}
}
},
{ "new": true },
function(err,user) {
callback(err,user);
}
);
}
],
function(err,user) {
if (err) throw err;
console.log("Modified: %s", logger(user));
mongoose.disconnect();
}
);
And the logged output with mongoose logging:
Mongoose: users.remove({}) {}
here
Mongoose: users.insert({ name: 'bob', _id: ObjectId("55cda2d2462283c90ea3f1ad"), __v: 0 })
Saved: {
"__v": 0,
"name": "bob",
"_id": "55cda2d2462283c90ea3f1ad"
}
Mongoose: users.findOne({ _id: ObjectId("55cda2d2462283c90ea3f1ad") }) { new: true, fields: undefined }
Modified: {
"_id": "55cda2d2462283c90ea3f1ad",
"name": "bob",
"__v": 0
}
So in true "What the Fudge?" style, there is a call there to .findOne()? Which is not what was asked. Moreover, nothing is altered in the database of course because the wrong call is made. So even the { "new": true } here is redundant.
This happens at all levels with "Mixed" schema types.
Personally I would not nest within "Objects" like this, and just make your "Object keys" part of the standard array as additional properties. Both MongoDB and mongoose are much happier with this, and it is much easier to query for information with such a structure.
var async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/school');
mongoose.set('debug',true);
var lectureSchema = new Schema({
"subject": String,
"topic": String,
"day": String,
"faculty": String
});
var userSchema = new Schema({
name: {
type: String,
required: true
},
lectures: [lectureSchema]
});
var User = mongoose.model( "User", userSchema );
function logger(data) {
return JSON.stringify(data, undefined, 2);
}
async.waterfall(
[
function(callback) {
User.remove({},function(err) {
callback(err);
});
},
function(callback) {
console.log("here");
var user = new User({ "name": "bob" });
user.save(function(err,user) {
callback(err,user);
});
},
function(user,callback) {
console.log("Saved: %s", logger(user));
User.findByIdAndUpdate(
user._id,
{
"$addToSet": {
"lectures": {
"subject": "physics",
"topic": "Fluid Mechanics",
"day": "Monday",
"faculty": "Nancy Wagner"
}
}
},
{ "new": true },
function(err,user) {
callback(err,user);
}
);
}
],
function(err,user) {
if (err) throw err;
console.log("Modified: %s", logger(user));
mongoose.disconnect();
}
);
Output:
Mongoose: users.remove({}) {}
here
Mongoose: users.insert({ name: 'bob', _id: ObjectId("55cda4dc40f2a8fb0e5cdf8b"), lectures: [], __v: 0 })
Saved: {
"__v": 0,
"name": "bob",
"_id": "55cda4dc40f2a8fb0e5cdf8b",
"lectures": []
}
Mongoose: users.findAndModify({ _id: ObjectId("55cda4dc40f2a8fb0e5cdf8b") }) [] { '$addToSet': { lectures: { faculty: 'Nancy Wagner', day: 'Monday', topic: 'Fluid Mechanics', subject: 'physics', _id: ObjectId("55cda4dc40f2a8fb0e5cdf8c") } } } { new: true, upsert: false, remove: false }
Modified: {
"_id": "55cda4dc40f2a8fb0e5cdf8b",
"name": "bob",
"__v": 0,
"lectures": [
{
"faculty": "Nancy Wagner",
"day": "Monday",
"topic": "Fluid Mechanics",
"subject": "physics",
"_id": "55cda4dc40f2a8fb0e5cdf8c"
}
]
}
So that works fine, and you don't need to dig to the native methods just to make it work.
Properties of an array make this much easy to query and filter, as well as "aggregate" information across the data, which for all of those MongoDB likes a "strict path" to reference all information. Otherwise you are diffing to only "specific keys", and those cannot be indexed or really searched without mentioning every possible "key combination".
Properties like this are a better way to go. And no bugs here.
I'm trying to use $setUnion in an aggregation query, it works just fine in the MongoDB console but when I try to do the same in my Node app with Mongoose, it fails with this error:
{
[MongoError: exception: invalid operator '$setUnion']
name: 'MongoError',
errmsg: 'exception: invalid operator \'$setUnion\'',
code: 15999,
ok: 0
}
Here is a simplified query that I tried and still failed in the app:
db.users.aggregate(
{
$match: { _id: ObjectId("52c9375c497ed763f41941bd") }
},
{
$project: {
activities: {
$setUnion: [[1,2], [3,4]]
}
}
}
);
Any ideas what I could be doing wrong?
MongoDB: 2.6.6, Mongoose: 3.8.21, Node: 0.10.26
Update: here is the Mongoose code
var User = require('./providers/db').User,
mongoose = require('mongoose');
User.aggregate({
$match: { _id: mongoose.Types.ObjectId("52c9375c497ed763f41941bd") }
},
{
$project: {
activities: {
$setUnion: [[1,2], [3,4]]
}
}
}, function (err, result) {
if (err) { return console.log(err); }
else { console.log(result); }
});
Really don't see the point here and did ask you to look at it. And cannot say I didn't warn you several times. I believe the term you are looking for is "thank you very much":
var testSchema = new Schema({},{ "strict": false });
var User = mongoose.model( "User", testSchema, "test" );
mongoose.connect("mongodb://localhost/test");
async.series(
[
function(callback) {
User.remove({},callback);
},
function(callback) {
User.create({ "a": 1 },callback);
},
function(callback) {
User.aggregate(
[
{ "$project": {
"activities": {
"$setUnion": [[1,2],[3,4]]
}
}}
],
function(err,result) {
if (err) throw err;
console.log( result );
process.exit();
}
);
}
],
function(err) {
if (err) throw err;
}
);
And returns :
[ { _id: 54b4c13decda256710226134, activities: [ 1, 4, 3, 2 ] } ]
just as would be expected.
I'm trying to remove the _Id from the returned documents, this is my code:
module.exports = function(app) {
// Module dependencies.
var mongoose = require('mongoose'),
Contacts = mongoose.models.Contacts,
api = {},
limit = 10;
api.contacts = function(req, res) {
Contacts.aggregate([{
$group: {
"_id": {
name: "$name",
city: "$city",
state: "$state"
}
}
}, {
$sort: {
AgencyTranslation: 1
}
}, {
$limit: req.query.limit | limit
}],
function(err, contacts) {
if (err) {
res.json(500, err);
} else {
res.json({
contacts: contacts
})
}
})
};
app.get('/api/contacts', api.contacts);
};
the current result-set looks like this:
{
"contacts":[
{"_id":{"name":"Joe","city":"ankorage","state":"AL"}},
{"_id":{"name":"Mark","city":"washington","state":"DC"}}
...
]
}
I tried to replace "_Id" with "$project", or $project, and adding "_Id": 0 to the object, as some have suggested elsewhere, but was not successful.
I also tried res.send(contacts), but that only stripped the super-object ('contacts').
Any suggestions are appreciated.
Like this
Contacts.aggregate( [
{ $group: { "_id": { name: "$name", city: "$city", state: "$state" } } },
{ $project: {_id: 0, name: '$_id.name', city: '$_id.city', state: '$_id.state'} },
{ $sort: { AgencyTranslation: 1 } },
{ $limit: req.query.limit | limit }
], function () {
});
Bunch of time but, here is the answer:
After making $group or $project, do this:
{ $unset: ["_id"] }