Related
I've been pulling my hair out for weeks over this one.
I have a collection (this is a cut down version):
const SubscriberSchema = new Schema({
publication: { type: Schema.Types.ObjectId, ref: "publicationcollection" },
buyer: { type: Schema.Types.ObjectId, ref: "buyercollection" },
postCode: { type: String },
modifiedBy: { type: String },
modified: { type: Date }
});
I also have a collection containing the 1.75 million UK Postcodes
const PostcodeSchema = new Schema({
postcode: { type: String }
});
What I want to do is to return any record in the Subscriber collection which doesn't exist within the Postcode collection.
When I try a very simple aggregation using Mongoose on anything >100 records in the Subscriber collection, I'm getting either a timeout or a >16MB return error.
Here's what I've tried so far:
router.get(
"/badpostcodes/:id",
passport.authenticate("jwt", { session: false }),
(req, res) => {
const errors = {};
Subscriber.aggregate([
{
$match: {
publication: mongoose.Types.ObjectId(req.params.id),
postCode: { "$ne": null, $exists: true }
}
},
{
$lookup: {
'from': 'postcodescollections',
'localField': 'postCode',
'foreignField': 'postcode',
'as': 'founditem'
}
},
// {
// $unwind: '$founditem'
// },
{
$match: {
'founditem': { $eq: [] }
}
}
], function (err, result) {
if (err) {
console.log(err);
} else {
if (result.length > 0) {
res.json(result);
} else {
res.json("0");
}
}
})
}
);
The unwind didn't seem to do anything but it's commented out to show I tried to use it.
I've also tried using a pipeline on the lookup instead but that didn't work, similar to the following (sorry, I don't have my original code attempt so this is from memory only):
$lookup: {
'from': 'postcodescollections',
'let': { 'postcode': "$postCode" },
'pipeline': [
{
'$match': {
'postcode': { $exists: false }
}
},
{
'$unwind': "$postCode"
}
],
'as': 'founditem'
}
Thanks in advance so I can hopefully retain some hair!
You are doing a match on all postcodes that don't match and then unwinding those - that will be a 1.75m documents for each subscriber! The syntax in $lookup is also incorrect I think.
I think you can try something like the following - adjust accordingly for your data:
Do a $lookup to find a matching postcode in postcodes, then do a match to filter those subscribers that that don't have any founditem elements: "founditem.0": {$exists: false}
See an example:
db.getCollection("subscribers").aggregate(
[
// Stage 1
{
$match: {
postCode: { "$ne": null, $exists: true }
}
},
// Stage 2
{
$project: {
_id: 1,
postCode: 1
}
},
// Stage 3
{
$lookup: {
from: "postcodescollections",
let: { p: "$postCode" },
pipeline: [
{
$match: {
$expr:
{
$eq: ["$$p","$postcode"] }
}
},
{ $project: { _id: 1 } }
],
as: "founditem"
}
},
// Stage 4
{
$match: {
"founditem.0": {$exists: false}
}
},
]
);
I need query to fetch the total count as per key value of document by matching some value from array using MongoDB. I am explaining my document and Input below.
dataArr=[
{'login_id':9937229853,'location':'Delhi'},
{'login_id':9937229854,'location':'JK'}
]
My document is given below.
feedback:
{
login_id:9937229853,
code: PTP,
remark:'Hello'
},
{
login_id:9937229853,
code: PTP,
remark:'Hii'
},
{
login_id:9937229853,
code: CB,
remark:'aaaaa'
},
{
login_id:9937229854,
code: PTP,
remark:'jjjjj'
},
{
login_id:9937229854,
code: CB,
remark:'dddd'
}
The above is my collection. Here I need as per user input login_id present inside array will match with document and the total count will be fetch as per document key and value. My expected output is given below. I am explaining my code below.
for(var i=0;i<dataArr.length;i++){
var login=dataArr[i]['login_id'];
//console.log('cdocs',dataArr[i]['login_id']);
Feedback.collection.count({login_id:dataArr[i]['login_id']},function(cerr,cdocs){
console.log('cdocs',login);
if (!cerr) {
if(cdocs > 0){
// console.log('login',cdocs);
db.collection.aggregate([
{
$match: {
keywords: { $not: {$size: 0} }
}
},
{ $unwind: "$keywords" },
{
$group: {
_id: {$toLower: '$keywords'},
count: { $sum: 1 }
}
},
{
$match: {
login_id: login
}
}
])
.toArray((err,docs)=>{
if (!err) {
// console.log('count::',docs);
finalArr=docs;
}
})
}
}
})
}
var data={'status':'success','data':finalArr}
res.send(data);
I need the expected result like below.
finalArr=[
{'login_id':9937229853,'location':'Delhi','PTP':2,'CB':1,'remark':3},
{'login_id':9937229854,'location':'JK','PTP':1,'CB':1,'remark':2},
]
But using my code I am getting the blank output. Please help me to resolve this issue.
You can do all this with a single aggregate operation. The first pipeline stage would be filtering the documents in the collection using the input array. You would need to map that array to just a list of ids though in order to use the $in query operator i.e.
const ids = dataArr.map(({ login_id }) => login_id)
which can then be used in the $match pipeline as
const match = { '$match': { 'login_in': { '$in': ids } } }
The next pipeline step will then use the $group stage to group the above filtered documents by the login_id key
const allGroup = { '$group': {
'_id': {
'login_id': '$login_id',
'code': '$code',
'remark': '$remark'
},
'count': { '$sum': 1 }
} }
Another $group pipeline stage to get the remarks counts as a list of key/value documents
const remarksGroup = { '$group': {
'_id': {
'login_id': '$_id.login_id',
'code': '$_id.code'
},
'remarks': {
'$push': {
'k': '$_id.remark',
'v': '$count'
}
},
'count': { '$sum': 1 }
} }
Get the code counts with a similar structure as above
const codeGroup = { '$group': {
'_id': '$_id.login_id',
'codes': {
'$push': {
'k': '$_id.code',
'v': '$count'
}
},
'remarks': { '$first': '$remarks' }
} }
You would need a final pipeline to convert the key/value pairs arrays to objects using $arrayToObject, merge the objects into one using $mergeObjects and replace the root document with the merged docs using $replaceRoot:
const projections = { '$replaceRoot': {
'newRoot': {
'$mergeObjects': [
{ 'login_id': '$_id' },
{ '$arrayToObject': '$codes' },
{ '$arrayToObject': '$remarks' }
]
}
} }
Your full aggregate pipeline operation would be:
(async () => {
try {
const ids = dataArr.map(({ login_id }) => login_id)
const match = { '$match': { 'login_in': { '$in': ids } } }
const allGroup = { '$group': {
'_id': {
'login_id': '$login_id',
'code': '$code',
'remark': '$remark'
},
'count': { '$sum': 1 }
} }
const remarksGroup = { '$group': {
'_id': {
'login_id': '$_id.login_id',
'code': '$_id.code'
},
'remarks': {
'$push': {
'k': '$_id.remark',
'v': '$count'
}
},
'count': { '$sum': 1 }
} }
const codeGroup = { '$group': {
'_id': '$_id.login_id',
'codes': {
'$push': {
'k': '$_id.code',
'v': '$count'
}
},
'remarks': { '$first': '$remarks' }
} }
const projections = { '$$replaceRoot': {
'newRoot': {
'$mergeObjects': [
{ 'login_id': '$_id' },
{ '$arrayToObject': '$codes' },
{ '$arrayToObject': '$remarks' }
]
}
} }
const result = await Feedback.aggregate([
match,
allGroup,
remarksGroup,
codeGroup,
projections
])
/* get the location key */
const data = result.map(item => {
const [{ location }, ...rest] = dataArr.filter(d => d.location_id === item.location_id)
return { location, ...item }
})
console.log(data)
res.send(data)
} catch (err) {
// handle error
}
})()
i am new in node js i need to find data daily,
weekly and monthly but i don't know how to do this?
currently i am finding today data . also need to find current week data and current month data.
i don't have idea how to do this, can you please help.
exports.getData = function(req,res)
{
getMyData().then(function(data){
res.status(200).json({
msg:'true',
data: data
});
}).catch(function(err){
res.status(401).json({
msg:'Not Logged In',
error: err
});
});
function getMyData(){
return new Promise(function(resolve,reject){
var token = req.headers['x-access-token'];
if (!token)
return reject({'msg':'No Token'});
jwt.verify(token,config.secret,function(err,decode){
if(err)
{
reject({'msg':'Failed to authenticate token.'});
}
else{
var start = new Date();
start.setHours(0,0,0,0);
var end = new Date();
end.setHours(23,59,59,999);
Data.find({userId: decode.id,created: {$gte: start, $lt: end}},function(err,data){
if(err){
reject(err);
}
else
{
resolve(data);
}
});
}
});
});
}
}
here is my schema
var DataSchema = new Schema({
userId: {
type: String,
required: 'Kindly enter your Report'
},data: {
type: String,
required: 'Kindly enter your Report'
},
created:{
type:Date,
default:Date.now
}
});
You can call find query based on different conditions from node.js.
async function getMyData() {
var token = req.headers['x-access-token'];
if (!token)
throw {
'msg': 'No Token'
};
jwt.verify(token, config.secret, function (err, decode) {
if (err) {
throw err;
} else {
let today = new Date();
today.setHours(0, 0, 0, 0)
let first = today.getDate() - today.getDay();
let last = first + 6;
let firstday = new Date(today.setDate(first)).toUTCString();
let lastday = new Date(today.setDate(last)).toUTCString();
let firstDayMonth = new Date(today.setDate(1));
let lastDayMonth = new Date(today.getFullYear(), today.getMonth() + 1, 0)
lastDayMonth.setHours(23, 59, 59, 0);
today = new Date().setHours(0, 0, 0, 0);
return await Promise.all([
Data.find({
userId: decode.id,
created: {
$gte: today
}
}).exec(),
Data.find({
userId: decode.id,
created: {
$gte: firstday,
$lte: lastday
}
}).exec(),
Data.find({
userId: decode.id,
created: {
$gte: firstDayMonth,
$lte: lastDayMonth
}
}).exec()
]);
}
});
}
MongoDB Aggregate query : only problem I'm seeing is $cond else block returns "". That you can handle in node js.
var today = new Date().setHours(0, 0, 0, 0);
var first = today.getDate() - today.getDay();
var firstDayWeek = new Date(today.setDate(first));
var lastDayWeek = new Date(today.setDate(first + 6));
var firstDayMonth = new Date(today.setDate(1));
var lastDayMonth = new Date(today.getFullYear(), today.getMonth() + 1, 0)
lastDayWeek.setHours(23, 59, 59, 0);
lastDayMonth.setHours(23, 59, 59, 0);
today = new Date().setHours(0, 0, 0, 0);
db.getCollection('TEST').aggregate([{
$match: {
userId: decode.id
}
}, {
$group: {
"_id": "",
"today": {
$push: {
$cond: {
if: {
$gte: ["$created", new Date(today)]
},
then: "$$ROOT",
else: ''
}
}
},
"week": {
$push: {
$cond: [{
$and: [{
$gte: ["$created", new Date(firstDayWeek)]
},
{
$lte: ["$created", new Date(lastDayWeek)]
}
]
},
"$$ROOT",
''
]
}
},
"month": {
$push: {
$cond: [{
$and: [{
$gte: ["$created", new Date(firstDayMonth)]
},
{
$lte: ["$created", new Date(lastDayMonth)]
}
]
},
"$$ROOT",
''
]
}
}
}
}])
//If you want to filter in mongo query
.forEach(function (data) {
data.today = data.today.filter(e => e != "")
data.week = data.week.filter(e => e != "")
print(data);
})
Output
{
"_id": "",
"today": [{
"_id": ObjectId("5aaa2605d52a86d42a362479"),
"created": ISODate("2018-03-15T07:51:33.014Z")
}
],
"week": [{
"_id": ObjectId("5aaa2605d52a86d42a362479"),
"created": ISODate("2018-03-15T07:51:33.014Z")
},
{
"_id": ObjectId("5aaa2606d52a86d42a36247a"),
"created": ISODate("2018-03-13T07:51:34.702Z")
}
],
"month": [{
"_id": ObjectId("5aaa2605d52a86d42a362479"),
"created": ISODate("2018-03-15T07:51:33.014Z")
},
{
"_id": ObjectId("5aaa2606d52a86d42a36247a"),
"created": ISODate("2018-03-13T07:51:34.702Z")
},
{
"_id": ObjectId("5aaa262ad52a86d42a36247b"),
"created": ISODate("2018-03-01T07:52:10.175Z")
}
]
}
We currently have the following code to group all documents from a collection by creation date:
this.aggregate( [
{ $match: { language: options.criteria.language, status: 1, type:{ $ne: "challenge" } }},
{ $group: {
_id: {
y: { '$year': '$created' },
m: { '$month': '$created' },
d: { '$dayOfMonth': '$created' },
h: { '$hour': '$created' },
min: { '$minute': '$created' },
s: { '$second': '$created' }
},
count: { $sum : 1 }
}},
{$project: {
date: "$_id", // so this is the shorter way
count: 1,
_id: 0
}},
{ $sort: { "date": 1 } }
], function(err, result){
if(err) {
return callback(err);
}
callback(null, result);
});
However, we now would like to group the results based on the start date instead of the creation date. The start date is not a field of the current collection, but it is a field of the currentRevision object, that is linked in this collection.
I tried this:
this.aggregate( [
{ $match: { language: options.criteria.language, status: 1, type:{ $ne: "challenge" } }},
{ $group: {
_id: {
y: { '$year': '$currentRevision.start_date' },
m: { '$month': '$currentRevision.start_date' },
d: { '$dayOfMonth': '$currentRevision.start_date' },
h: { '$hour': '$currentRevision.start_date' },
min: { '$minute': '$currentRevision.start_date' },
s: { '$second': '$currentRevision.start_date' }
},
count: { $sum : 1 }
}},
{$project: {
date: "$_id", // so this is the shorter way
count: 1,
_id: 0
}},
{ $sort: { "date": 1 } }
], function(err, result){
if(err) {
return callback(err);
}
callback(null, result);
});
but that just gives me an error: "failed to query db"
any idea on how to solve this?
I'm using Mongoose (MongoDB in node.js), and after reading this answer:
Replace value in array
I have another question:
Is it possible to do in the same sentence: push element into array or replace if this element is existing in the array?
Maybe something like this? (The example doesn't work)
Model.findByIdAndUpdate(id,
{
$pull: {"readers": {user: req.user.id}},
$push:{"readers":{user: req.user.id, someData: data}}
},{multi:true},callback)
Message error:
errmsg: 'exception: Cannot update \'readers\' and \'readers\' at the same time
Reference:
https://stackoverflow.com/a/15975515/4467741
Thank you!
Multiple operations on the same property path are simply not allowed in a single request, with the main reason being that the operations themselves have "no particular order" in the way the engine assigns them as the document is updated, and therefore there is a conflict that should be reported as an error.
So the basic abstraction on this is that you have "two" update operations to perform, being one to "replace" the element where it exists, and the other to "push" the new element where it does not exist.
The best way to implement this is using "Bulk" operations, which whilst still "technically" is "two" update operations, it is however just a "single" request and response, no matter which condition was met:
var bulk = Model.collection.initializeOrderedBulkOp();
bulk.find({ "_id": id, "readers.user": req.user.id }).updateOne({
"$set": { "readers.$.someData": data } }
});
bulk.find({ "_id": id, "readers.user": { "$ne": req.user.id } }).updateOne({
"$push": { "readers": { "user": req.user.id, "someData": data } }
});
bulk.execute(function(err,result) {
// deal with result here
});
If you really "need" the updated object in result, then this truly becomes a "possible" multiple request following the logic where the array element was not found:
Model.findOneAndUpdate(
{ "_id": id, "readers.user": req.user.id },
{ "$set": { "readers.$.someData": data } },
{ "new": true },
function(err,doc) {
if (err) // handle error;
if (!doc) {
Model.findOneAndUpdate(
{ "_id": id, "readers.user": { "$ne": req.user.id } },
{ "$push": { "readers":{ "user": req.user.id, "someData": data } } },
{ "new": true },
function(err,doc) {
// or return here when the first did not match
}
);
} else {
// was updated on first try, respond
}
}
);
And again using you preferred method of not nesting callbacks with either something like async or nested promise results of some description, to avoid the basic indent creep that is inherrent to one action being dependant on the result of another.
Basically probably a lot more efficient to perform the updates in "Bulk" and then "fetch" the data afterwards if you really need it.
Complete Listing
var async = require('async'),
mongoose = require('mongoose')
Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/test');
var userSchema = new Schema({
name: String
});
var dataSchema = new Schema({
user: { type: Schema.Types.ObjectId, ref: 'User' },
someData: String
},{ "_id": false });
var testSchema = new Schema({
name: String,
readers: [dataSchema]
});
var User = mongoose.model( 'User', userSchema ),
Test = mongoose.model( 'Test', testSchema );
var userId = null,
id = null;
async.series(
[
// Clean models
function(callback) {
async.each([User,Test],function(model,callback) {
model.remove({},callback);
},callback);
},
// Create a user
function(callback) {
User.create({ name: 'bill' },function(err,user) {
userId = user._id;
callback(err);
});
},
function(callback) {
Test.create({ name: 'Topic' },function(err,topic) {
id = topic._id;
console.log("initial state:");
console.log(topic);
callback(err);
});
},
// 1st insert array 2nd update match 1 modified
function(callback) {
var bulk = Test.collection.initializeOrderedBulkOp();
bulk.find({ "_id": id, "readers.user": userId }).updateOne({
"$set": { "readers.$.someData": 1 }
});
bulk.find({ "_id": id, "readers.user": { "$ne": userId }}).updateOne({
"$push": { "readers": { "user": userId, "someData": 1 } }
});
bulk.execute(function(err,result) {
if (err) callback(err);
console.log("update 1:");
console.log(JSON.stringify( result, undefined, 2));
Test.findById(id,function(err,doc) {
console.log(doc);
callback(err);
});
});
},
// 2nd replace array 1st update match 1 modified
function(callback) {
var bulk = Test.collection.initializeOrderedBulkOp();
bulk.find({ "_id": id, "readers.user": userId }).updateOne({
"$set": { "readers.$.someData": 2 }
});
bulk.find({ "_id": id, "readers.user": { "$ne": userId }}).updateOne({
"$push": { "readers": { "user": userId, "someData": 2 } }
});
bulk.execute(function(err,result) {
if (err) callback(err);
console.log("update 2:");
console.log(JSON.stringify( result, undefined, 2));
Test.findById(id,function(err,doc) {
console.log(doc);
callback(err);
});
});
},
// clear array
function(callback) {
Test.findByIdAndUpdate(id,
{ "$pull": { "readers": {} } },
{ "new": true },
function(err,doc) {
console.log('cleared:');
console.log(doc);
callback(err);
}
);
},
// cascade 1 inner condition called on no array match
function(callback) {
console.log('update 3:');
Test.findOneAndUpdate(
{ "_id": id, "readers.user": userId },
{ "$set": { "readers.$.someData": 1 } },
{ "new": true },
function(err,doc) {
if (err) callback(err);
if (!doc) {
console.log('went inner');
Test.findOneAndUpdate(
{ "_id": id, "readers.user": { "$ne": userId } },
{ "$push": { "readers": { "user": userId, "someData": 1 } } },
{ "new": true },
function(err,doc) {
console.log(doc)
callback(err);
}
);
} else {
console.log(doc);
callback(err);
}
}
);
},
// cascade 2 outer condition met on array match
function(callback) {
console.log('update 3:');
Test.findOneAndUpdate(
{ "_id": id, "readers.user": userId },
{ "$set": { "readers.$.someData": 2 } },
{ "new": true },
function(err,doc) {
if (err) callback(err);
if (!doc) {
console.log('went inner');
Test.findOneAndUpdate(
{ "_id": id, "readers.user": { "$ne": userId } },
{ "$push": { "readers": { "user": userId, "someData": 2 } } },
{ "new": true },
function(err,doc) {
console.log(doc)
callback(err);
}
);
} else {
console.log(doc);
callback(err);
}
}
);
}
],
function(err) {
if (err) throw err;
mongoose.disconnect();
}
);
Output:
initial state:
{ __v: 0,
name: 'Topic',
_id: 55f60adc1beeff6b0a175e98,
readers: [] }
update 1:
{
"ok": 1,
"writeErrors": [],
"writeConcernErrors": [],
"insertedIds": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 1,
"nModified": 1,
"nRemoved": 0,
"upserted": []
}
{ _id: 55f60adc1beeff6b0a175e98,
name: 'Topic',
__v: 0,
readers: [ { user: 55f60adc1beeff6b0a175e97, someData: '1' } ] }
update 2:
{
"ok": 1,
"writeErrors": [],
"writeConcernErrors": [],
"insertedIds": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 1,
"nModified": 1,
"nRemoved": 0,
"upserted": []
}
{ _id: 55f60adc1beeff6b0a175e98,
name: 'Topic',
__v: 0,
readers: [ { user: 55f60adc1beeff6b0a175e97, someData: '2' } ] }
cleared:
{ _id: 55f60adc1beeff6b0a175e98,
name: 'Topic',
__v: 0,
readers: [] }
update 3:
went inner
{ _id: 55f60adc1beeff6b0a175e98,
name: 'Topic',
__v: 0,
readers: [ { someData: '1', user: 55f60adc1beeff6b0a175e97 } ] }
update 3:
{ _id: 55f60adc1beeff6b0a175e98,
name: 'Topic',
__v: 0,
readers: [ { someData: '2', user: 55f60adc1beeff6b0a175e97 } ] }