I am learning NodeJS and Mongo Db and I got stuck. Could you please help me If you know solution?
I have 2 questions
1.)I have 2 collections in mongo db
I tried to find out which projects user has. For example michaela has projects :nodejs, test2
jasmin has projects: test2 (I don't want to know who are coders for current project but I want to know what are projects for current user)
data what I want to get should look something like this
[{
id: "some value"
username: "michaela"
email:"some value"
password:"some value"
projects:"[{
project_name:"node js"
project_description:"learn node js"
deadline:"2021-11-12"
status:"todo"
},
{
project_name:"test2"
project_description:"learn node js"
deadline:"2021-11-12"
status:"todo"
}]
},
id: "some value"
username: jasmin"
email:"some value"
password:"some value"
projects:"[
{
project_name:"test2"
project_description:"learn node js"
deadline:"2021-11-12"
status:"todo"
}]
}]
I don't understand exactly how look up works. I tried following example(test) but it doesn't work
I also don't understand how to connect 2 db for example allMembers and all projects I merged into one array but I don't think this is correct way how to do it.
this is my model
const usersCollection = require('../db').db().collection("users")
const projectsCollection = require('../db').db().collection("projects")
Team.prototype.getAllMembers = function(){
return new Promise (async (resolve, reject) => {
try{
let allMembers = await usersCollection.find({}).toArray()
let allprojects = await projectsCollection.find({}).toArray()
let test = await usersCollection.aggregate([
{$lookup: {from: "projects", localField: "username", foreignField: "coder", as: "projects"}},
{$project: {
username:1,
projects:{
name:{$arrayElemAt: ["$projects.project_name", 0]},
description:{$arrayElemAt: ["$projects.project_description", 0]},
},
email:1,
password:1,
}}
]).toArray()
console.log("test", test)
let arr3 = [...allMembers, ...allprojects]
resolve(arr3)
}catch{
reject()
}
})
}
this is my controller
exports.home = function(req, res) {
//when user is log in
if (req.session.user) {
let teamMember = new Team(req.body)
teamMember.getAllMembers().then((data)=>{
console.log("members data", data)
if(data){
res.render('members', {data: data })
}else{
res.send("on this page are no data")
}
}).catch((err)=>{
res.send("problem")
})
//when user is not log in, flash tie errory z db automaticky vymaze po pouziti
} else {
res.render('login', {errors: req.flash('errors'), regErrors: req.flash('regErrors')})
}
}
and this is how it looks in db
Thank you in advance
I think you should aggregate on user collection like this match user name and then project first element of projects
lookup in here put project to projects field
db.users.aggregate([
{
$match: {
username: "jasmin"
}
},
{
"$lookup": {
"from": "projects",
"localField": "username",
"foreignField": "coder",
"as": "projects"
}
},
{
"$project": {
username: 1,
email:1,
password:1,
projects: 1
}
}
])
https://mongoplayground.net/p/U6hNw2WdQLE
if you want to get all data with just first project in that use this
db.users.aggregate([
{
"$lookup": {
"from": "projects",
"localField": "username",
"foreignField": "coder",
"as": "projects"
}
},
{
"$project": {
username: 1,
email:1,
password:1,
projects:{$arrayElemAt: ["$projects", 0]}
}
}
])
Related
I want to create an API which will query users in my mongodb and returns the data of all users. For each user i need to perform additional query to get count in 2 other schema as below:
const getUsersSummary = async (req, res, next) => {
try {
const users = await User.fetch();
const usersWithCount = await Promise.all(
users.map(async (user) => {
let tests;
let enrollments;
try {
tests = await Test.countDocuments(
{ user: user._id }
);
enrollments = await Enrollment.countDocuments(
{ user: user._id }
);
} catch (e) {
tests = 0;
enrollments = 0;
}
return {
_id: user._id,
name: user.name,
address: user.address,
tests: tests,
enrollments: enrollments,
};
})
);
return res.json({
users: usersWithCount,
});
} catch (err) {
next(err);
}
};
I want to know if this is a good way to do it.
The code works. But i am concerned about the performance and load it will put on my server.
I think you might consider use the MongoDB aggregation framework. You can combine user data with trial and registration data using the $lookup operator to join the user data with the tests and enrollments data, and then use the $group operator to get the count of tests and enrollments for each user. This will reduce the number of database queries and improve the performance of your API.
const aggregate = User.aggregate([
{
$lookup: {
from: "tests",
localField: "_id",
foreignField: "user",
as: "tests"
}
},
{
$lookup: {
from: "enrollments",
localField: "_id",
foreignField: "user",
as: "enrollments"
}
},
{
$project: {
_id: 1,
name: 1,
address: 1,
tests: { $size: "$tests" },
enrollments: { $size: "$enrollments" }
}
}
]);
const usersWithCount = await aggregate.exec();
return res.json({
users: usersWithCount,
});
I am using mongo and nodejs.
Current I have the following collections:
entryMore {
entryId
...
}
filesize {
entryId
...
}
entryIam {
entryId
...
}
entryStatus {
entryId
...
}
entryPubStatus {
entryId
....
}
They are all one to one mapping with entryId
I am joining all of the collections in nodejs with the following code
// db
const { MongoClient } = require('mongodb');
const fs = require('fs');
const Json2csvParser = require('json2csv').Parser;
// run catch err
run()
.catch(error => console.error(error.stack));
async function run() {
// connect
const client = await MongoClient.connect('mongodb://user:pass#127.0.0.1:27017');
// db
const db = client.db('kal');
// The only way to use `$lookup` in MongoDB 3.2 and 3.4
const docs = await db.collection('entryMore').aggregate([
// filesize
{
$lookup: {
from: 'filesize',
localField: 'entryId',
foreignField: 'entryId',
as: 'filesize'
}
},
{
// deconstruct and map to one by one
$unwind: '$filesize'
},
// Iam
{
$lookup: {
from: 'entryIam',
localField: 'entryId',
foreignField: 'entryId',
as: 'entryIam'
}
},
{
$unwind: '$entryIam'
},
// entry status
{
$lookup: {
from: 'entryStatus',
localField: 'entryId',
foreignField: 'entryId',
as: 'entryStatus'
}
},
{
$unwind: '$entryStatus'
},
// pub status
{
$lookup: {
from: 'entryPubStatus',
localField: 'entryId',
foreignField: 'entryId',
as: 'entryPubStatus'
}
},
{
$unwind: '$entryPubStatus'
},
// Final
{
$project: {
_id: 0,
entryId: 1,
name: 1,
//description: 1,
userId: 1,
creatorId: 1,
msDuration: 1,
categories: 1,
categoriesIds: 1,
createdAt: 1,
updatedAt: 1,
tags: 1,
downloadUrl: 1
}
}
]);
const json2csvParser = new Json2csvParser({field});
const csv = json2csvParser.parse(docs);
await writeFile('./csv/entrySheet.csv', csv);
console.log('done!');
process.exit();
}
Each collection has 97k records. It takes whole date to generate the csv. I wonder is there any way to improve it?
I agree with what is written by #NeilLunn. But to still clarify your questions, $lookup performs an equality match on the foreignField to the localField from the input documents. If a document in the from collection does not contain the foreignField, the $lookup treats the value as null for matching purposes.
So go ahead and create an index on your foreign field so you'll be doing 97k index hits and not 97k table scans. It is bound to get the time way lower
I have to mongodb collections, like this:
UserGroup collection
fields:
name: String
group_id: Number
User collection
fields:
user_name: String
group_id: Number
I want generate a report like this:
ADMINISTRATORS
--------------------------
jlopez
rdiaz
OPERATORS
--------------------------
amiralles
dcamponits
But I get the following report:
ADMINISTRATORS
--------------------------
OPERATORS
--------------------------
jlopez
rdiaz
amiralles
dcamponits
Following is the code to generate the report:
UserGroup.find({}, (err, groups) => {
for(var i in groups){
console.log(groups[i].name)
console.log("--------------------")
User.find( {group_id : groups[i].group_id}, (err, users) =>{
for(var j in users){
console.log(users[j].user_name)
}
})
}
})
Clearly, this is a problem of the NodeJs/Mongoose asynchronicity.
QUESTION: How do I make the first For cycle wait until the internal cycle ends for each UserGrop?
Thanks in advance,
David.
You can run an aggregation pipeline that uses $lookup to do a "left-join" to another collection in the same database to filter in documents from the "joined" collection for processing. With this you won't need any async library:
UserGroup.aggregate([
{
"$lookup": {
"from": "users",
"localField": "group_id",
"foreignField": "group_id",
"as": "users"
}
},
{
"$project": {
"_id": 0,
"name": 1,
"users": {
"$map": {
"input": "$users",
"as": "user",
"in": "$$user.user_name"
}
}
}
}
], (err, groups) => {
if (err) throw err;
console.log(JSON.stringify(groups, null, 4));
})
Sample Output
[
{
"name": "ADMINISTRATORS",
"users": ["jlopez", "rdiaz"]
},
{
"name": "OPERATORS",
"users": ["amiralles", "dcamponits"]
}
]
Add support for promises to mongoose. I use q, but you can use bluebird too.
mongoose.Promise = require('q').Promise;
Then you can use q.all to resolve once all of the user queries have completed.
var promises = [];
var groups = [];
UserGroup.find({}, (err, groups) => {
for(var i in groups){
groups.push(groups[i]);
promises.push(User.find( {group_id : groups[i].group_id}));
}
});
q.all(promises).then( function(usersByGroup){
var indx = 0;
usersByGroup.forEach(function(users){
var grp = groups[indx];
console.log(groups[i].name);
console.log("--------------------");
for(var j in users){
console.log(users[j].user_name)
}
indx++;
});
});
This is a good use case for asyc, you can get a get basic idea from following code. it is based on async each & waterfall. [ Please add proper error handling for the following code yourself.]
UserGroup.find({}, (err, groups) => {
async.each(groups, (group, callback) =>{
async.waterfall([
(wCallback) => {
User.find({group_id : group.group_id}, wCallback)
},
(users, wCallback) => {
console.log(group.name)
console.log("--------------------")
for(var j in users){
console.log(users[j].user_name)
}
wCallback()
}
], callback)
})
})
Still a NooB to Node/Mongo and am stuck on this.
I have two mongo collections, Tenants and Rent. Rent collection has the tenant _id in the schema. The following function is searching through all active tenants and for each of those pulling out some attributes for the latest rent document.
The first part of the function populates the tenant object with the results. all working good.
The second .then starts to iterate through the tenant object pulling out the _id to use in the Rent query. (join).
the issue is the for loop seems to iterate through and print the _id correctly, but the second find query seems to only print out the last document in the object. I am just not sure why this is happening
thanks in advance
app.get('/chargerenttest/:date', (req,res) => {
//check date is valid
var rentChargeDate = new Date(req.params.date);
var tenant = "";
//for each active tenant
Tenant .find({
activeTenant : true
})
.then ((tenant) => {
if (!tenant) {
return res.status(404).send();
}
console.log("found tenents")
return tenant
})
.then ((tenant) => {
for (var i in tenant) {
console.log(i)
console.log(tenant[i]._id)
Rent
.find({
"paymentType" :"Rent",
"tenantID" : tenant[i]._id,
"activeEntry": true})
.limit(1)
.sort({datePaid: -1})
// sort in decending date ( latested on top)
.then ((rent) => {
lastPayment = rent[0].datePaid;
lastAmountPaid = rent[0].amountPaid;
console.log("--------",i)
console.log("tenant",tenant[i]._id)
console.log("rentamount",tenant[i].rentAmount)
console.log("lastpayment", lastPayment)
});
}
})
})
Your query can be simplified by running an aggregate operation that makes use of a pipeline with the $lookup operator which allows you to perform a left outer join to another collection in the same database to filter in documents from the "joined" collection for processing.
Consider running the following pipeline:
Rent.aggregate([
{
"$match": {
"paymentType": "Rent",
"activeEntry": true
}
},
{
"$lookup": {
"from": "tenants",
"localField": "tenantID",
"foreignField": "_id",
"as": "tenants"
}
},
{ "$match": { "tenants": { "$ne": [] }, "tenants.activeTenant": true } },
//{ "$unwind": "$tenants" },
{ "$sort": { "datePaid": -1 } },
{ "$limit": 1 }
]).exec((err, rent) => {
if (err) throw err;
lastPayment = rent[0].datePaid;
lastAmountPaid = rent[0].amountPaid;
tenant = rent[0].tenants[0];
console.log("tenant",tenant._id)
console.log("rentamount",tenant.rentAmount)
console.log("lastpayment", lastPayment)
});
This module #coolgk/mongo could make joining multiple collections a lot simpler.
Examples
SQL to Mongo Join
Left Join
SELECT * FROM a LEFT JOIN b ON a.b_id = b.id
becomes
model.find({}, {
join: [ { on: 'b_id' } ]
})
Result:
[{
_id: '5a8bde4ae2ead929f89f3c42',
a_name: 'aname1',
b_id: {
_id: '5a8bde4ae2ead929f89f3c41',
b_name: 'bname1'
}
}, { ... }, ... ]
Inner Join with Constraints
SELECT * FROM a, b WHERE a.b_id = b.id AND b.b_name = 'bname1'
becomes
model.find({}, {
join: [ { on: 'b_id', filters: { b_name: 'bname1' } } ]
})
Result:
[{
_id: '5a8bdfb05d44ea2a08fa8a4c',
a_name: 'aname2',
b_id: {
_id: '5a8bdfb05d44ea2a08fa8a4b',
b_name: 'bname2'
}
}]
Inner Join on Mulitple Collections
SELECT * FROM a, b, c WHERE a.b_id = b.id AND b.c_id = c.id AND c.c_name = 'cname3'
becomes
modela.find({}, {
join: [{
on: 'b_id',
join: [{
on: 'c_id',
filters: { c_name: 'cname3' }
}]
}]
})
Result:
[{
_id: '5a8bdfc1b07af22a12cb1f0b',
a_name: 'aname3',
b_id: {
_id: '5a8bdfc1b07af22a12cb1f0a',
b_name: 'bname3',
c_id: {
_id: '5a8bdfc1b07af22a12cb1f09',
c_name: 'cname3'
}
}
}]
I've been building an application in Node.JS using the native MongoDB driver - it includes contacts, and when a user accepts a contact, it should remove from "pending" and "sent" contacts, then add to "contacts".
Example code and documents:
/*
=============================
User "john"
=============================
{
username: "john",
contacts: ["jim"],
pending_contacts: ["bob"]
}
=============================
User "bob"
=============================
{
username: "bob",
contacts: ["dave"],
sent_contacts: ["john"]
}
=============================
What SHOULD happen
=============================
{
username: "bob",
contacts: ["dave", "john"],
sent_contacts: []
},
{
username: "john",
contacts: ["jim", "bob"],
pending_contacts: []
}
=============================
What ACTUALLY happens
=============================
{
username: "john",
contacts: ["jim", "bob"],
pending_contacts: ["bob"]
},
{
username: "bob",
contacts: ["dave", "john"],
sent_contacts: ["john"]
}
*/
var col = this.db.collection('users');
var contact = "bob", username = "john";
var who = [contact, username];
var finishCount = 0;
// finish will run 3 times before callback
function finish(name) {
console.log(name, ' has finished');
finishCount++;
if(finishCount<3) return;
callback(false, null);
}
// run if there's an error
function failed(err) {
callback(err, null)
}
console.log('removing %s and %s from pending and sent', username, contact)
col.update(
{username: { $in: who }},
{
$pullAll: {
sent_contacts: who,
pending_contacts: who
}
}, {multi: 1},
function(err,data) {
if(err) return failed(err);
finish('REMOVE_CONTACTS');
}
);
col.update(
{username: username}, {$addToSet: {contacts: contact}},
function(err,res) {
if(err) return failed(err);
console.log('added 1');
finish('ADD_TO_USER');
}
);
col.update(
{username: contact}, {$addToSet: {contacts: username}},
function(err,res) {
if(err) return failed(err);
console.log('added 2');
finish('ADD_TO_CONTACT');
}
);
The first update removes the contact and the owner from each-others pending/sent list, the second and third update add the owner to the contact's contact list and vice versa.
The issue is, the final result appears to be as if the removal never happened, though the removal query works perfectly fine by itself. I don't know if this is a problem with MongoDB itself (or if it's intended), or if it's an issue with the driver, so I hope someone can at least clarify this for me.
NOTE: Yes I know they run asynchronously. Running them one after the other by putting each update in the previous callback does NOT make a difference. Before anyone complains about how awful this code looks, I previously had it set up within Async.JS but I removed it from this code sample to ensure that Asyn.cJS was not responsible for the issues.
Using the node native driver this works for me every time:
var mongodb = require('mongodb'),
async = require('async'),
MongoClient = mongodb.MongoClient;
var user = "john",
contact = "bob";
var contactsList = [
{
"username": "john",
"contacts": [
"jim"
],
"pending_contacts": [
"bob"
]
},
{
"username": "bob",
"contacts": [
"dave"
],
"sent_contacts": [
"john"
]
}
];
MongoClient.connect('mongodb://localhost/test',function(err,db) {
var coll = db.collection("contacts");
async.series(
[
// Wipe clean
function(callback) {
coll.remove({},callback)
},
// Init collection
function(callback) {
async.each(contactsList,function(contact,callback) {
coll.insert(contact,callback);
},callback);
},
// Do updates
function(callback) {
// Init batch
var bulk = coll.initializeOrderedBulkOp();
// Add to user and pull from pending
bulk.find({
"username": user,
"contacts": { "$ne": contact },
}).updateOne({
"$push": { "contacts": contact },
"$pull": { "pending_contacts": contact }
});
// Add to contact and pull from sent
bulk.find({
"username": contact,
"contacts": { "$ne": user },
"sent_contacts": user
}).updateOne({
"$push": { "contacts": user },
"$pull": { "sent_contacts": user }
});
// Execute
bulk.execute(function(err,response) {
console.log( response.toJSON() );
callback(err);
});
},
// List collection
function(callback) {
coll.find({}).toArray(function(err,results) {
console.log(results);
callback(err);
});
}
],
function(err) {
if (err) throw err;
db.close();
}
);
});
And the output:
{ ok: 1,
writeErrors: [],
writeConcernErrors: [],
insertedIds: [],
nInserted: 0,
nUpserted: 0,
nMatched: 2,
nModified: 2,
nRemoved: 0,
upserted: [] }
[ { _id: 55b0c16934fadce812cdcf9d,
username: 'john',
contacts: [ 'jim', 'bob' ],
pending_contacts: [] },
{ _id: 55b0c16934fadce812cdcf9e,
username: 'bob',
contacts: [ 'dave', 'john' ],
sent_contacts: [] } ]
Improvements here are basically to use the Bulk Operations API and send all updates at once to the server and get a single response. Also note the use of operators in the updates and the query selection as well.
Simply put, you already know the "user" as well as the "contact" they are accepting. The contact to be accepted is "pending" and the contact themselves have the user in "sent".
These are really just simple $push and $pull operations on either array as is appropriate. Rather than using $addToSet here, the query conditions make sure that the expected values are present when performing the update. This also preserves "order" which $addToSet can basically not guarantee, because it's a "set", which is un-ordered.
One send to the server and one callback response, leaving both users updated correctly. Makes more sense then sending multiple updates and waiting for the callback response from each.
Anyhow, this is a complete self contained listing with only the two named dependencies, so you can easily run it yourself and confirm the results.
When I say "Complete and self contained" it means start a new project and simply run the code. Here's the complete instruction:
mkdir sample
cd sample
npm init
npm install mongodb --save
npm install async --save
Then create a file with the code listing in that folder, say test.js and then run:
node test.js