How to export the value from the promise - node.js

I am fairly new to node and mongoose and I am having some issues with module.exports. While trying to make my code a bit more clean I have been trying to use modules to take the mongoDB queries since they take too much space and it does not that easy to read, but I got an error.
Here I am merging 2 collections to get a result but the console on index.js tells me undefined. When I do a console.log on operations.js and then execute on index, the console from inside return the array but not the one outside.
After a bit of reading I see that is because the value i want is inside of the promise, but how can I get it back then?
operations.js
const express = require('express')
const mongoose = require('mongoose')
const CheckOutOrder = require('../models/CheckOutOrder')
function ordersQuery(){
console.log('lol')
CheckOutOrder.aggregate([
{
$match:{
'is_open':1
}
},
{
$addFields:{
'itemToMatch':{
$toObjectId:'$idItem'
}
}
},
{
$lookup:{
from: 'e_menu_items',
localField: 'itemToMatch',
foreignField: '_id',
as: 'itemsReceipt'
}
},
{
$unwind:'$itemsReceipt'
}
]).then(info => {
console.log(info)
return info
}).catch(e => {
console.log('e')
return e
})
}
module.exports.totOrders = ordersQuery
index.js
const mongoQueries = require('../mongo/operations')
if(ActionDOM=='test'){
let orders = []
orders = mongoQueries.totOrders()
console.log('------------------------------------------------------')
console.log(orders)
}
console
lol
------------------------------------------------------
undefined
[
{
_id: new ObjectId("61da4036bc966b2a284e8075"),
idReceipt: 4,
idUser: '61d1606420f281d51d52e2e1',
idItem: '61d7f9205bda9c3be60b14ed',
is_open: 1,
date_added: 2022-01-09T01:53:58.017Z,
__v: 0,
itemToMatch: new ObjectId("61d7f9205bda9c3be60b14ed"),
itemsReceipt: {
_id: new ObjectId("61d7f9205bda9c3be60b14ed"),
menuItem: 'Molletes',
description: 'Frijoles, queso y salsa',
price: 18,
id_cat: 1,
is_active: 1,
date_added: 2022-01-07T08:26:08.990Z,
date_modified: 2022-01-07T08:26:08.990Z,
__v: 0
}
},
{
_id: new ObjectId("61da4036bc966b2a284e8079"),
idReceipt: 4,
idUser: '61d1606420f281d51d52e2e1',
idItem: '61d7f9205bda9c3be60b14ed',
is_open: 1,
date_added: 2022-01-09T01:53:58.128Z,
__v: 0,
itemToMatch: new ObjectId("61d7f9205bda9c3be60b14ed"),
itemsReceipt: {
_id: new ObjectId("61d7f9205bda9c3be60b14ed"),
menuItem: 'Molletes',
description: 'Frijoles, queso y salsa',
price: 18,
id_cat: 1,
is_active: 1,
date_added: 2022-01-07T08:26:08.990Z,
date_modified: 2022-01-07T08:26:08.990Z,
__v: 0
}
}
]
What am I doing wrong?

You can use async/await to make the code more "linear":
async function ordersQuery() {
console.log('lol');
try {
const info = await CheckOutOrder.aggregate([
{
$match: {
is_open: 1,
},
},
{
$addFields: {
itemToMatch: {
$toObjectId: '$idItem',
},
},
},
{
$lookup: {
from: 'e_menu_items',
localField: 'itemToMatch',
foreignField: '_id',
as: 'itemsReceipt',
},
},
{
$unwind: '$itemsReceipt',
},
]);
console.log(info);
return info;
} catch (e) {
console.log(e);
return e;
}
}
and then call it with await too (the calling function must be declared async):
async function someFunc(/*....*/) {
// ...
if (ActionDOM == 'test') {
let orders = [];
orders = await mongoQueries.totOrders();
console.log('------------------------------------------------------');
console.log(orders);
}
// ...
}

Related

Update MongoDb data inside a field

I have a MongoDb Database with a collection called rooms. In rooms, I want to search for a particular object by the roomId property. I want to update the array contents in the found object. For instance, initially, before making a request to that endpoint, the desired data looks like this:
{
available: true,
_id: 60817a403170bf49185c7db7,
player1: "Jack",
player2: "Adam",
roomId: "ABCDE",
pieces: [
{
point: [6, 0],
player: "1",
type: "P"
},
...
],
__v: 0
}
After making a request like http://localhost:8000/room/ABCDE?x1=6&y1=0&x2=4&y2=0, the data in mongodb should update to
{
available: true,
_id: 60817a403170bf49185c7db7,
player1: "Jack",
player2: "Adam",
roomId: "ABCDE",
pieces: [
{
point: [4, 0], /* DATA UPDATED */
player: "1",
type: "P"
},
...
],
__v: 0
}
This is my api.js file, what should the code be in the comments in order to execute this?
const express = require('express');
const GameRoom = require('../models/room');
const router = express.Router();
router.put('/room/:id', (req, res, next) => {
GameRoom.findOne({roomId: req.params.id})
.then(data => {
const pieces = data.pieces;
for (let i = 0; i < pieces.length; i++) {
if (pieces[i].point[0] === parseInt(req.query.x1) &&
pieces[i].point[1] === parseInt(req.query.y1)) {
pieces[i].point = [req.query.x2, req.query.y2];
break;
}
}
// Take the modified pieces array and update it
}).catch(next);
});
You can use $[<identifier>] to find and update the points:
GameRoom.updateOne(
{ roomId: req.params.id },
{ $set: { "pieces.$[el].point.0": req.query.x2, "pieces.$[el].point.1": req.query.y2 }},
{ arrayFilters: [ { "el.point.0": req.query.x1, "el.point.1": req.query.y1 } ]}
);
Or if you just want to update the first pieces that match the condition then you can use $elemMatch with $ operator:
GameRoom.updateOne(
{
roomId: req.params.id,
pieces: {$elemMatch: {"point.0": req.query.x1, "point.1": req.query.y1}}}
},
{ $set: {"pieces.$.point.0": req.query.x2, "pieces.$.point.1": req.query.y2} }
);

fetching data from mongodb and find closest value

My code look like this:
const express = require('express');
const mongoose= require('mongoose');
const Schema = mongoose.Schema;
const ourDataSchema = new Schema ({
rank : Number,
totalPoints : Number
});
const rankTotalpoint = mongoose.model("rankTotalpoint", ourDataSchema);
const ourData = [
{rank :1, totalPoints : 2000},
{rank :2, totalPoints : 1980},
{rank :3, totalPoints : 1940},
{rank :4, totalPoints : 1890},
{rank :5, totalPoints : 1830},
{rank :6, totalPoints : 1765}
];
rankTotalpoint.create(ourData, function (error) {
console.log('saved!');
if (error) {
console.log(error)
}
});
...
I have 2 questions:
How can I fetch a rank by it's totalPoint?
I tried find() but it didn't work, I guess I'm using it in a wrong way.
I'm getting an input(Number) from user, I want to match the number with the totalPoint(if exists) that we fetched from our database and return it's rank, and if the exact number didn't exist, I want to match it with the closest totalPoint in our database and return the rank as a response.
Please at least answer my first question!
Highly appreciate your answers guys,
I'm STUCK!
These are my controller file codes where I'm getting the user input and passed it to the code that you send me which I put them in my model files.
const express = require('express');
const model = require('../model/logic');
exports.index = (req, res, next) => {
res.status(200).json({message : 'INSERT INPUTS HERE'});
};
exports.getUserData = (req, res, next) => {
const literature = req.body.literature * 4;
const arabic = req.body.arabic * 2;
const religion = req.body.religion * 3;
const english = req.body.english * 2;
const math = req.body.math * 4;
const physics = req.body.physics * 3;
const chemistry = req.body.chemistry *2;
const TOTALPOINT = literature + arabic + religion + english + math + physics + chemistry;
let result = model.result(TOTALPOINT);
res.status(200).json(result);
};
And this is my model/logic file which I imported them into controller above:
const express = require('express');
const mongoose= require('mongoose');
const Schema = mongoose.Schema;
const ourDataSchema = new Schema ({
rank : Number,
totalPoints : Number
});
const rankTotalpoint = mongoose.model("rankTotalpointData", ourDataSchema);
const ourData = [
{rank : 1, totalPoints : 2000},
{rank : 2, totalPoints : 1980},
{rank: 3, totalPoints : 1940},
{rank:4, totalPoints : 1890},
{rank :5, totalPoints : 1830},
{rank : 6, totalPoints : 1765},
{rank : 7, totalPoints : 1600}
];
rankTotalpoint.create(ourData, function (error, data) {
if (error) {
console.log(error)
}
else {
console.log('saved!');
}
});
exports.result = function (param) {
const finalResult = rankTotalpoint.aggregate([
{
$project: {
diff: {
$abs: {
$subtract: [
param, // <<<----------------------- THIS IS THE USER SUPPLIED VALUE
"$totalPoints"
]
}
},
doc: "$$ROOT"
}
},
{
$sort: {
diff: 1
}
},
{
$limit: 1
},
{
$project: {
_id: 0,
rank: "$doc.rank"
}
}
])
return finalResult;
}
When I'm testing my app with postman I get this response in there :
{
"_pipeline": [
{
"$project": {
"diff": {
"$abs": {
"$subtract": [
16,
"$totalPoints"
]
}
},
"doc": "$$ROOT"
}
},
{
"$sort": {
"diff": 1
}
},
{
"$limit": 1
},
{
"$project": {
"_id": 0,
"rank": "$doc.rank"
}
}
],
"options": {}
}
Question #1:
To find a rank by its totalPoint, you can do:
You can check out a live demo here
db.collection.find({
totalPoints: 2000 // <<<------------------ The exact value you want to find
},
{
_id: 0,
rank: 1
})
Question #2:
To find closest rank, by a user supplied totalPoints value, you should be able to use the following query...
You can check out a live demo here
db.collection.aggregate([
{
$project: {
diff: {
$abs: {
$subtract: [
1800, // <<<----------------------- THIS IS THE USER SUPPLIED VALUE
"$totalPoints"
]
}
},
doc: "$$ROOT"
}
},
{
$sort: {
diff: 1
}
},
{
$limit: 1
},
{
$project: {
_id: 0,
rank: "$doc.rank"
}
}
])
UPDATE/FINAL ANSWER:
Your issue is because Mongoose is promise/async based. You are not awaiting anything, so your code returns a variable that has not been set yet by your query..
I was testing using 2 files: myMongoose.js and index.js..
// myMongoose.js
// ** CODE THAT SAVES DATA TO DATABASE HAS BEEN REMOVED FOR BREVITY **
require('dotenv').config();
const mongoose = require('mongoose');
const RankTotalpointSchema = new mongoose.Schema({
rank: Number,
totalPoints: Number
});
mongoose.set('useCreateIndex', true);
const mongoConnection = mongoose.createConnection(process.env.MONGO_DB_STRING, {
useUnifiedTopology: true,
useNewUrlParser: true,
useFindAndModify: false,
});
const RankTotalpoint = mongoConnection.model("RankTotalpoint", RankTotalpointSchema, 'Testing');
/**
* ~~~~~~ **** THIS HAS TO BE AN ASYNC FUNCTION **** ~~~~~~
*/
exports.result = async function (param) {
const finalresult = await RankTotalpoint.aggregate([{
$project: {
diff: {
$abs: {
$subtract: [
param, // <<<----------------------- THIS IS THE USER SUPPLIED VALUE
"$totalPoints"
]
}
},
doc: "$$ROOT"
}
},
{
$sort: {
diff: 1
}
},
{
$limit: 1
},
{
$project: {
_id: 0,
rank: "$doc.rank"
}
}
])
return finalresult;
};
...and then in index.js:
// index.js
const { result } = require('./myMongoose');
// Use it like this:
async function init() {
try {
const d = await result(1800);
console.log(d);
} catch (err) {
console.error(err);
}
}
init(); // -> [ { rank: 5 } ]
// --------------------------------------------------------------------
// ...or like this:
(async () => {
try {
const d = await result(1800);
console.log(d); // -> [ { rank: 5 } ]
} catch (err) {
console.error(err);
}
})()
// --------------------------------------------------------------------
// ...or like this:
result(1800)
.then(d => console.log(d)) // -> [ { rank: 5 } ]
.catch(err => console.error(err))

Cannot read property 'ClientSession' of undefined [duplicate]

I am using MongoDB Atlas cloud(https://cloud.mongodb.com/) and Mongoose library.
I tried to create multiple documents using transaction concept, but it is not working.
I am not getting any error. but, it seems rollback is not working properly.
app.js
//*** more code here
var app = express();
require('./models/db');
//*** more code here
models/db.js
var mongoose = require( 'mongoose' );
// Build the connection string
var dbURI = 'mongodb+srv://mydb:pass#cluster0-****.mongodb.net/mydb?retryWrites=true';
// Create the database connection
mongoose.connect(dbURI, {
useCreateIndex: true,
useNewUrlParser: true,
});
// Get Mongoose to use the global promise library
mongoose.Promise = global.Promise;
models/user.js
const mongoose = require("mongoose");
const UserSchema = new mongoose.Schema({
userName: {
type: String,
required: true
},
pass: {
type: String,
select: false
}
});
module.exports = mongoose.model("User", UserSchema, "user");
myroute.js
const db = require("mongoose");
const User = require("./models/user");
router.post("/addusers", async (req, res, next) => {
const SESSION = await db.startSession();
await SESSION.startTransaction();
try {
const newUser = new User({
//*** data for user ***
});
await newUser.save();
//*** for test purpose, trigger some error ***
throw new Error("some error");
await SESSION.commitTransaction();
//*** return data
} catch (error) {
await SESSION.abortTransaction();
} finally {
SESSION.endSession();
}
});
Above code works without error, but it still creates user in the DB. It suppose to rollback the created user and the collection should be empty.
I don't know what I have missed here. Can anyone please let me know whats wrong here?
app, models, schema and router are in different files.
You need to include the session within the options for all read/write operations which are active during a transaction. Only then are they actually applied to the transaction scope where you are able to roll them back.
As a bit more complete listing, and just using the more classic Order/OrderItems modelling which should be pretty familiar to most people with some relational transactions experience:
const { Schema } = mongoose = require('mongoose');
// URI including the name of the replicaSet connecting to
const uri = 'mongodb://localhost:27017/trandemo?replicaSet=fresh';
const opts = { useNewUrlParser: true };
// sensible defaults
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
mongoose.set('useFindAndModify', false);
mongoose.set('useCreateIndex', true);
// schema defs
const orderSchema = new Schema({
name: String
});
const orderItemsSchema = new Schema({
order: { type: Schema.Types.ObjectId, ref: 'Order' },
itemName: String,
price: Number
});
const Order = mongoose.model('Order', orderSchema);
const OrderItems = mongoose.model('OrderItems', orderItemsSchema);
// log helper
const log = data => console.log(JSON.stringify(data, undefined, 2));
// main
(async function() {
try {
const conn = await mongoose.connect(uri, opts);
// clean models
await Promise.all(
Object.entries(conn.models).map(([k,m]) => m.deleteMany())
)
let session = await conn.startSession();
session.startTransaction();
// Collections must exist in transactions
await Promise.all(
Object.entries(conn.models).map(([k,m]) => m.createCollection())
);
let [order, other] = await Order.insertMany([
{ name: 'Bill' },
{ name: 'Ted' }
], { session });
let fred = new Order({ name: 'Fred' });
await fred.save({ session });
let items = await OrderItems.insertMany(
[
{ order: order._id, itemName: 'Cheese', price: 1 },
{ order: order._id, itemName: 'Bread', price: 2 },
{ order: order._id, itemName: 'Milk', price: 3 }
],
{ session }
);
// update an item
let result1 = await OrderItems.updateOne(
{ order: order._id, itemName: 'Milk' },
{ $inc: { price: 1 } },
{ session }
);
log(result1);
// commit
await session.commitTransaction();
// start another
session.startTransaction();
// Update and abort
let result2 = await OrderItems.findOneAndUpdate(
{ order: order._id, itemName: 'Milk' },
{ $inc: { price: 1 } },
{ 'new': true, session }
);
log(result2);
await session.abortTransaction();
/*
* $lookup join - expect Milk to be price: 4
*
*/
let joined = await Order.aggregate([
{ '$match': { _id: order._id } },
{ '$lookup': {
'from': OrderItems.collection.name,
'foreignField': 'order',
'localField': '_id',
'as': 'orderitems'
}}
]);
log(joined);
} catch(e) {
console.error(e)
} finally {
mongoose.disconnect()
}
})()
So I would generally recommend calling the variable session in lowercase, since this is the name of the key for the "options" object where it is required on all operations. Keeping this in the lowercase convention allows for using things like the ES6 Object assignment as well:
const conn = await mongoose.connect(uri, opts);
...
let session = await conn.startSession();
session.startTransaction();
Also the mongoose documentation on transactions is a little misleading, or at least it could be more descriptive. What it refers to as db in the examples is actually the Mongoose Connection instance, and not the underlying Db or even the mongoose global import as some may misinterpret this. Note in the listing and above excerpt this is obtained from mongoose.connect() and should be kept within your code as something you can access from a shared import.
Alternately you can even grab this in modular code via the mongoose.connection property, at any time after a connection has been established. This is usually safe inside things such as server route handlers and the like since there will be a database connection by the time that code is called.
The code also demonstrates the session usage in the different model methods:
let [order, other] = await Order.insertMany([
{ name: 'Bill' },
{ name: 'Ted' }
], { session });
let fred = new Order({ name: 'Fred' });
await fred.save({ session });
All the find() based methods and the update() or insert() and delete() based methods all have a final "options block" where this session key and value are expected. The save() method's only argument is this options block. This is what tells MongoDB to apply these actions to the current transaction on that referenced session.
In much the same way, before a transaction is committed any requests for a find() or similar which do not specify that session option do not see the state of the data whilst that transaction is in progress. The modified data state is only available to other operations once the transaction completes. Note this has effects on writes as covered in the documentation.
When an "abort" is issued:
// Update and abort
let result2 = await OrderItems.findOneAndUpdate(
{ order: order._id, itemName: 'Milk' },
{ $inc: { price: 1 } },
{ 'new': true, session }
);
log(result2);
await session.abortTransaction();
Any operations on the active transaction are removed from state and are not applied. As such they are not visible to resulting operations afterwards. In the example here the value in the document is incremented and will show a retrieved value of 5 on the current session. However after session.abortTransaction() the previous state of the document is reverted. Note that any global context which was not reading data on the same session, does not see that state change unless committed.
That should give the general overview. There is more complexity that can be added to handle varying levels of write failure and retries, but that is already extensively covered in documentation and many samples, or can be answered to a more specific question.
Output
For reference, the output of the included listing is shown here:
Mongoose: orders.deleteMany({}, {})
Mongoose: orderitems.deleteMany({}, {})
Mongoose: orders.insertMany([ { _id: 5bf775986c7c1a61d12137dd, name: 'Bill', __v: 0 }, { _id: 5bf775986c7c1a61d12137de, name: 'Ted', __v: 0 } ], { session: ClientSession("80f827fe077044c8b6c0547b34605cb2") })
Mongoose: orders.insertOne({ _id: ObjectId("5bf775986c7c1a61d12137df"), name: 'Fred', __v: 0 }, { session: ClientSession("80f827fe077044c8b6c0547b34605cb2") })
Mongoose: orderitems.insertMany([ { _id: 5bf775986c7c1a61d12137e0, order: 5bf775986c7c1a61d12137dd, itemName: 'Cheese', price: 1, __v: 0 }, { _id: 5bf775986c7c1a61d12137e1, order: 5bf775986c7c1a61d12137dd, itemName: 'Bread', price: 2, __v: 0 }, { _id: 5bf775986c7c1a61d12137e2, order: 5bf775986c7c1a61d12137dd, itemName: 'Milk', price: 3, __v: 0 } ], { session: ClientSession("80f827fe077044c8b6c0547b34605cb2") })
Mongoose: orderitems.updateOne({ order: ObjectId("5bf775986c7c1a61d12137dd"), itemName: 'Milk' }, { '$inc': { price: 1 } }, { session: ClientSession("80f827fe077044c8b6c0547b34605cb2") })
{
"n": 1,
"nModified": 1,
"opTime": {
"ts": "6626894672394452998",
"t": 139
},
"electionId": "7fffffff000000000000008b",
"ok": 1,
"operationTime": "6626894672394452998",
"$clusterTime": {
"clusterTime": "6626894672394452998",
"signature": {
"hash": "AAAAAAAAAAAAAAAAAAAAAAAAAAA=",
"keyId": 0
}
}
}
Mongoose: orderitems.findOneAndUpdate({ order: ObjectId("5bf775986c7c1a61d12137dd"), itemName: 'Milk' }, { '$inc': { price: 1 } }, { session: ClientSession("80f827fe077044c8b6c0547b34605cb2"), upsert: false, remove: false, projection: {}, returnOriginal: false })
{
"_id": "5bf775986c7c1a61d12137e2",
"order": "5bf775986c7c1a61d12137dd",
"itemName": "Milk",
"price": 5,
"__v": 0
}
Mongoose: orders.aggregate([ { '$match': { _id: 5bf775986c7c1a61d12137dd } }, { '$lookup': { from: 'orderitems', foreignField: 'order', localField: '_id', as: 'orderitems' } } ], {})
[
{
"_id": "5bf775986c7c1a61d12137dd",
"name": "Bill",
"__v": 0,
"orderitems": [
{
"_id": "5bf775986c7c1a61d12137e0",
"order": "5bf775986c7c1a61d12137dd",
"itemName": "Cheese",
"price": 1,
"__v": 0
},
{
"_id": "5bf775986c7c1a61d12137e1",
"order": "5bf775986c7c1a61d12137dd",
"itemName": "Bread",
"price": 2,
"__v": 0
},
{
"_id": "5bf775986c7c1a61d12137e2",
"order": "5bf775986c7c1a61d12137dd",
"itemName": "Milk",
"price": 4,
"__v": 0
}
]
}
]
I think this is the quickest way to start performing transaction with mongoose
const mongoose = require("mongoose");
// starting session on mongoose default connection
const session = await mongoose.startSession();
mongoose.connection.transaction(async function executor(session) {
try {
// creating 3 collections in isolation with atomicity
const price = new Price(priceSchema);
const variant = new Variant(variantSchema);
const item = new Item(itemSchema);
await price.save({ session });
await variant.save({ session });
// throw new Error("opps some error in transaction");
return await item.save({ session });
} catch (err) {
console.log(err);
}
});

merge aggregation and find results in MongoDB

I have this model:
const recordSchema = new Schema({
user: {
type: Schema.ObjectId,
ref: 'Person',
required: true
},
date: {
type: Date,
default: Date.now()
},
time: {
type: Date,
default: Date.now()
}
});
So, when I make a HTTP Get request I receive an array of records:
[{/*record1*/}, {/*record2*/}, ...]
The point is that I'm using aggregation to get the number of records of each user (I got that cover), but I would like merge this with the find results to receive something like this:
{
"records": [{/*record1*/}, {/*record2*/}, ...],
"stats": [
{
"_id" : ObjectId("5b6393f2a1d3de31d9547f63"),
"count" : 3.0
},
{
"_id" : ObjectId("5b5d22d8b6195d1b6a5d2574"),
"count" : 17.0
}
]
}
So, how do I get this?
Note: I'm using this for data for some charts, should I handle this on node or on the front-end?
This can be done in the back-end and if you're using MongoDB Server version 3.4.4 or greater in your backend, $facet aggregate pipeline should cover your needs.
Consider the following example which runs two aggregate queries in the same aggregate pipeline using $facet: one returns the records for today and the other returns the counts for each user in the collection:
let start = new Date();
start.setHours(0,0,0,0);
let end = new Date();
end.setHours(23,59,59,999);
Record.aggregate([
{ '$facet': {
'records': [
{ '$match': {
'date': {
'$gte': start,
'$lte': end
}
} }
],
'stats': [
{ '$group': {
'_id': '$user',
'count': { '$sum': 1 }
} }
]
} }
]).exec((err, results) => {
if (err) {
console.error(err);
throw new Error(err);
}
const data = results[0];
console.log(JSON.stringify(data, null, 4));
})
For MongoDB 3.2 and below
1. Using Promises
const recordsQuery = Record.find({ 'date': {
'$gte': start, // date object representing start of today
'$lte': end // date object representing end of today
} }).lean().exec();
const statsQuery = Record.aggregate([
{ '$group': {
'_id': '$user',
'count': { '$sum': 1 }
} }
]).exec();
Promise.all([ recordsQuery, statsQuery ]).then(([ recordsData, statsData ]) => {
const records = recordsData[0];
const stats = statsData[0];
const data = { records, stats };
console.log(JSON.stringify(data, null, 4));
}).catch(err => console.error(err));
2. Using async/await
(async () => {
try {
const recordsQuery = await Record.find({ 'date': {
'$gte': start, // date object representing start of today
'$lte': end // date object representing end of today
} }).lean().exec();
const statsQuery = await Record.aggregate([
{ '$group': {
'_id': '$user',
'count': { '$sum': 1 }
} }
]).exec();
const records = recordsQuery[0];
const stats = statsQuery[0];
const data = { records, stats };
console.log(JSON.stringify(data, null, 4));
} catch (err) {
console.error(err);
}
})();
You can use the $lookup operator to link the original records for the users by Id
{
$lookup:
{
from: <collection to join>,
localField: <field from the input documents>,
foreignField: <field from the documents of the "from" collection>,
as: <output array field>
}
}

Is there any way to make join faster, when dealing large amount of data?

I am using mongo and nodejs.
Current I have the following collections:
entryMore {
entryId
...
}
filesize {
entryId
...
}
entryIam {
entryId
...
}
entryStatus {
entryId
...
}
entryPubStatus {
entryId
....
}
They are all one to one mapping with entryId
I am joining all of the collections in nodejs with the following code
// db
const { MongoClient } = require('mongodb');
const fs = require('fs');
const Json2csvParser = require('json2csv').Parser;
// run catch err
run()
.catch(error => console.error(error.stack));
async function run() {
// connect
const client = await MongoClient.connect('mongodb://user:pass#127.0.0.1:27017');
// db
const db = client.db('kal');
// The only way to use `$lookup` in MongoDB 3.2 and 3.4
const docs = await db.collection('entryMore').aggregate([
// filesize
{
$lookup: {
from: 'filesize',
localField: 'entryId',
foreignField: 'entryId',
as: 'filesize'
}
},
{
// deconstruct and map to one by one
$unwind: '$filesize'
},
// Iam
{
$lookup: {
from: 'entryIam',
localField: 'entryId',
foreignField: 'entryId',
as: 'entryIam'
}
},
{
$unwind: '$entryIam'
},
// entry status
{
$lookup: {
from: 'entryStatus',
localField: 'entryId',
foreignField: 'entryId',
as: 'entryStatus'
}
},
{
$unwind: '$entryStatus'
},
// pub status
{
$lookup: {
from: 'entryPubStatus',
localField: 'entryId',
foreignField: 'entryId',
as: 'entryPubStatus'
}
},
{
$unwind: '$entryPubStatus'
},
// Final
{
$project: {
_id: 0,
entryId: 1,
name: 1,
//description: 1,
userId: 1,
creatorId: 1,
msDuration: 1,
categories: 1,
categoriesIds: 1,
createdAt: 1,
updatedAt: 1,
tags: 1,
downloadUrl: 1
}
}
]);
const json2csvParser = new Json2csvParser({field});
const csv = json2csvParser.parse(docs);
await writeFile('./csv/entrySheet.csv', csv);
console.log('done!');
process.exit();
}
Each collection has 97k records. It takes whole date to generate the csv. I wonder is there any way to improve it?
I agree with what is written by #NeilLunn. But to still clarify your questions, $lookup performs an equality match on the foreignField to the localField from the input documents. If a document in the from collection does not contain the foreignField, the $lookup treats the value as null for matching purposes.
So go ahead and create an index on your foreign field so you'll be doing 97k index hits and not 97k table scans. It is bound to get the time way lower

Resources