If I were to perform this query with mongoose;
Schema.find({
_id: {
$in: ['abcd1234', 'abcd1234', 'abcd1234']
}
});
The query will only return something like:
[{
'property1': 'key1',
'property2': 'key2'
}]
With the array only having one object, obviously because I passed in all the same id's. However, I actually want duplicate objects returned. How can I do this?
Mongo itself will only return objects with no duplicates. But you can then build an array of objects with duplicates from that.
For example, if array is the array of objects returned my Mongo - in this case:
var array = [{
_id: 'abcd1234',
property1: 'key1',
property2: 'key2'
}];
and ids is your list of IDs that you want with duplicates - in your case:
var ids = ['abcd1234', 'abcd1234', 'abcd1234'];
then you can do:
var objects = {};
array.forEach(o => objects[o._id] = o);
var dupArray = ids.map(id => objects[id]);
Now dupArray should contain the objects with duplicates.
Full example:
var ids = ['abcd1234', 'abcd1234', 'abcd1234'];
Schema.find({_id: {$in: ids}}, function (err, array) {
if (err) {
// handle error
} else {
var objects = {};
array.forEach(o => objects[o._id] = o);
var dupArray = ids.map(id => objects[id]);
// here you have objects with duplicates in dupArray:
console.log(dupArray);
}
});
Related
Script logic - script receives data from Binance API > Then I have aggregation $avg to calculate the average of one asset. I will have more than one collection so I need to calculate average of every asset.
I have an array where I store collection names for MongoDB.
const symbols = ["ADABTC", "AEBTC", "AIONBTC"]
And I want to calculate average from MongoDB collection.
const collection = db.collection(symbols);
Here - symbols doesn't work for me, but if I simply add "ADABTC" then it works, but it doesn't fixes my problem since I want to use different collection names one after another one.
How I can pass an array if it's required to be a string? I need to use more than 1 collection names.
FULL CODE
const MongoClient = require('mongodb').MongoClient;
const assert = require('assert');
// Connection URL
const url = 'mongodb://username:password#serveripadress:port/dbname?retryWrites=true&w=majority';
const symbols = ["ADABTC", "AEBTC", "AIONBTC"]
// Database Name
const dbName = 'Crypto';
// Create a new MongoClient
const client = new MongoClient(url, { useUnifiedTopology: true });
// Use connect method to connect to the Server
client.connect(function(err, client) {
assert.equal(null, err);
console.log("Connected correctly to server");
const db = client.db(dbName);
simplePipeline(db, function() {
client.close();
});
});
function simplePipeline(db, callback) {
const collection = db.collection(symbols);
collection.aggregate(
[{
'$group': {
_id: null,
'Volume': {
'$avg': '$Volume'
}
}
}],
function(err, cursor) {
assert.equal(err, null);
cursor.toArray(function(err, documents) {
console.log(documents)
callback(documents);
});
}
);
}
It is not possible to pass an array into a function that is asking for a string. In your case what you need to do is join three collections. If you need to aggregate across multiple collections you can use the $lookup aggregation pipeline operator. You can connect using the first collection:
db.collection(symbols) => db.collection(symbols[0])
Then modify your query to join the three collections:
// Join with AEBTC collection
{
$lookup:{
from: symbols[1],
localField: //name of a value in first collection
foreignField: //name of same value in second collection
as: //what you want to call it in the second table
}
},
{ $unwind: //what you called it },
// Join with AIONBTC collection
{
$lookup:{
from: symbols[2],
localField: //name of value in joined collections 1 and 2
foreignField: //name of that value in collection 3,
as: //whatever you want to call it in the joined collection
}
},
{ $unwind: //what you called it },
// define some conditions here
{
$match {}
},
// define which fields are you want to fetch
{
$group: {
_id: null,
'Volume': {
'$avg': '$Volume'
}
}
I'm using MongoDB in node.js. I'm trying to update or insert many documents based on different conditions; however, MongoDB update (with upsert) only works with a single document or many documents with the same condition. Currently, I have an array containing the objects that I want to insert (Or update if the unique index exists) and I'm looping through the array and calling the updateOnce; however, I believe this method is not very efficient for a large number of objects that I'm going to have.
What is a better way to achieve this?
var mongoUtil = require( './database.js' );
var db = mongoUtil.getDb();
//array of objects to insert:
//NOTE: First + Last name is a unique index
var users = [
{firstName:"John", lastName:"Doe", points:300},
{firstName:"Mickey", lastName:"Mouse", points:200}
];
var collection = db.collection( 'points' );
for(var i = 0; i < users.length; i++) {
//If firstName and lastName already exists, update points. Otherwise insert new object
collection.updateOne(
{firstName: users[i].firstName, lastName: users[i].lastName},
{$set: {points: users[i].points}},
{upsert: true},
function(err,docs) {
if(err)
console.log(err);
}
)
}
I solved this issue using .bulkWrite():
var mongoUtil = require( './database.js' );
var db = mongoUtil.getDb();
var collection = db.collection( 'points' );
var users = [
{firstName:"John", lastName:"Doe", points:300},
{firstName:"Mickey", lastName:"Mouse", points:200}
];
let userUpdate = users.map(user => ({
updateOne: {
filter: {firstName: user.firstName, lastName: user.lastName},
update: {$set: user},
upsert: true
}
}));
collection.bulkWrite(userUpdate).catch(e => {
console.log(e);
});
I'm getting an array of ObjectId from a query, then I need to use them inside another query.
Here is the function:
exports.getAllByShop = wrap(async(req, res, next) => {
const products = await Product.find({ _shop: mongoose.Types.ObjectId(req.params.shopId) }).select('_id');
// how to make this query?
const sales = await Sale.find({ _product: { $in: [ products ]}});
res.status(200).json(sales);
});
The result of the first query products looks like this:
[
{ _id: 5be3601f90e40b35547ae6b4 },
{ _id: 5be4b5a3443abf196cb4cc9a }
]
What I'm trying to achieve:
const sales = await Sale.find({ _product: {
$in:
[
mongoose.Types.ObjectId(prod1),
mongoose.Types.ObjectId(prod2),
...
]
}});
I need to find all the sales that have their _product equals to the values of the above array. Right now it doesn't work and show this message: "Cast to ObjectId failed for value "[ { _id: 5be3601f90e40b35547ae6b4 },↵ { _id: 5be4b5a3443abf196cb4cc9a } ]" at path "_product" for model "sale"". How to make this last query work?
youre inserting an array, into an array, try this instead
const productIds = products.map((v) => v)
const sales = await Sale.find({ _product: { $in: productIds }});
I have 2 collections(data, metaData)
data schema is
{
_id: ......,
name: ......, //not unique
mobile: ......, // unique or null
email: ......, // unique or null
uniqueId: ......, // unique or null
}
at least one of unique data is required for insert
metaData schema is
{
_id: ......,
dataId: ......,//refrence from _id of data collection
key: ......,
value: ......
}
JSON array is getting from client
[{
name: "abc",
mobile: 9999999999,
mData: {
c1: 123,
c2: "xyz"
}
},
{
name: "qwerty",
email: 'qwerty#mail.com',
mData: {
c1: 123,
c2: "zxc"
}
}
......
]
I am iterating through the array and inserting each of them in both collections into MongoDB.
let Bulk = Data.collection.initializeUnorderedBulkOp();
dataArr.forEach(function(item) {
let data = service.generateData(item);
// data.query: {mobile: ..., email: ..., uniqueId: ...}
// if value exists then keys is also exists for mobile, email, uniqueId in query
Bulk.find(data.query).upsert().updateOne(data.doc);
});
Bulk.execute((e, d) => {
let metaBulk = MetaData.collection.initializeOrderedBulkOp();
let length = dataArr.length;
dataArr.forEach(function(data) {
Data.findOne(data.query).exec(function(err, data) {
length--;
for(let key in data["mData"]) {
let value = data["mData"][key] || "";
let mData = service.generateMdata(key, value, data._id);
metaBulk.find(mData.query).upsert().updateOne(mData.doc);
}
if(length == 0) {
metaBulk.execute();
}
});
});
});
my solution is working fine right now but it's taking so much time to iterating data collection for finding ids for metaData collection.
I need a way of inserting the data in bulk into MongoDB without find query for data id. Is there any option to perform bulk upserts with mongoose for multiple collections in a single query.
No multiple collection update in a single command for your scenario. In your case if you can include metadata array inside parent collection it can insert data with single command with updateMany(). MongoDB also supports bulk insert through the db.collection.insertMany().
db.data.insertMany( [{ name: "abc",mobile: 9999999999, mData: { c1: 123, c2: "xyz"} },
{name: "qwerty",email: 'qwerty#mail.com',mData: { c1: 123, c2: "zxc" }}]);
Also you can use db.collection.bulkWrite() as well.
I think what you can do is:
async.each(jsonArray, function(jsonData,callback){
//first insert data in data schema
var data = new data(jsonData);
data.save(function(err){
if err throw err;
//then you save the data in metaData collection
async.each(jsonData.mData, function(metadata, callback2){
var metaDataObj = new metaData(metadata);
metaDataObj.dataId = data._id;
metaDataObj.save(function(err){
callback2();
});
}, function(err, results1){
callback();
});
});
}, function(err, results){
console.log('Data is saved');
});
I have a collection of documents with many different characteristics. I want to get from Mongoose an array where each item in the array is a unique value for an attribute, in this case color. In other words, I do not want an array of each item, just of the color values. So in this case, if I have 100 different products, but only 8 colors between all 100, then I would like the to have an array of the 8 colors. Here is the way I have been using but I am wondering if there is a more concise way of doing it?
var allproducts = Products.find(function (err, products) {
// Get all products
var collection = [];
// Make an array to contain a product with each of the 8 colors
var uniqueArray = [];
// Make an array to check the current color against (if already used)
for (var i = products.length - 1; i >= 0; i--) {
if (uniqueArray.indexOf(products[i].color) === -1) {
// Check to see if the color is in the unique color array
uniqueArray.push(products[i].color);
// Add it if not
collection.push(products[i]);
// Add the product to the collection array if its color is not in the unique array
}
}
});
Attempting to use the Mongoose aggregate method:
router.get('/', function (req, res) {
Products.aggregate([{
$group: {
_id: '$color'
}
}], {}, function (err, collection) {
console.log(err, collection);
if (err) {
throw err
}
res.end();
// res.json(collection);
});
});
This can be easily solved using lodash
npm install lodash --save
var _ = require('lodash');
var allproducts = Products.find(function (err, products) {
...
var uniqueArray = _.pluck(products, 'color');
uniqueArray = _.uniq(uniqueArray);
....
});
I. Directly in mongodb, you have to this request :
db.colors.aggregate( { $group: {_id:"$color"} } )
Result :
{ "_id" : "yellow" }
{ "_id" : "blue" }
{ "_id" : "red" }
MongoDB Tutorial for aggregation
II. In mongoose you can do :
Products.aggregate(
{$group: {
_id: '$color' // grouping key - group by field district
}
}).exec( function( err, products ) {
if ( err ) {
return res.status( 400 ).send({
message: errorHandler.getErrorMessage( err )
});
} else {
res.json( products );
}
});
Mongoose : Model aggregate