I have 2 collections(data, metaData)
data schema is
{
_id: ......,
name: ......, //not unique
mobile: ......, // unique or null
email: ......, // unique or null
uniqueId: ......, // unique or null
}
at least one of unique data is required for insert
metaData schema is
{
_id: ......,
dataId: ......,//refrence from _id of data collection
key: ......,
value: ......
}
JSON array is getting from client
[{
name: "abc",
mobile: 9999999999,
mData: {
c1: 123,
c2: "xyz"
}
},
{
name: "qwerty",
email: 'qwerty#mail.com',
mData: {
c1: 123,
c2: "zxc"
}
}
......
]
I am iterating through the array and inserting each of them in both collections into MongoDB.
let Bulk = Data.collection.initializeUnorderedBulkOp();
dataArr.forEach(function(item) {
let data = service.generateData(item);
// data.query: {mobile: ..., email: ..., uniqueId: ...}
// if value exists then keys is also exists for mobile, email, uniqueId in query
Bulk.find(data.query).upsert().updateOne(data.doc);
});
Bulk.execute((e, d) => {
let metaBulk = MetaData.collection.initializeOrderedBulkOp();
let length = dataArr.length;
dataArr.forEach(function(data) {
Data.findOne(data.query).exec(function(err, data) {
length--;
for(let key in data["mData"]) {
let value = data["mData"][key] || "";
let mData = service.generateMdata(key, value, data._id);
metaBulk.find(mData.query).upsert().updateOne(mData.doc);
}
if(length == 0) {
metaBulk.execute();
}
});
});
});
my solution is working fine right now but it's taking so much time to iterating data collection for finding ids for metaData collection.
I need a way of inserting the data in bulk into MongoDB without find query for data id. Is there any option to perform bulk upserts with mongoose for multiple collections in a single query.
No multiple collection update in a single command for your scenario. In your case if you can include metadata array inside parent collection it can insert data with single command with updateMany(). MongoDB also supports bulk insert through the db.collection.insertMany().
db.data.insertMany( [{ name: "abc",mobile: 9999999999, mData: { c1: 123, c2: "xyz"} },
{name: "qwerty",email: 'qwerty#mail.com',mData: { c1: 123, c2: "zxc" }}]);
Also you can use db.collection.bulkWrite() as well.
I think what you can do is:
async.each(jsonArray, function(jsonData,callback){
//first insert data in data schema
var data = new data(jsonData);
data.save(function(err){
if err throw err;
//then you save the data in metaData collection
async.each(jsonData.mData, function(metadata, callback2){
var metaDataObj = new metaData(metadata);
metaDataObj.dataId = data._id;
metaDataObj.save(function(err){
callback2();
});
}, function(err, results1){
callback();
});
});
}, function(err, results){
console.log('Data is saved');
});
Related
Script logic - script receives data from Binance API > Then I have aggregation $avg to calculate the average of one asset. I will have more than one collection so I need to calculate average of every asset.
I have an array where I store collection names for MongoDB.
const symbols = ["ADABTC", "AEBTC", "AIONBTC"]
And I want to calculate average from MongoDB collection.
const collection = db.collection(symbols);
Here - symbols doesn't work for me, but if I simply add "ADABTC" then it works, but it doesn't fixes my problem since I want to use different collection names one after another one.
How I can pass an array if it's required to be a string? I need to use more than 1 collection names.
FULL CODE
const MongoClient = require('mongodb').MongoClient;
const assert = require('assert');
// Connection URL
const url = 'mongodb://username:password#serveripadress:port/dbname?retryWrites=true&w=majority';
const symbols = ["ADABTC", "AEBTC", "AIONBTC"]
// Database Name
const dbName = 'Crypto';
// Create a new MongoClient
const client = new MongoClient(url, { useUnifiedTopology: true });
// Use connect method to connect to the Server
client.connect(function(err, client) {
assert.equal(null, err);
console.log("Connected correctly to server");
const db = client.db(dbName);
simplePipeline(db, function() {
client.close();
});
});
function simplePipeline(db, callback) {
const collection = db.collection(symbols);
collection.aggregate(
[{
'$group': {
_id: null,
'Volume': {
'$avg': '$Volume'
}
}
}],
function(err, cursor) {
assert.equal(err, null);
cursor.toArray(function(err, documents) {
console.log(documents)
callback(documents);
});
}
);
}
It is not possible to pass an array into a function that is asking for a string. In your case what you need to do is join three collections. If you need to aggregate across multiple collections you can use the $lookup aggregation pipeline operator. You can connect using the first collection:
db.collection(symbols) => db.collection(symbols[0])
Then modify your query to join the three collections:
// Join with AEBTC collection
{
$lookup:{
from: symbols[1],
localField: //name of a value in first collection
foreignField: //name of same value in second collection
as: //what you want to call it in the second table
}
},
{ $unwind: //what you called it },
// Join with AIONBTC collection
{
$lookup:{
from: symbols[2],
localField: //name of value in joined collections 1 and 2
foreignField: //name of that value in collection 3,
as: //whatever you want to call it in the joined collection
}
},
{ $unwind: //what you called it },
// define some conditions here
{
$match {}
},
// define which fields are you want to fetch
{
$group: {
_id: null,
'Volume': {
'$avg': '$Volume'
}
}
I want to achieve the result as obtained by
SELECT AGE FROM COLL WHERE NAME="AYUSH";
I took the following approach
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/";
MongoClient.connect(url, function(err, db) {
if (err) throw err;
var dbo = db.db("new");
//var query = { name:'ayush' };
//var age = {age : 1, _id:0};
dbo.collection("coll").find(
{ name:'ayush' },
{ age : 1, _id:0}).toArray(function(err, result) {
if (err) throw err;
console.log(result);
db.close();
});
});
the result that i am getting is
[ { _id: 5a818b71d2029813505d736a,
name: 'ayush',
age: '22',
sex: 'm' } ]
From MongoDB documentation : Project Fields to Return from Query
Return the Specified Fields and the _id Field Only
A projection can explicitly include several fields by setting the
<field> to 1 in the projection document. The following operation
returns all documents that match the query. In the result set, only
the item, status and, by default, the _id fields return in the
matching documents.
db.inventory.find( { status: "A" }, { item: 1, status: 1 } )
The operation corresponds to the following SQL statement:
SELECT _id, item, status from inventory WHERE status = "A"
In your case, if you only want the field age, you have to suppress the other fields _id, name and sex as following :
dbo.collection("coll").find({ name:'ayush' },{age:1, _id:0, name:0, sex:0})...
The easiest way to do this would be to convert the result in to an array and then send that array as the response.
The code would look like:
dbo.collection("coll").find(
{ name:'ayush' },
{ age : 1, _id:0}).toArray(function(err, result) {
if (err) throw err;
var array = [];
array.push(result[0].age);
res.send(array);});
Moreover don't use mongoClient,use mongoose instead.its easier and better
If I were to perform this query with mongoose;
Schema.find({
_id: {
$in: ['abcd1234', 'abcd1234', 'abcd1234']
}
});
The query will only return something like:
[{
'property1': 'key1',
'property2': 'key2'
}]
With the array only having one object, obviously because I passed in all the same id's. However, I actually want duplicate objects returned. How can I do this?
Mongo itself will only return objects with no duplicates. But you can then build an array of objects with duplicates from that.
For example, if array is the array of objects returned my Mongo - in this case:
var array = [{
_id: 'abcd1234',
property1: 'key1',
property2: 'key2'
}];
and ids is your list of IDs that you want with duplicates - in your case:
var ids = ['abcd1234', 'abcd1234', 'abcd1234'];
then you can do:
var objects = {};
array.forEach(o => objects[o._id] = o);
var dupArray = ids.map(id => objects[id]);
Now dupArray should contain the objects with duplicates.
Full example:
var ids = ['abcd1234', 'abcd1234', 'abcd1234'];
Schema.find({_id: {$in: ids}}, function (err, array) {
if (err) {
// handle error
} else {
var objects = {};
array.forEach(o => objects[o._id] = o);
var dupArray = ids.map(id => objects[id]);
// here you have objects with duplicates in dupArray:
console.log(dupArray);
}
});
I am using sub-documents in my MEAN project, to handle orders and items per order.
These are my (simplified) schemas:
var itemPerOrderSchema = new mongoose.Schema({
itemId: String,
count: Number
});
var OrderSchema = new mongoose.Schema({
customerId: String,
date: String,
items: [ itemPerOrderSchema ]
});
To insert items in itemPerOrderSchema array I currently do:
var orderId = '123';
var item = { itemId: 'xyz', itemsCount: 7 };
Order.findOne({ id: orderId }, function(err, order) {
order.items.push(item);
order.save();
});
The problem is that I obviously want one item per itemId, and this way I obtain many sub-documents per item...
One solution could be to loop through all order.items, but this is not optimal, of course (order.items could me many...).
The same problem could arise when querying order.items...
The question is: how do I insert items in itemPerOrderSchema array without having to loop through all items already inserted on the order?
If you can use an object instead of array for items, maybe you can change your schema a bit for a single-query update.
Something like this:
{
customerId: 123,
items: {
xyz: 14,
ds2: 7
}
}
So, each itemId is a key in an object, not an element of the array.
let OrderSchema = new mongoose.Schema({
customerId: String,
date: String,
items: mongoose.Schema.Types.Mixed
});
Then updating your order is super simple. Let's say you want to add 3 of items number 'xyz' to customer 123.
db.orders.update({
customerId: 123
},
{
$inc: {
'items.xyz': 3
}
},
{
upsert: true
});
Passing upsert here to create the order even if the customer doesn't have an entry.
The downsides of this:
it is that if you use aggregation framework, it is either impossible to iterate over your items, or if you have a limited, known set of itemIds, then very verbose. You could solve that one with mapReduce, which can be a little slower, depending on how many of them you have there, so YMMB.
you do not have a clean items array on the client. You could fix that with either client extracting this info (a simple let items = Object.keys(order.items).map(key => ({ key: order.items[key] })); or with a mongoose virtual field or schema.path(), but this is probably another question, already answered.
First of all, you probably need to add orderId to your itemPerOrderSchema because the combination of orderId and itemId will make the record unique.
Assuming that orderId is added to the itemPerOrderSchema, I would suggest the following implementation:
function addItemToOrder(orderId, newItem, callback) {
Order.findOne({ id: orderId }, function(err, order) {
if (err) {
return callback(err);
}
ItemPerOrder.findOne({ orderId: orderId, itemId: newItem.itemId }, function(err, existingItem) {
if (err) {
return callback(err);
}
if (!existingItem) {
// there is no such item for this order yet, adding a new one
order.items.push(newItem);
order.save(function(err) {
return callback(err);
});
}
// there is already item with itemId for this order, updating itemsCount
itemPerOrder.update(
{ id: existingItem.id },
{ $inc: { itemsCount: newItem.itemsCount }}, function(err) {
return callback(err);
}
);
});
});
}
addItemToOrder('123', { itemId: ‘1’, itemsCount: 7 }, function(err) {
if (err) {
console.log("Error", err);
}
console.log("Item successfully added to order");
});
Hope this may help.
I would like to take the last n documents from my Cloudant database using a Node query. So far I have narrowed it down to the find() function, but the documentation only really explains how to retrieve all documents containing an absolute value, for example:
db.find({selector:{name:'Alice'}}, function(er, result) {
...
});
(taken from https://www.npmjs.com/package/cloudant#cloudant-query)
What I'm looking for is the equivalent of this SQL:
SELECT * FROM db WHERE name = "Alice" LIMIT 10
The code I have so far is this:
var cloudant = require('cloudant');
cloudant({account: username, password: password}, function (err, conn) {
if (err) {
callback("Could not initialize connection to Cloudant: " + err);
} else {
var db = conn.db.use('mydb');
db.find(???, function(err, data) {
if (err) {
callback("No data found: " + err);
} else {
...
}
});
}
});
If I need to make design documents, I'd do so in the Cloudant online interface, so don't worry too much about making an executable answer for that if it's necessary.
It's important to note that Cloudant Query requires you to define your index before performing the query e.g. to index the 'name' field from your documents:
db.index( {name:'nameindex', type:'json', index:{fields:['name']}}
We can now query the data using the find function as you indicated:
var query = { selector: { name: 'Alice' }};
db.find(query, function(err, data) {
});
The interesting thing about your question is the phrase 'the last n documents'. We can retrieve 'n' documents by adding a 'limit' to our query.
var query = { selector: { name: 'Alice' }, limit: 10};
db.find(query, function(err, data) {
});
but this doesn't necessarily indicate the last ten documents; it just limits the result set to ten.
If you want your query results to appear in time order, then you'll need something in your documents that indicates the time e.g.
a time string : { "name": "Alice", "datetime": "2015-11-26 10:22:24 Z" }
a timestamp : { "name": "Alice", "ts": "123456678" }
When your document contains a field which represents time, then your index can be created incorporate this into the index e.g.
db.index( {name:'nameindex', type:'json', index:{fields:['name','time']}}
and documents can be queried to appear in *reverse order * (to get the latest first):
var query = { selector: { name: 'Alice' }, sort: [ { name: "desc"}, { ts: "desc"]};
db.find(query, function(err, data) {
});
You may also want to look at type:"text" indexes too. See https://docs.cloudant.com/cloudant_query.html
var query = { selector: { name: 'Alice' }, sort: [ { name: "desc"}, { ts: "desc"]};
should be
var query = { selector: { name: 'Alice' }, sort: [ { name: "desc"}, { ts: "desc"}];