Atomic deep merge for document MongoDB - node.js

I want to atomically deep merge a document in MongoDB. (I am using Node.js)
For example,
I have this document in my DB -
{
id: 123,
field1: {
field2: "abc"
}
}
And I am given this object:
{
field1: {
field3: "def"
}
}
So I'm looking for my document in the DB to change to this:
{
id: 123,
field1: {
field2: "abc",
field3: "def"
}
}
Of course I can do this in a non-atomic way by:
First fetching the document from the DB.
Then in JS do the merge and save it to a new object.
And finally overwrite the new item to the DB.
But I want this to happen in an atomic matter (so the read and the write stages will not happen separately). Is there any way of doing this?
I have heard the aggregation methods $merge and $mergeObjects might help me with that. Question is - are they atomic?
Thanks all :)

You can use the $function operator and go recursive.
Consider this doc in the DB:
{
field1: {
field6: "pre",
field2: "abc",
fieldX: {
aa: [1,2,3],
bb: "pre",
cc: "pre"
}
}
};
This is our candidate merge/overwrite doc:
var my_object = {
field1: {
field3: "def",
field6: "post",
fieldX: {
bb: "post",
zz: "post"
}
},
field4: 77,
field22: "qqq"
};
field4 and field22 are "easy"; they just get added to the doc. Where fields exist like field1, we have to test for objects vs. non-object. For objects, we will recursively descend into the db object and the candidate. For non-objects, we will let the candidate "win" and its value simply overwrites that in the db. This pipeline:
c = db.foo.aggregate([
{$replaceRoot: {newRoot: {$function: {
body: function(db_object, my_object) {
var process = function(myo, key, dbo) {
db_value = dbo[key];
if(undefined == db_value) {
// Easy; db does not have entry at all; just set it:
dbo[key] = myo[key];
} else {
// my_value has to be non-null because this function
// is driven from the keyset in myo.
my_value = myo[key];
if(db_value instanceof Object && my_value instanceof Object) {
// Both objects with same name; descend!
walkObj(db_value, my_value)
} else {
// my_object "wins" the slot and just overwrites dbo:
dbo[key] = myo[key];
}
}
};
var walkObj = function(dbo,myo) {
// Drive the walk from my_object; it overwrites/appends the db:
Object.keys(myo).forEach(function(k) {
process(myo, k, dbo);
});
}
walkObj(db_object, my_object); // entry point!
return db_object;
},
args: [ "$$CURRENT", my_object ],
lang: "js"
}}
}}
// Take each doc and put it back using _id as the key:
,{$merge: {
into: "foo",
on: [ "_id" ],
whenMatched: "merge",
whenNotMatched: "fail"
}}
]);
will produce this result in the DB:
{
_id: ObjectId("63f0f5e45d0b16d1033771e7"),
field1: {
field6: 'post',
field2: 'abc',
fieldX: { aa: [ 1, 2, 3 ], bb: 'post', cc: 'pre', zz: 'post' },
field3: 'def'
},
field22: 'qqq',
field4: 77
}

Related

Compare two nested objects and return an object with the keys which are into both of object

Let say that I have these two nested objects:
const sourceKeys = {
school: {
name: "Elisa Lemonnier",
students: 250
},
house: {
room : 2,
kids: true,
assets: "elevator",
}
}
const targetKeys = {
school: {
name: "Lucie Faure",
students: 150,
address: "123 Main St.",
phone: "555-555-5555"
},
house: {
room : 4,
kids: false,
assets: "Garden",
shop: "Auchan"
}
}
And I want the targetKeys keep ONLY the keys that are in sourceKeys. So I will get that :
const targetKeysMatchingSourceKeys = {
school: {
name: "Lucie Faure",
students: 150,
},
house: {
room : 4,
kids: false,
assets: "Garden",
}
}
I don't know how to proceed given that is a nested object. So, I will appreciate any help.
thanks you
I have find the solution, here is
const filteredJSON = Object.assign({}, TargetJsonToObject)
// Recursive function to filter the properties of the object
function filterObject(SourceJsonToObject, filteredObj) {
for (const key of Object.keys(filteredObj)) {
// If the key is not present in the source JSON, delete it from filtered JSON
if (!SourceJsonToObject.hasOwnProperty(key)) {
delete filteredObj[key]
} else if (typeof filteredObj[key] === "object") {
// If the key is present in the source JSON and the value is an object, recursively call the function on the nested object
filterObject(SourceJsonToObject[key], filteredObj[key])
}
}
}
filterObject(SourceJsonToObject, TargetJsonToObject)

How to pass an optional argument in Mongoose/MongoDb

I have the following query:
Documents.find({
$and: [
{
user_id: {$nin:
myUserId
}
},
{ date: { $gte: dateMax, $lt: dateMin } },
{documentTags: {$all: tags}}
],
})
What I'm trying to do is make the documentTags portion of the query optional. I have tried building the query as follows:
let tags = " ";
if (req.body.tags) {
tags = {videoTags: {$all: req.body.tags}};
}
let query = {
$and: [
{
user_id: {$nin:
myUserId
}
},
{ date: { $gte: dateMax, $lt: dateMin } },
tags
],
}
and then Document.find(query). The problem is no matter how I modify tags (whether undefined, as whitespace, or otherwise) I get various errors like $or/$and/$nor entries need to be full objects and TypeError: Cannot read property 'hasOwnProperty' of undefined.
Is there a way to build an optional requirement into the query?
I tried the option below and the query is just returning everything that matches the other fields. For some reason it isn't filtering by tags. I did a console.log(queryArr) and console.log(query) get the following respectively:
[
{ user_id: { '$nin': [Array] } },
{
date: {
'$gte': 1985-01-01T00:00:00.000Z,
'$lt': 2020-01-01T00:00:00.000Z
}
},
push: { documentTags: { '$all': [Array] } }
]
console.log(query)
{
'$and': [
{ user_id: [Object] },
{ date: [Object] },
push: { documentTags: [Object] }
]
}
You are almost there. Instead you could construct the object outside the query and just put the constructed query in $and when done..
let queryArr = [
{
user_id: {$nin: myUserId}
},
{ date: { $gte: dateMax, $lt: dateMin } }
];
if (req.body.tags) {
queryArr.push({videoTags: {$all: req.body.tags}});
}
let query = {
$and: queryArr
}
Now you can control the query by just pushing object into the query Array itself.
I figured out why it wasn't working. Basically, when you do myVar.push it creates a key-value pair such as [1,2,3,push:value]. This would work if you needed to append a k-v pair in that format, but you'll have difficulty using it in a query like mine. The right way for me turned out to be to use concact which appends the array with just the value that you set, rather than a k-v pair.
if (req.body.tags){
queryArgs = queryArgs.concat({documentTags: {$all: tags}});
}
let query = {
$and: queryArgs
}

$addToSet Based on Object key exists

I have array 'pets': [{'fido': ['abc']} that is a embeded document. When I add a pet to the array, how can I check to see if that pet already exists? For instance, if I added fido again... how can I check if only fido exists and not add it? I was hoping I could use $addToSet but I only want to check part of the set(the pets name).
User.prototype.updatePetArray = function(userId, petName) {
userId = { _id: ObjectId(userId) };
return this.collection.findOneAndUpdate(userId,
{ $addToSet: { pets: { [petName]: [] } } },
{ returnOriginal: false,
maxTimeMS: QUERY_TIME });
Result of adding fido twice:
{u'lastErrorObject': {u'updatedExisting': True, u'n': 1}, u'ok': 1, u'value': {u'username': u'bob123', u'_id': u'56d5fc8381c9c28b3056f794', u'location': u'AT', u'pets': [{u'fido': []}]}}
{u'lastErrorObject': {u'updatedExisting': True, u'n': 1}, u'ok': 1, u'value': {u'username': u'bob123', u'_id': u'56d5fc8381c9c28b3056f794', u'location': u'AT', u'pets': [{u'fido': [u'abc']}, {u'fido': []}]}}
If there is always going to be "variable" content within each member of the "pets" array ( i.e petName as the key ) then $addToSet is not for you. At least not not at the array level where you are looking to apply it.
Instead you basically need an $exists test on the "key" of the document being contained in the array, then either $addToSet to the "contained" array of that matched key with the positional $ operator, or where the "key" was not matched then $push directly to the "pets" array, with the new inner content directly as the sole array member.
So if you can live with not returning the modified document, then "Bulk" operations are for you. In modern drivers with bulkWrite():
User.prototype.updatePetArray = function(userId, petName, content) {
var filter1 = { "_id": ObjectId(userId) },
filter2 = { "_id": ObjectId(userId) },
update1 = { "$addToSet": {} },
update2 = { "$push": { "pets": {} } };
filter1["pets." + petName] = { "$exists": true };
filter2["pets." + petName] = { "$exists": false };
var setter1 = {};
setter1["pets.$." + petName] = content;
update1["$addToSet"] = setter1;
var setter2 = {};
setter2[petName] = [content];
update2["$push"]["pets"] = setter2;
// Return the promise that yields the BulkWriteResult of both calls
return this.collection.bulkWrite([
{ "updateOne": {
"filter": filter1,
"update": update1
}},
{ "updateOne": {
"filter": filter2,
"update": update2
}}
]);
};
If you must return the modified document, then you are going to need to resolve each call and return the one that actually matched something:
User.prototype.updatePetArray = function(userId, petName, content) {
var filter1 = { "_id": ObjectId(userId) },
filter2 = { "_id": ObjectId(userId) },
update1 = { "$addToSet": {} },
update2 = { "$push": { "pets": {} } };
filter1["pets." + petName] = { "$exists": true };
filter2["pets." + petName] = { "$exists": false };
var setter1 = {};
setter1["pets.$." + petName] = content;
update1["$addToSet"] = setter1;
var setter2 = {};
setter2[petName] = [content];
update2["$push"]["pets"] = setter2;
// Return the promise that returns the result that matched and modified
return new Promise(function(resolve,reject) {
var operations = [
this.collection.findOneAndUpdate(filter1,update1,{ "returnOriginal": false}),
this.collection.findOneAndUpdate(filter2,update2,{ "returnOriginal": false})
];
// Promise.all runs both, and discard the null document
Promise.all(operations).then(function(result) {
resolve(result.filter(function(el) { return el.value != null } )[0].value);
},reject);
});
};
In either case this requires "two" update attempts where only "one" will actually succeed and modify the document, since only one of the $exists tests is going to be true.
So as an example of that first case, the "query" and "update" are resolving after interpolation as:
{
"_id": ObjectId("56d7b759e955e2812c6c8c1b"),
"pets.fido": { "$exists": true }
},
{ "$addToSet": { "pets.$.fido": "ccc" } }
And the second update as:
{
"_id": ObjectId("56d7b759e955e2812c6c8c1b"),
"pets.fido": { "$exists": false }
},
{ "$push": { "pets": { "fido": ["ccc"] } } }
Given varibles of:
userId = "56d7b759e955e2812c6c8c1b",
petName = "fido",
content = "ccc";
Personally I would not be naming keys like this, but rather change the structure to:
{
"_id": ObjectId("56d7b759e955e2812c6c8c1b"),
"pets": [{ "name": "fido", "data": ["abc"] }]
}
That makes the update statements easier, and without the need for variable interpolation into the key names. For example:
{
"_id": ObjectId(userId),
"pets.name": petName
},
{ "$addToSet": { "pets.$.data": content } }
and:
{
"_id": ObjectId(userId),
"pets.name": { "$ne": petName }
},
{ "$push": { "pets": { "name": petName, "data": [content] } } }
Which feels a whole lot cleaner and can actually use an "index" for matching, which of course $exists simply cannot.
There is of course more overhead if using .findOneAndUpdate(), since this is afterall "two" actual calls to the server for which you need to await a response as opposed to the Bulk method which is just "one".
But if you need the returned document ( option is the default in the driver anyway ) then either do that or similarly await the Promise resolve from the .bulkWrite() and then fetch the document via .findOne() after completion. Albeit that doing it via .findOne() after the modification would not truly be "atomic" and could possibly return the document "after" another similar modification was made, and not only in the state of that particular change.
N.B Also assuming that apart from the keys of the subdocuments in "pets" as a "set" that your other intention for the array contained was adding to that "set" as well via the additional content supplied to the function. If you just wanted to overwrite a value, then just apply $set instead of $addToSet and similarly wrap as an array.
But it sounds reasonable that the former was what you were asking.
BTW. Please clean up by horrible setup code in this example for the query and update objects in your actual code :)
As a self contained listing to demonstrate:
var async = require('async'),
mongodb = require('mongodb'),
MongoClient = mongodb.MongoClient;
MongoClient.connect('mongodb://localhost/test',function(err,db) {
var coll = db.collection('pettest');
var petName = "fido",
content = "bbb";
var filter1 = { "_id": 1 },
filter2 = { "_id": 1 },
update1 = { "$addToSet": {} },
update2 = { "$push": { "pets": {} } };
filter1["pets." + petName] = { "$exists": true };
filter2["pets." + petName] = { "$exists": false };
var setter1 = {};
setter1["pets.$." + petName] = content;
update1["$addToSet"] = setter1;
var setter2 = {};
setter2[petName] = [content];
update2["$push"]["pets"] = setter2;
console.log(JSON.stringify(update1,undefined,2));
console.log(JSON.stringify(update2,undefined,2));
function CleanInsert(callback) {
async.series(
[
// Clean data
function(callback) {
coll.deleteMany({},callback);
},
// Insert sample
function(callback) {
coll.insert({ "_id": 1, "pets": [{ "fido": ["abc"] }] },callback);
}
],
callback
);
}
async.series(
[
CleanInsert,
// Modify Bulk
function(callback) {
coll.bulkWrite([
{ "updateOne": {
"filter": filter1,
"update": update1
}},
{ "updateOne": {
"filter": filter2,
"update": update2
}}
]).then(function(res) {
console.log(JSON.stringify(res,undefined,2));
coll.findOne({ "_id": 1 }).then(function(res) {
console.log(JSON.stringify(res,undefined,2));
callback();
});
},callback);
},
CleanInsert,
// Modify Promise all
function(callback) {
var operations = [
coll.findOneAndUpdate(filter1,update1,{ "returnOriginal": false }),
coll.findOneAndUpdate(filter2,update2,{ "returnOriginal": false })
];
Promise.all(operations).then(function(res) {
//console.log(JSON.stringify(res,undefined,2));
console.log(
JSON.stringify(
res.filter(function(el) { return el.value != null })[0].value
)
);
callback();
},callback);
}
],
function(err) {
if (err) throw err;
db.close();
}
);
});
And the output:
{
"$addToSet": {
"pets.$.fido": "bbb"
}
}
{
"$push": {
"pets": {
"fido": [
"bbb"
]
}
}
}
{
"ok": 1,
"writeErrors": [],
"writeConcernErrors": [],
"insertedIds": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 1,
"nModified": 1,
"nRemoved": 0,
"upserted": []
}
{
"_id": 1,
"pets": [
{
"fido": [
"abc",
"bbb"
]
}
]
}
{"_id":1,"pets":[{"fido":["abc","bbb"]}]}
Feel free to change to different values to see how different "sets" are applied.
Please try this one with string template, here is one example running under mongo shell
> var name = 'fido';
> var t = `pets.${name}`; \\ string temple, could parse name variable
> db.pets.find()
{ "_id" : ObjectId("56d7b5019ed174b9eae2b9c5"), "pets" : [ { "fido" : [ "abc" ]} ] }
With the following update command, it will not update it if the same pet name exists.
> db.pets.update({[t]: {$exists: false}}, {$addToSet: {pets: {[name]: []}}})
WriteResult({ "nMatched" : 0, "nUpserted" : 0, "nModified" : 0 })
If the pets document is
> db.pets.find()
{ "_id" : ObjectId("56d7b7149ed174b9eae2b9c6"), "pets" : [ { "fi" : [ "abc" ] } ] }
After update with
> db.pets.update({[t]: {$exists: false}}, {$addToSet: {pets: {[name]: []}}})
WriteResult({ "nMatched" : 1, "nUpserted" : 0, "nModified" : 1 })
The result shows add the pet name if it does Not exist.
> db.pets.find()
{ "_id" : ObjectId("56d7b7149ed174b9eae2b9c6"), "pets" : [ { "fi" : [ "abc" ] }, { "fido" : [ ] } ] }

MongoError: The dollar ($) prefixed field '$push' in '$push' is not valid for storage

I am trying to upsert a dataset to a Mongo collection.
The intended document may or may not exist.
If it does exist, it will have at least one item in an embedded document (zips), and should append to that document rather than overwrite it.
If it does not exist, it should insert the new document to the collection.
When I run the below code, I am getting an error: MongoError: The dollar ($) prefixed field '$push' in '$push' is not valid for storage.
I put this together based on the docs: https://docs.mongodb.org/getting-started/node/update/#update-multiple-documents
Versions:
MongoDB (windows) = 3.2.0;
mongodb (npm package) = 2.1.4
var query = {
county: aCountyName,
state: aStateName
}
var params = {
'$set': {
county: 'Boone',
state: 'MO',
'$push': {
zips: {
'$each': [ '65203' ]
}
}
}
}
(could also be)
var params = {
'$set': {
county: 'Pierce',
state: 'WA',
'$push': {
zips: {
'$each': [ '98499', '98499' ]
}
}
}
}
db.collection(collectionName).updateMany(query, params, {'upsert': true},
function(err, results) {
callback();
}
);
I don't think $push is valid within a $set. Instead try adding it as another parameter, e.g.:
var params = {
'$set': {
county: 'Pierce',
state: 'WA'
},
'$push': {
zips: {
'$each': ['98499',
'98499']
}
}
}
The reason is because you didn't close the } so MongoDB think $push is a field's name and as mentioned in the documentation:
Field names cannot contain dots (i.e. .) or null characters, and they must not start with a dollar sign (i.e. $).
var query = {
county: aCountyName,
state: aStateName
};
var params = {};
params['$set'] = { county: 'Boone', state: 'MO' };
params['$push'] = { zips: { '$each': [ '65203' ] } };
Then:
db.collection(collectionName).updateMany(query, params, {'upsert': true},
function(err, results) {
callback();
}
);

using db.mycollection.distinct('field1', 'field2') with mongodb / mongoskin

I have the following line in an exports declaration in a user.js file:
db.collection('myList').distinct('field1', 'field2'), function(err, items) {
res.json(items);
}
The db lookup works fine from the mongo command line. However, when I run the webpage, I get:
TypeError: string is not a function
I'm not certain how to proceed at this point. Should I be casting the return to a string somehow before returning it?
Thanks.
Your question does not seem to be well defined, consider the logic:
> db.info.insert({ field1: "this", field2: "that" })
> db.info.insert({ field1: "this", field2: "another" })
> db.info.insert({ field1: "this", field2: "that" })
> db.info.insert({ field1: "this", field2: "another" })
> db.info.distinct('field1', 'field2')
[ "this" ]
But what you actually want is:
db.info.aggregate([
{$group: {_id: { field1: "$field1", field2: "$field2" } } }
])
Which gives:
{
"result" : [
{
"_id" : {
"field1" : "this",
"field2" : "another"
}
},
{
"_id" : {
"field1" : "this",
"field2" : "that"
}
}
],
"ok" : 1
}
The distinct results of the combined fields.
I hope we are learning now.
For some odd reason MongoSkin abstraction doesn't provide access to the MongoDB 'distinct' method. To get distinct to work, I had to use the native MongoDB collection and not the abstracted MongoSkin collection. Here is how I did it
var store = mongo.db(connectionString, ackOptions);
store.open(function(err,resp){
resp.collection(this.options.collection, {strict: true}, function(err, myCollection) {
myCollection.distinct('tag',function(err,res){
console.log(err)
console.log(res)
})
});
})

Resources