How to save array data in nodejs using mongoose? - node.js

I am stuck to save data in mongoDb. Here data is in array and i need to insert data if mongodb does not have. Please look code:-
var contactPersonData = [{
Name: 'Mr. Bah',
Organization: 'Ashima Limited - Point 2'
}, {
Name: 'Mr. Sel',
Organization: 'Ashima Limited - Point 2'
}, {
Name: 'Mr.ATEL',
Organization: 'Ashima Limited - Point 2'
}, {
Name: 'ANISH',
Organization: 'Ashima Limited - Point 2'
}, {
Name: 'sunny ji',
Organization: 'Ashima Limited - Point 2'
}, {
Name: 'ashish',
Organization: 'Ashima Limited - Point 2'
}]
console.log('filedata', contactPersonData);
var escapeData = [];
var tempArr = [];
function saveContact(personObj, mainCallback) {
var tempC = personObj['Organization'].trim();
var insertData = {};
Contact.findOne({ companyName: tempC })
.exec(function(err, contact) {
if (err)
return mainCallback(err);
console.log('find com', contact)
if (contact) {
//document exists
mainCallback(null, insertData);
} else {
var newContact = new Contact({ companyName: tempC, createdBy: '58ae5d18ba71d4056f30f7b1' });
newContact.save(function(err, contact) {
if (err)
return mainCallback(err);
console.log('new contact', contact)
insertData.contactId = contact._id;
insertData.name = personObj['Name'];
insertData.email = personObj['Email'];
insertData.contactNumber = { number: personObj['Phone'] };
insertData.designation = personObj['Designation'];
tempArr.push(insertData);
mainCallback(null, insertData);
})
}
});
}
async.map(contactPersonData, saveContact, function(err, result) {
console.log(err)
console.log(result)
},
function(err) {
if (err)
return next(err);
res.status(200).json({ unsaved: escapeData })
})
As per above code it has to insert six document instead of one. I think that above iteration not wait to complete previous one. So, the if condition is always false and else is executed.

Your saveContact() function is fine. The reason you get 6 documents instead of 1 document is that async.map() runs you code in parallel. All the 6 requests are made in parallel, not one after the another.
From the async.map() function's documentation -
Note, that since this function applies the iteratee to each item in parallel, there is no guarantee that the iteratee functions will complete in order.
As a result before the document is created in your database all queries are already run and all the 6 queries are not able to find that document since its still in the process of creation. Therefore your saveContact() method creates all the 6 documents.
If you run your code again then no more documents will be formed because by that time your document would be formed.
You should try running your code using async.mapSeries() to process your request serially. Just replace map() with mapSeries() in your above code. This way it will wait for one request to complete and then execute another and as a result only one document will be created. More on async.mapSeries() here.

You seem to be using async.map() wrong.
First, async.map() only has 3 parameters (i.e. coll, iteratee, and callback) so why do you have 4? In your case, coll is your contactPersonData, iteratee is your saveContact function, and callback is an anonymous function.
Second, the whole point of using async.map() is to create a new array. You are not using it that way and, instead, are using it more like an async.each().
Third, you probably should loop through the elements sequentially and not in parallel. Therefore, you should use async.mapSeries() instead of async.map().
Here's how I would revise/shorten your code:
var contactPersonData = [{
Name: 'Mr. Bah',
Organization: 'Ashima Limited - Point 2'
}, {
Name: 'Mr. Sel',
Organization: 'Ashima Limited - Point 2'
}, {
Name: 'Mr.ATEL',
Organization: 'Ashima Limited - Point 2'
}, {
Name: 'ANISH',
Organization: 'Ashima Limited - Point 2'
}, {
Name: 'sunny ji',
Organization: 'Ashima Limited - Point 2'
}, {
Name: 'ashish',
Organization: 'Ashima Limited - Point 2'
}];
function saveContact(personObj, mainCallback) {
var tempC = personObj.Organization.trim();
Contact.findOne({ companyName: tempC }, function (err, contact) {
if (err)
return mainCallback(err);
console.log('found contact', contact);
// document exists, so mark it as complete and pass the old item
if (contact)
return mainCallback(null, contact);
// document does not exist, so add it
contact = new Contact({ companyName: tempC, createdBy: '58ae5d18ba71d4056f30f7b1' });
contact.save(function (err, contact) {
if (err)
return mainCallback(err);
console.log('created new contact', contact)
// mark it as complete and pass a new/transformed item
mainCallback(null, {
contactId: contact._id,
name: personObj.Name,
email: personObj.Email, // ??
contactNumber: { number: personObj.Phone }, // ??
designation: personObj.Designation // ??
});
});
});
};
async.mapSeries(contactPersonData, saveContact, function (err, contacts) {
if (err)
return next(err);
// at this point, contacts will have an array of your old and new/transformed items
console.log('transformed contacts', contacts);
res.json({ unsaved: contacts });
});
In terms of ?? comments, it means you don't have these properties in your contactPersonData and would therefore be undefined.

Related

Resolve Promise in Nodejs Nested Map Functions and Return Array of Objects From API Call [duplicate]

I'm building an API to add movies to wishlist. I have an endpoint to get all movies in wishlist. My approach was to get the movie ids (not from mongodb) and make an API request to another API to get the movie objects.
This has been successful so far but the problem now is I am getting two objects fused into one object like below:
{
id: 7,
url: 'https://www.tvmaze.com/shows/7/homeland',
name: 'Homeland',
language: 'English',
genres: [ 'Drama', 'Thriller', 'Espionage' ],
status: 'Ended',
runtime: 60,
averageRuntime: 60,
premiered: '2011-10-02',
officialSite: 'http://www.sho.com/sho/homeland/home',
schedule: { time: '21:00', days: [ 'Sunday' ] },
rating: { average: 8.2 },
image: {
medium: 'https://static.tvmaze.com/uploads/images/medium_portrait/230/575652.jpg',
original: 'https://static.tvmaze.com/uploads/images/original_untouched/230/575652.jpg'
},
summary: '<p>The winner of 6 Emmy Awards including Outstanding Drama Series, <b>Homeland</b> is an edge-of-your-seat sensation. Marine Sergeant Nicholas Brody is both a decorated hero and a serious threat. CIA officer Carrie Mathison is tops in her field despite being bipolar. The delicate dance these two complex characters perform, built on lies, suspicion, and desire, is at the heart of this gripping, emotional thriller in which nothing short of the fate of our nation is at stake.</p>',
}
This is the second object below. Notice how there's no comma separating both objects
{
id: 1,
url: 'https://www.tvmaze.com/shows/1/under-the-dome',
name: 'Under the Dome',
language: 'English',
genres: [ 'Drama', 'Science-Fiction', 'Thriller' ],
status: 'Ended',
runtime: 60,
averageRuntime: 60,
premiered: '2013-06-24',
schedule: { time: '22:00', days: [ 'Thursday' ] },
rating: { average: 6.6 },
image: {
medium: 'https://static.tvmaze.com/uploads/images/medium_portrait/81/202627.jpg',
original: 'https://static.tvmaze.com/uploads/images/original_untouched/81/202627.jpg'
},
summary: "<p><b>Under the Dome</b> is the story of a small town that is suddenly and inexplicably sealed off from the rest of the world by an enormous transparent dome. The town's inhabitants must deal with surviving the post-apocalyptic conditions while searching for answers about the dome, where it came from and if and when it will go away.</p>",
}
My question now is how do I convert both objects to an array and send as a response from my own API. API code is below:
module.exports = {
fetchAll: async (req, res, next) => {
var idsArr = [];
var showsArr;
var shows;
try {
let wishlist = await Wishlist.find({});
if (wishlist == null) {
res.status(404)
.json({
success: false,
msg: 'No Movies Found in Wishlist',
wishlist: []
})
}
// console.log(wishlist);
wishlist.map((item) => {
idsArr.push(item.id);
})
console.log(idsArr);
idsArr.map(async (id) => {
shows = await axios.get(`https://api.tvmaze.com/shows/${id}`);
console.log(shows.data);
// console.log(showsArr);
// showsArr = [shows.data];
})
console.log(showsArr);
return res.status(200)
.json({
success: true,
msg: 'All Movies in Wishlist Fetched',
wishlist: showsArr
})
} catch (err) {
console.log(err);
next(err);
}
},
... // other methods
}
I have tried creating an empty array. shows.data which is the actual response and then I've tried adding it to my array using showsArr.push(shows.data) previously without much success. I get undefined when I log to console.
Here the ids range from 1 to 240+, in case one wants to try out the endpoint - https://api.tvmaze.com/shows/${id}
How would I go about achieving this? Thanks.
Just like when converting the wishlist array to an array of ids, you would need to push the data items into your new showsArr.
However, this doesn't actually work, since it's asynchronous - you also need to wait for them, using Promise.all on an array of promises. And you actually shouldn't be using push at all with map, a map call already creates an array containing the callback return values for you. So you can simplify the code to
module.exports = {
async fetchAll(req, res, next) {
try {
const wishlist = await Wishlist.find({});
if (wishlist == null) {
res.status(404)
.json({
success: false,
msg: 'No Movies Found in Wishlist',
wishlist: []
})
}
const idsArr = wishlist.map((item) => {
// ^^^^^^^^^^^^^^
return item.id;
// ^^^^^^
});
console.log(idsArr);
const promisesArr = idsArr.map(async (id) => {
const show = await axios.get(`https://api.tvmaze.com/shows/${id}`);
console.log(shows.data);
return shows.data;
// ^^^^^^^^^^^^^^^^^^
});
const showsArr = await Promise.all(promisesArr);
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
console.log(showsArr);
return res.status(200)
.json({
success: true,
msg: 'All Movies in Wishlist Fetched',
wishlist: showsArr
})
} catch (err) {
console.log(err);
next(err);
}
}
};

one query inside another MongoDB hangs

I am trying to get only get Notes [from the notes collection] that come from meetings [from the meetings collection] that don't contain the word 'test' in them:
function getNotes(done) {
noteSchema.find({}).exec((err, notes) => {
var numNotes = 0;
async.each(notes, (n, next) => {
userSchema.findById(n.userId, (err, user) => {
if (err || !user) { next(); return; }
var emailsStr = utils.getEmailsString(user.emails);
if (!utils.toSkipEmail(emailsStr)) {
meetingSchema.findById(n.meetingId, (err, meeting) => {
if (err || !meeting) { next(); return; }
if (meeting.name.displayValue.indexOf('test', 'Test') == -1) {
numNotes++;
}
next();
});
}
})
}, (err, result) => {
console.log(util.format('Total Number of Notes: %d', numNotes));
done(null);
});
});
}
The code works fine without adding in the lines to find the meetings by ID. It hangs at that point.
For reference, here is the start of a function that comes later to filter out any 'test' or 'Test' containing meetings.
function getMeetings(done) {
meetingSchema.find({
'name.displayValue': { '$regex' : '(?!.*test)^.*$' , '$options' : 'i' }
}).exec((err, meetings) => {
Relevant lines of Notes Schema:
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var noteSchema = mongoose.Schema({
meetingId: {type: String, default: ''},
});
exports.Note = mongoose.model('Note', noteSchema);
The meeting schema has no notes field.
So, if I were going to go after a solution like this, where I wanted to get all notes that weren't part of a meeting w/ the word 'test' in the name, I'd probably go the other way with it, unless there's a whole ton of notes you could just get all notes and populate them with their meetings, and then do the filtering. assuming your NoteSchema defines something like:
meeting : { type: ObjectId, ref: 'Meeting'}
then in your query you could do (given the notATest function that returns true or false appropriately:
Note.find({}).populate('meeting').exec((e, n) => {
_.omit(n, (note) => { return notATest(note.meeting.name); });
});
Alternatively, you could search for all meetings that are not a test first, and call .populate('notes') on them, if the 'ref' goes the other way.

Conditionally create new entries using promises in Waterline (Sails.js)

I have an array of "products".
I want to save these products to the database if the database is empty, and when all of the db operations finish i want to display a message.
I could not manage to do it using bluebird promises (using .all or .map). I was able to create an item by just returning Product.create(products[0]). I can't wrap my head around it, I am new to promises.
This is the bootstrap file of my sails.js project but this question is about how to use bluebird promises. How can I manage to wait for multiple async tasks (create 3 products) to finish and then continue?
products = [
{
barcode: 'ABC',
description: 'seed1',
price: 1
},
{
barcode: 'DEF',
description: 'seed2',
price: 2
},
{
barcode: 'GHI',
description: 'seed3',
price: 3
}
];
Product.count()
.then(function(numProducts) {
if (numProducts > 0) {
// if database is not empty, do nothing
console.log('Number of product records in db: ', numProducts);
} else {
// if database is empty, create seed data
console.log('There are no product records in db.');
// ???
return Promise.map(function(product){
return Product.create(product);
});
}
})
.then(function(input) {
// q2) Also here how can decide to show proper message
//console.log("No seed products created (no need, db already populated).");
// vs
console.log("Seed products created.");
})
.catch(function(err) {
console.log("ERROR: Failed to create seed data.");
});
Figured it out...
products = [
{
barcode: 'ABC',
description: 'seed1',
price: 1
},
{
barcode: 'DEF',
description: 'seed2',
price: 2
},
{
barcode: 'GHI',
description: 'seed3',
price: 3
}
];
Product.count()
.then(function(numProducts) {
//if (numProducts > 0) {
if(false) {
// if database is not empty, do nothing
console.log('Number of product records in db: ', numProducts);
return [];
} else {
// if database is empty, create seed data
console.log('There are no product records in db.');
return products;
}
})
.map(function(product){
console.log("Product created: ", product);
return Product.create(product);
})
.then(function(input) {
console.log("Seed production complete.");
})
.catch(function(err) {
console.log("ERROR: Failed to create seed data.");
});

MongoDB: how to insert a sub-document?

I am using sub-documents in my MEAN project, to handle orders and items per order.
These are my (simplified) schemas:
var itemPerOrderSchema = new mongoose.Schema({
itemId: String,
count: Number
});
var OrderSchema = new mongoose.Schema({
customerId: String,
date: String,
items: [ itemPerOrderSchema ]
});
To insert items in itemPerOrderSchema array I currently do:
var orderId = '123';
var item = { itemId: 'xyz', itemsCount: 7 };
Order.findOne({ id: orderId }, function(err, order) {
order.items.push(item);
order.save();
});
The problem is that I obviously want one item per itemId, and this way I obtain many sub-documents per item...
One solution could be to loop through all order.items, but this is not optimal, of course (order.items could me many...).
The same problem could arise when querying order.items...
The question is: how do I insert items in itemPerOrderSchema array without having to loop through all items already inserted on the order?
If you can use an object instead of array for items, maybe you can change your schema a bit for a single-query update.
Something like this:
{
customerId: 123,
items: {
xyz: 14,
ds2: 7
}
}
So, each itemId is a key in an object, not an element of the array.
let OrderSchema = new mongoose.Schema({
customerId: String,
date: String,
items: mongoose.Schema.Types.Mixed
});
Then updating your order is super simple. Let's say you want to add 3 of items number 'xyz' to customer 123.
db.orders.update({
customerId: 123
},
{
$inc: {
'items.xyz': 3
}
},
{
upsert: true
});
Passing upsert here to create the order even if the customer doesn't have an entry.
The downsides of this:
it is that if you use aggregation framework, it is either impossible to iterate over your items, or if you have a limited, known set of itemIds, then very verbose. You could solve that one with mapReduce, which can be a little slower, depending on how many of them you have there, so YMMB.
you do not have a clean items array on the client. You could fix that with either client extracting this info (a simple let items = Object.keys(order.items).map(key => ({ key: order.items[key] })); or with a mongoose virtual field or schema.path(), but this is probably another question, already answered.
First of all, you probably need to add orderId to your itemPerOrderSchema because the combination of orderId and itemId will make the record unique.
Assuming that orderId is added to the itemPerOrderSchema, I would suggest the following implementation:
function addItemToOrder(orderId, newItem, callback) {
Order.findOne({ id: orderId }, function(err, order) {
if (err) {
return callback(err);
}
ItemPerOrder.findOne({ orderId: orderId, itemId: newItem.itemId }, function(err, existingItem) {
if (err) {
return callback(err);
}
if (!existingItem) {
// there is no such item for this order yet, adding a new one
order.items.push(newItem);
order.save(function(err) {
return callback(err);
});
}
// there is already item with itemId for this order, updating itemsCount
itemPerOrder.update(
{ id: existingItem.id },
{ $inc: { itemsCount: newItem.itemsCount }}, function(err) {
return callback(err);
}
);
});
});
}
addItemToOrder('123', { itemId: ‘1’, itemsCount: 7 }, function(err) {
if (err) {
console.log("Error", err);
}
console.log("Item successfully added to order");
});
Hope this may help.

Mongoose subquery and append results to mainquery

I have been struggling with the questions for a months now still no solution.
Basically I have 2 mongodb database structures.
One is called Users and another is called Items.
One user can have multiple Items.
User structure is simple =
Users = [{
_id: 1,
name: "Sam",
email: "sam#gmail.com",
group: "Rangers"
},
{
_id: 2,
name: "Michael",
email: "michael#gmail.com"
group: "Muse"
},
{
_id: 3,
name: "John",
email: "john#gmail.com"
group: "Merchant"
},
.....
]
The Items structures are as follows and each item is assigned to a user.
Items = [
{
_id: 1,
user_id: 1,
item_name: "Flying Sword",
timestamp: ...
},
{
_id: 3,
user_id: 1,
item_name: "Invisible Cloak",
timestamp: ...
},
{
_id: 4,
user_id: 2,
item_name: "Iron Shield"
},
{
_id: 5,
user_id: 7,
item_name: "Splashing Gun",
timestamp: ...
},
...
]
I want to run a mongoose query that queries the user as primary object.
And upon the returning the results of the user object I want to query the all the Items objects with the filtered users and append them as subdocuments to each user objects previously queried.
For example I want to query
Users.find({group: "Muse"}, function(err, users){
I DON"T KNOW WHAT TO WRITE INSIDE
})
Basically the results should be:
[
{
_id: 4,
name: "Jack",
email: "jack#gmail.com",
group: "Muse",
items: [
{
_id: 8
name: "Magic Wand",
user_id: 4,
timestamp: ...
}
{
_id: 12
name: "Blue Potion",
user_id: 4,
timestamp: ...
},
{
_id: 18
name: "Teleportation Scroll",
user_id: 4,
timestamp: ...
}
]
}
.....
More USERS of similar structure
]
Each user will return a maximum of three items which are sorted by timestamp.
Thanks in advance, I tried so many times and failed.
This is a multiple step question. So lets list out the steps:
Get a list of user documents that match a particular group.
Get a list of item documents that are assigned to each matched user from step 1.
Assign the appropriate item documents to a new property on the corresponding user document.
This can be tackled a few ways. A first pass might be to retrieve all the user documents and then iterating over them in memory retrieving the list of item documents for each user and appending that list to the user document. If your lists are smallish this shouldn't be too much of an issue but as scale comes into play and this becomes a larger list it could become a memory hog.
NOTE: all of the following code is untested so it might have typos or the like.
Users.find({group: "Muse"}, function(err, users){
var userIDs;
if (err) {
// do error handling
return;
}
userIDs = users.map(function (user) { return user._id; });
Items.find({user_id: {$in: userIDs}}, function (err, items) {
if (err) {
// do error handling
return;
}
users.forEach(function (user) {
user.items = items.filter(function (item) {
return item.user_id === user._id;
});
});
// do something with modified users object
});
});
While this will solve the problem there are plenty of improvements that can be made to make it a bit more performant as well as "clean".
For instance, lets use promises since this involves async operations anyway. Assuming Mongoose is configured to use the native Promise object or a then/catch compliant library
Users.find({group: "Muse"}).exec().then(function(users) {
var userIDs = users.map(function(user) {
return user._id;
});
// returns a promise
return Promise.all([
// include users for the next `then`
// avoids having to store it outside the scope of the handlers
users,
Items.find({
user_id: {
$in: userIDs
}
}).exec()
]);
}).then(function(results) {
var users = results[0];
var items = results[1];
users.forEach(function(user) {
user.items = items.filter(function(item) {
return item.user_id === user._id;
});
});
return users;
}).catch(function (err) {
// do something with errors from either find
});
This makes it subjectively a bit more readable but doesn't really help since we are doing a lot of manipulation in memory. Again, this might not be a concern if the document collections are smallish. However if is, there is a tradeoff that can be made with breaking up the request for items into one-per-user. Thus only working on chunks of the item list at a time.
We will also use Bluebird's map to limit the number of concurrent requests for items.
Users.find({group: "Muse"}).exec().then(function(users) {
return bluebird.map(users, function(user) {
return Items.find({user_id: user._id}).exec().then(function (items) {
user.items = items;
return user;
});
}, {concurrency: 5});
}).then(function(users) {
// do something with users
}).catch(function(err) {
// do something with errors from either find
});
This limits the amount of in memory manipulation for items but still leaves us iterating over users in memory. That can be tackled as well by using mongoose streams but I will leave that up to you to explore on your own (there are also other questions already on SO on how to use streams).
This makes it subjectively a bit more readable but doesn't really help since we are doing a lot of manipulation in memory. Again, this might not be a concern if the document collections are smallish. However if is, there is a tradeoff that can be made with breaking up the request for items into one-per-user. Thus only working on chunks of the item list at a time.

Resources