I want to increment many documents in mongodb - node.js

router.put('/stockAccepted/:id', (req, res, next) =>{
stockSchema.findOneAndUpdate({_id: req.params.id}, {
$set:{
stockAccepted: req.body.stockAccepted
}
},(err, result) => {
if(err) res.json(err);
else res.json(result);
});
let stockItem = req.body.stockItem;
stockItem.forEach((element) => {
console.log("counter")
productSchema.update({_id: element.productId}, {
$inc:{
productQuantity: element.productQuantity
}
}
,(err, result) => {
if(err) res.json(err);
else res.json(result);
});
});
});
what is the best approach to achieve it?
This code throws an error that "headers are already sent".
How can I implement this correctly?

The problem is you are trying to send a JSON response in each db update callback. You can only send a response once for a request. You can either use Promise.all to wait for the stock and product(s) to be updated, or use Mongoose's bulkWrite feature.
The bulkWrite is more efficient as it only involves one request to the DB rather than multiple.
Here is an (untested) example of how bulkWrite could work with your code:
router.put('/stockAccepted/:id', (req, res, next) => {
stockSchema.findOneAndUpdate({
_id: req.params.id
}, {
$set: {
stockAccepted: req.body.stockAccepted
}
}).then((result) => {
let stockItem = req.body.stockItem;
let updates = []
stockItem.forEach((element) => {
updates.push({
updateOne: {
filter: {
_id: element.productId
},
update: {
$inc: {
productQuantity: element.productQuantity
}
}
}
})
})
return productSchema.bulkWrite(updates)
}).then((result) => {
res.json(result);
}).catch((err) => {
res.json(err);
})
});
I'm using the promise returned from a DB call in mongoose rather than the callbacks, as it reduces nesting.
Unlike your version, this code will wait for stock to be updated in DB. Then it will create a bulk operation and send that to the DB. Once this is completed, it will finally return the response from the bulk operation as JSON.
Note that the final catch handler will catch errors in both the stock update, or the bulk product update.

You are iterating over stockItem with a forEach loop, nothing wrong about that.
What you probably miss here is that after handling correctly the first element you use directly the res params which is why the next iteration when you process the same logic it fails with "headers are already sent" because you indeed, already fire an http response with res.
What I suggest is to aggregate your result during the loop and only fire the res with your aggregate results as json as below :
router.put('/stockAccepted/:id', (req, res, next) =>{
stockSchema.findOneAndUpdate({_id: req.params.id}, {
$set:{
stockAccepted: req.body.stockAccepted
}
},(err, result) => {
if(err) res.json(err);
else res.json(result);
});
let stockItem = req.body.stockItem;
const results = [];
const errors = [];
stockItem.forEach((element) => {
console.log("counter")
productSchema.update({_id: element.productId}, {
$inc:{
productQuantity: element.productQuantity
}
}
,(err, result) => {
if(err) errors.push(err);
else result.push(result);
});
});
res.json({
results,
errors
});
});

Related

Editing Large Collection in MongoDB

I am trying to edit an entire collection in my MongoDB Database.
The collection is about 12k documents in size.
I was trying to edit the files from my angular controller
let promises = [];
array.forEach(each => {
promises.push(this.commonService.postObject('editObject', each));
});
forkJoin(promises).subscribe(data =>{})
My node function
module.exports.editObject = (model) =>{
return function (req, res, next) {
model.findOneAndUpdate({
'_id': req.body._id
}, req.body, {
upsert: true
}, function (err, doc) {
if (err) return res.send(500, {
error: err
});
return res.send(req.body);
});
};
}
But I get the error Message
ERR_INSUFFICIENT_RESOURCES
Is there a smarter way to do that?

Mongo FindOne return results in error parameter

I am doing a login system in NodeJS with a Mongo database, so I try to look in my collection if the user exist. For my tests, I have a user registered, and I'm trying to find him.
My problem is that findOne method return the full user' data as I wish, but in the error parameter, not in the results parameter, and I have no logs to understand why...
Did somebody know why ?
Here is my code:
app.post('/login', (req, res) =>{
console.log(req.body.identifier);
console.log(req.body.password);
client.connect().then(() => {
let newUser = {identifier : req.body.identifier}
res.redirect(req.body.locator)
return client.db(`${process.env.MONGODB}`).collection(`${process.env.MONGOCOLLECTION}`).findOne(newUser).then((err, res) => {
if (err){
console.log("ERROR: "+err.role)
throw err;
}else if(res){
console.log("user found");
console.log(res.role)
}
})
}).catch( e => { console.error(e) }).then(() => {
console.log("--------------------------------");
})
})
And this is what I got :
mail#mail.com
azer
ERROR: USER
{
_id: 6087d850ad9f6f2e0ce97045,
identifier: 'mail#mail.com',
password: '7657d9148a5720dcf4eb4b8bc998498e5d701ce7beb302f398c3d5c0dbd0f857f824b7bfaa45c2a8aba4f85c4ab8b12c99bfb28328e72a89afe11326dc1d3a38349c1c36790a24c910528ada34529e6736ae45f0e5d87ce6b109207e21169bc9b4056fff',
role: 'USER',
sign_up: 2021-04-27T09:24:32.616Z,
name: 'Test',
firstName: 'Test'
}
--------------------------------
This occurs because your code mixes Promises and callbacks. In particular:
findOne(newUser).then((err, res) => {
// (this does not work because .then() takes a function with 1 argument)
})
Promises have a built-in error handling mechanism where a resolver can either resolve or reject the promise, triggering the .then() (success) or the .catch() (failure) branch.
See the note at the top of the relevant manual page. Also, MongoDB provides a short guide on Promises vs. callbacks - pay attention especially to this snippet:
collection
.updateOne({ name: "Mount McKinley" }, { $set: { meters: 6190 } })
.then(
res => console.log(`Updated ${res.result.n} documents`),
err => console.error(`Something went wrong: ${err}`),
);
Note how .then() accepts two functions above - these are separate callbacks for the "success" case and the "error" case.
Read - Promises and Callbacks
Here you're using promise - then will give the result and catch will give the error.
client.db(`${process.env.MONGODB}`)
.collection(`${process.env.MONGOCOLLECTION}`)
.findOne(newUser)
.then( res => { ... }) // result
.catch(err => { ... }) // error
Callback style
client.db(`${process.env.MONGODB}`)
.collection(`${process.env.MONGOCOLLECTION}`)
.findOne(newUser, (err, res) => { ... })
The mongodb was returning the data, and the data was taken as first argument in the then block named as err, try changing to this:
app.post("/login", (req, res) => {
console.log(req.body.identifier);
console.log(req.body.password);
client
.connect()
.then(() => {
let newUser = { identifier: req.body.identifier };
res.redirect(req.body.locator);
return client
.db(`${process.env.MONGODB}`)
.collection(`${process.env.MONGOCOLLECTION}`)
.findOne(newUser, (err, res) => {
if (err) {
console.log("ERROR: " + err.role);
throw err;
} else if (res) {
console.log("user found");
console.log(res.role);
}
});
})
.catch((e) => {
console.error(e);
})
.then(() => {
console.log("--------------------------------");
});
});

Deleting the model data through lodash and save() not persisting model in mongodb

I am trying to remove one object from the User collection like this
router.post('/accept-trades', function (req, res, next) {
const {senderName, receiverName, senderId} = req.body;
const user = req.user;
console.log(senderName, receiverName);
if (senderName) {
User.findOne({ name: senderName })
.then(sender => {
_.remove(user.receivedTradeRequest, {username: senderName});
_.remove(sender.sentTradeRequest, {username: receiverName});
console.log('user.receivedTradeRequest', user.receivedTradeRequest);
console.log('\n\nuser.sentTradeRequest', user.sentTradeRequest);
async.parallel([
function (cb) {
user.save()
.then(isSave => {
cb(null, true);
})
.catch(err => {
cb(err, null);
});
},
function (cb) {
sender.save()
.then(isSave => {
cb(null, true);
})
.catch(err => {
cb(err, null);
});
}
], (err, results) => {
if (err) {
return res.status(500).json({
message: 'Error: Trade is invalid as Card is already traded!',
});
}
res.send('done');
//res.redirect('/trade');
});
})
.catch(err => {
throw err;
});
} else {
return res.status(500).json({
message: 'Only accessible to logged in users!',
});
}
});
Here, user is accessed by req.user (i'm using passport).
When i log the user after removal, user.receivedTradeRequest and sender.sentTradeRequest printing empty array which is the correct behaviour.
But when i see the mongodb the array still present for the username.
Could you please suggest what is wrong with the code ?
PS: I know about the mongodb $pull for removal. I am doing some other computation on the user data so had to do with above approach.
I was able to solve it by re-assigning the array after removing the element. Used _.filter instead of _.remove solves the problem.
One thing i don;t understand is the lodash _.remove update the original array after deletion but that clearly is not the case here.

ExpressJS Multiple middlewares connected to callbacks

I have an ExpressJS app which takes form data and does the following:
1. checks all required values are supplied,
2. validates the data is valid,
3. adds a record to the database to get a unique ID,
4. uses the ID and data to call a separate server,
5. upon response from the server, update the database record with details of the response.
I'm using mongoskin for the database.
My question relates to how I control the flow. Essentially I have written each of the above steps as a middleware function because I need to call next() on the success (or next(err) on error) at each callback.
It seems like I'm writing too much middleware and should be able to group the steps into larger sets of middleware containing multiple 'sub-functions' but I'm not sure how to do this in Express since I need to call next() every time an async function call completes. Is there a correct way to do this or is this 'one middleware per step' approach really the right way to run this?
EDIT: Posting some code as requested. This is partial code for the sake of brevity:
function validateFields(req, res, next) {
//...
//iterate over req.body to confirm all fields provided
//...
if (allDataProvided) {
//...
//iterate over req.body to confirm all fields valid
//...
if (allDataValid) {
return(next());
} else {
return(next(err));
}
} else {
return(next(err));
}
},
//get an auto incrementing ID fields from a mongodb collection (counters)
function getNextID(req, res, next) {
counters.findAndModify(
{ _id: "receiptid" },
[['_id','asc']],
{ $inc: { seq: 1 } },
{},
function(err, doc) {
if (err) {
return next(err);
} else {
req.receiptid = doc.seq;
return next();
}
});
},
//insert a new record into the transaction collection (txns) using the new ID
function createTransaction(req, res, next) {
txns.insert(
{ _id : req.receiptid,
body : req.body,
status : "pending"},
{},
function(err, r) {
if (err) {
return next(err);
} else {
return next();
}
});
},
//process the data on the remote web service using the provider's API (remoteapi)
function processTransaction(req, res, next) {
remoteapi.processTransaction(
{ data: req.body,
receiptid: req.receiptid },
function(err, r) {
if (err) {
return next(err);
} else {
req.txnReceipt = r;
return next();
}
});
},
//update the record in the database collection (txns) with the server response
function updateDatabase(req, res, next) {
txns.updateById(req.receiptid,
{ $set :{status : "success",
receipt: req.txnReceipt }
}, function (err, r) {
if (err) {
return next(err);
} else {
return next();
}
});
}
And as it currently stands with the above functions, my route which utilises this middleware starts like this:
router.post('/doTransaction',
validateFields,
getNextID,
createTransaction,
processTransaction,
updateDatabase,
function(req, res, next) { //...
It seems like I should be able to create one middleware function which does all of these things in a row without each having to be a separate middleware, but since each middleware has an async function in it and I need to call next() in the resulting callback, this is the only way I can see it working.
Thanks
Aaron
It's fairly easy to implement all your steps in one middleware. I've included some pseudo-code below (that makes various assumptions on how your code is structured, because you didn't provide implementation details, but it's just to give an idea).
It uses the on-headers package to "catch" responses.
var onHeaders = require('on-headers')
// Your middleware function
app.use(function(req, res, next) {
// Update the database when the response is being sent back.
onHeaders(res, function() {
// Do database update if we have a document id.
if (req._newDocumentId) {
db.collection.update(req._newDocumentId, data, function() {
// can't do a lot here!
});
}
});
// Perform the requires steps
if (! checkValuesAreSupplied(req)) {
return next(new Error(...));
}
if (! validateValues(req)) {
return next(new Error(...));
}
// Insert into database.
db.collection.insert(data, function(err, doc) {
if (err) return next(err);
...process the newly created doc...
// Store _id in the request for later.
req._newDocumentId = doc._id;
// Make the call to the separate server
makeCallToOtherServer(otherData, function(err, response) {
if (err) return next(err);
...process response...
return next();
});
});
});
You can put everything in one module and just use callbacks to go trought each step but in this case you can get "callback hell".
So I can suggest the async npm package which I think the better way.
using this library your code will look like:
function allInOneMiddleware(req, res, next) {
async.waterfall([
function (callback) {
validateFields(req, res, callback);
},
getNextID,
createTransaction,
processTransaction,
updateDatabase
], function (err) {
if (err) {
return next(err);
}
// response?
});
}
function validateFields(req, res, callback) {
//...
//iterate over req.body to confirm all fields provided
//...
if (allDataProvided) {
//...
//iterate over req.body to confirm all fields valid
//...
if (allDataValid) {
return callback(null, req.body);
}
return callback(err);
}
return callback(err);
}
//get an auto incrementing ID fields from a mongodb collection (counters)
function getNextID(body, callback) {
counters.findAndModify(
{_id: "receiptid"},
[['_id', 'asc']],
{$inc: {seq: 1}},
{},
function (err, doc) {
if (err) {
return callback(err);
}
callback(null, body, doc.seq);
});
}
//insert a new record into the transaction collection (txns) using the new ID
function createTransaction(body, receiptid, callback) {
txns.insert(
{
_id: receiptid,
body: body,
status: "pending"
},
{},
function (err, r) {
if (err) {
return callback(err);
}
callback(null, body, receiptid);
});
}
//process the data on the remote web service using the provider's API (remoteapi)
function processTransaction(body, receiptid, callback) {
remoteapi.processTransaction(
{
data: body,
receiptid: receiptid
},
function (err, r) {
if (err) {
return callback(err);
}
callback(null, receiptid, r);
});
}
//update the record in the database collection (txns) with the server response
function updateDatabase(receiptid, txnReceipt, callback) {
txns.updateById(receiptid,
{
$set: {
status: "success",
receipt: txnReceipt
}
}, callback);
}
Thanks Nicolai and robertklep for the answers. Whilst I think both answers do answer the question, I realised as I was working through this myself that I had failed to see the forest for the trees.
I could just pass the next function through each callback function until I reached the final one and call it to pass the control back to the middleware stack. This also allows me to simply call next(err) inside any of those functions.
So my answer is very similar to the concept outlined by Nicolai except I don't think I need to use the async package in this case because I don't feel like this particular case took me to callback hell.
Here is my answer to my own question:
function validateFields(req, res, next) {
//...
//iterate over req.body to confirm all fields provided
//...
if (allDataProvided) {
//...
//iterate over req.body to confirm all fields valid
//...
if (allDataValid) {
getNextID(req, res, next)
} else {
return(next(err));
}
} else {
return(next(err));
}
},
//get an auto incrementing ID fields from a mongodb collection (counters)
function getNextID(req, res, next) {
counters.findAndModify(
{ _id: "receiptid" },
[['_id','asc']],
{ $inc: { seq: 1 } },
{},
function(err, doc) {
if (err) {
return next(err);
} else {
req.receiptid = doc.seq;
createTransaction(req, res, next);
}
});
},
//insert a new record into the transaction collection (txns) using the new ID
function createTransaction(req, res, next) {
txns.insert(
{ _id : req.receiptid,
body : req.body,
status : "pending"},
{},
function(err, r) {
if (err) {
return next(err);
} else {
processTransaction(req, res, next);
}
});
},
//process the data on the remote web service using the provider's API (remoteapi)
function processTransaction(req, res, next) {
remoteapi.processTransaction(
{ data: req.body,
receiptid: req.receiptid },
function(err, r) {
if (err) {
return next(err);
} else {
req.txnReceipt = r;
updateDatabase(req, res, next);
}
});
},
//update the record in the database collection (txns) with the server response
function updateDatabase(req, res, next) {
txns.updateById(req.receiptid,
{ $set :{status : "success",
receipt: req.txnReceipt }
}, function (err, r) {
if (err) {
return next(err);
} else {
return next();
}
});
}
So instead of calling next() on successful completion of each async function and having to write another middleware for the next step, I simply pass next on to the next function until it's required.
This was, I can just call the first function as my middleware, like this:
router.post('/doTransaction',
validateFields,
function(req, res, next) { //...
and in turn, the remaining steps are called in sequence when each action completes.

NodeJS / Mongoose Filter JSON

I am building a JSON API with ExpressJS, NodeJS and Mongoose:
Input -> id:
app.get('/folder/:id', function (req, res){
return Cars.find({reference: req.params.id}, function (err, product) {
if (!err) {
console.log(product);
return res.send(product);
} else {
return console.log(err);
}
});
});
It shows well the JSON:
[{"_id":"B443U433","date":"2014-08-12","reference":"azerty","file":"087601.png","
....:.
{"_id":"HGF6789","date":"2013-09-11","reference":"azerty","file":"5678.pnf","
...
I just want to display the _id in the JSON, so it is good when I have lots of data.
How I can do that? Something like a filter?
You can chain calls to select and lean to retrieve just the fields you want from the docs you're querying:
app.get('/folder/:id', function (req, res){
return Cars.find({reference: req.params.id}).select('_id').lean().exec(
function (err, product) {
if (!err) {
console.log(product);
return res.send(product);
} else {
return console.log(err);
}
});
});
You would have to iterate over your "products" object to obtain the ids
Something like this:
(Disclaimer: I haven't tested this)
app.get('/folder/:id', function (req, res){
return Cars.find({reference: req.params.id}, function (err, product) {
if (!err) {
console.log(product);
var ids = new Array();
for(var i = 0; i < product.length; i++){
ids.push(product[i]._id);
}
return res.send(JSON.stringify(ids));
} else {
return console.log(err);
}
});
});
--Edit
Also, "products" may already be a JSON string. You may want to parse it before looping.
product = JSON.parse(product);
Other answers are true but I think it's better to limit data in mongoose like this :(it's same as mongo shell commands)
app.get('/folder/:id', function (req, res){
Cars.find({reference: req.params.id} ,{ _id : true } ,function (err, product) {
if (!err) {
console.log(product);
} else {
console.log(err);
}
});
});

Resources