NodeJS, ExpressJS, Mongoose 4.4, Nonce for concurrency and Document.update - node.js

I have the following combination, NodeJS, Express, MongoDB, and Mongoose. I have implemented a nonce with mongoose promises to allow for concurrent edits. See the following.:
//schema
var item_schema = {
//_id: {type: Schema.ObjectId, required: true},
name: {type: String, required: true, index: { unique: true }},
code: {type: String, required: true, index: { unique: true }},
date_time_created: {type: Date, required: true},
date_time_updated: {type: Date, required: true},
nonce: {type: Schema.ObjectId}
};
//model
var item_model = mongoose.model('item', schema);
//update
var concurrency_update = function(req, res, retries) {
var promise = model.findById(req.params.id).exec();
var updated_nonce = mongoose.Types.ObjectId();
promise.then(function(document){ //find document response
if(!document) {
res.status = 404;
return Promise.reject( { "message" : req.params.id + ' document does not exist' } );
}
var now = new Date();
if(req.body.code) {
document.code = req.body.code;
document.date_time_updated = now;
}
if(req.body.name) {
document.name = req.body.name;
document.date_time_updated = now;
}
if(!document.nonce) {
document.nonce = updated_nonce;
var old_nonce = document.nonce;
}
else {
var old_nonce = document.nonce;
document.nonce = updated_nonce;
}
return document.update({ "_id" : req.params.id, "nonce" : old_nonce }).exec();
}).then(function(raw){ //update response
var number_affected = raw.n;
console.log(raw);
if(!number_affected && retries < 10){
//we weren't able to update the doc because someone else modified it first, retry
console.log("Unable to update, retrying ", retries);
//retry with a little delay
setTimeout(function(){
concurrency_update(req, res, (retries + 1));
}, 20);
} else if (retries >= 10){
//there is probably something wrong, just return an error
return Promise.reject({ "message" : "Couldn't update document after 10 retries in update"});
} else {
res.json({"message": req.params.id + ' document was update'});
}
}).catch(function(err){
res.send(err.message);
});
The concurrency update is based of of this:
http://www.mattpalmerlee.com/2014/03/22/a-pattern-for-handling-concurrent/
and reading the mongoose docs the update is based off of this.
http://mongoosejs.com/docs/api.html#document_Document-update
However, when the code enters the final .then (//update response) I see the raw.n (numberAffected) = 1 but the database never gets updated?
The answers probably close but I am missing it.
What am I missing on this?

After the comment by #blakes_seven, I was able to remove the use of nonce and apply updates using atomic modifiers. Here is the updated tested code.
//schema
var item_schema = {
//_id: {type: Schema.ObjectId, required: true},
name: {type: String, required: true, index: { unique: true }},
code: {type: String, required: true, index: { unique: true }},
date_time_created: {type: Date, required: true},
date_time_updated: {type: Date, required: true},
nonce: {type: Schema.ObjectId}
};
//model
var item_model = mongoose.model('item', schema);
//update
var concurrency_update = function(req, res, retries) {
var updated_data = {};
if(req.body.code) {
updated_data.code = req.body.code;
}
if(req.body.name) {
updated_data.name = req.body.name;
}
if(!req.body.name.nonce) {
updated_data.nonce = mongoose.Types.ObjectId();
}
if(updated_data !== {}) {
var update = {
$currentDate: {
date_time_updated: true
},
$set: updated_data
};
var promise = model.update({"_id": req.params.id}, update).exec();
promise.then(function (raw) {
var number_affected = raw.nModified;
console.log(raw);
if (!number_affected && retries < 10) {
//we weren't able to update the doc because someone else modified it first, retry
console.log("Unable to update, retrying ", retries);
//retry with a little delay
setTimeout(function () {
concurrency_update(req, res, (retries + 1));
}, 20);
} else if (retries >= 10) {
//there is probably something wrong, just return an error
return Promise.reject({"message": "Couldn't update document after 10 retries in update"});
} else {
res.json({"message": req.params.id + ' document was update'});
}
}).catch(function (err) {
res.send(err.message);
});
}
else {
res.status = 400;
res.send({"message": 'There is nothing specified in the payload to update!'})
}
};

Related

nodeJS - findOne query did not cast into JSON object even query result was good

findOne was able to find document in MongoDB. However, the result was not rendered into JSON object employeeLs. Findone result was printed in console and it was clear that query returned good result.
I am new to Angular, NodeJS and MongoDB. Many hours have been spent on researching this issue and could not resolve this "strange" issue. "strange" because findOne was initiated by "employeeLs." and expected result to be directly passed to object employeeLs. Besides, it was a straight ask of all fields for employeeLs. Any help is appreciated.
Below pls find coding info.
route coding
const express = require('express');
const mongoose = require('mongoose');
//const Schema = mongoose.Schema;
const bodyParser = require('body-parser');
const { Console } = require('console');
const mongooseClient = require('mongoose'); /* this is important */
var employeeLs = require('../model/staffLeave');
const staffLeaveRouter = express.Router();
const util = require('util');
staffLeaveRouter.use(bodyParser.json());
staffLeaveRouter.use(bodyParser.urlencoded({ extended: false }));
staffLeaveRouter.route('/:staff_id')
.get((req,res,next) => {
employeeLs.findOne({staff_id : req.params.staff_id})
.populate(staffRecords._id)
.then(result => {
console.log(result);
console.log('lllll - inside retreival by staffid',JSON.stringify(util.inspect(employeeLs,{showHidden:false,depth:5})));
/* console.dir(employeeLs,this.options); */
employeeLs : result;
}
)
.then((employeeLs) => {
if (employeeLs != null)
{res.statusCode = 200;
res.setHeader('Content-Type', 'application/json');
res.json(employeeLs)}
else{
err = new Error('Staffleave using staff ID to find '+req.params.staff_id+' not found');
err.status = 404;
return next(err);
}
}, (err) => next(err))
.catch((err) => next(err));
})
Schema coding
const mongoose = require('mongoose');
const Schema = mongoose.Schema;
require('mongoose-currency').loadType(mongoose);
const Currency = mongoose.Types.Currency;
const staffLeaveSchema = new Schema({
_id: {type: mongoose.Types.ObjectId, required: true},
staff_id: {type: mongoose.Types.ObjectId, required: true,
ref: 'staffRecord'},
staffUsername: {type: String, required: true},
year: {type: Number, required: true},
annualLeaveEntitlement: {type: Number, required: true},
carryOver: {type: Number, required: true},
thisYearTotalAllowed: {type: Number, required: true},
taken: {type: Number, required: false},
scheduled: {type: Number, required: false},
thisYearRemaing: {type: Number, required: false},
sickLeaveTaken: {type: Number, required: false},
maternityLeaveTaken: {type: Number, required: false},
noPayLeaveTaken: {type: Number, required: false},
familyLeaveTaken: {type: Number, required: false},
});
var staffLeave = mongoose.model('staffLeave', staffLeaveSchema);
module.exports = staffLeave;
From console log, findone result returned as below. However, system still said employeeLs was still null. It was not rendered as expected.
{
_id: 5f069b7821233471ec530b80,
staff_id: 5edfd3bf07cc253f1065de24,
staffUsername: 'dwong',
year: 2020,
annualLeaveEntitlement: 10,
carryOver: 2,
thisYearTotalAllowed: 12,
taken: 0,
scheduled: 0,
thisYearRemaing: 12,
sickLeaveTaken: 0,
maternityLeaveTaken: 0,
noPayLeaveTaken: 0,
familyLeaveTaken: 0,
__v: 0
}
I dont really understand what this code bellow want achieve :
.then(result => {
console.log(result);
console.log('lllll - inside retreival by staffid',JSON.stringify(util.inspect(employeeLs,{showHidden:false,depth:5})));
/* console.dir(employeeLs,this.options); */
employeeLs : result;
}
)
do you want the next(second) .then to get the result from first .then ?
if yes
.then(result => { return result })
.then((employeeLs) => {
if (employeeLs != null)
{res.statusCode = 200;
res.setHeader('Content-Type', 'application/json');
res.json(employeeLs)}
else{
err = new Error('Staffleave using staff ID to find '+req.params.staff_id+' not found');
err.status = 404;
return next(err);
}
})
Let me expect what do you want achieve, do you want fetch a document with mongoose and send it as response json?
I think you just need something like this
staffLeaveRouter.route('/:staff_id')
.get((req,res,next) => {
employeeLs.findOne({staff_id : req.params.staff_id})
.populate(staffRecords._id)
.then(result => {
if (result) {
res.status(200).send({data : result})
} else {
res.status(204).end()
}
})
.catch(err => {
console.log(err)
res.status(500).send({ err : "error" })
})
})

Mongoose populate() returns empty array

I want to query the subdocument array by the property 'token' in clientSchema. But I'm not able to populate the subdocument array. It always returns empty value.
This is what I'm tried
var performAuthAsync = promise.promisify(performAuth);
var response = {};
performAuthAsync(req).then(function (client) {
sendStatus(res, 200, { "success": "true", "value": client });
}).catch(ApiError, function (e) {
response.error = "true";
response.message = e.message;
if (e.message == "Invalid Authorization" || e.message == "Unauthorized access") {
console.log(e.message);
sendStatus(res, 401, response, req.query.type);
}
else {
sendStatus(res, 500, response, req.query.type);
}
});
PerformAuth method
function performAuth(req, callback) {
try {
var authHeader = req.headers.authorization;
console.log(authHeader);
//error in req format
if (!authHeader || !authHeader.startsWith("Basic ")) {
console.log("inside fail authheader");
return callback(new ApiError("Invalid Authorization"));
}
authHeader = authHeader.replace("Basic ", "");
authHeader = Buffer.from(authHeader, 'base64').toString('ascii');
console.log(authHeader);
//temporary populate check
clientApp.findOne({}).populate({
path: 'appClients',
model: 'TClient'
}).exec(function (error, apps) {
console.log("populated apps check " + apps); //object containing empty array
//{ _id: 5987099f2cb916a0de80f067,
// appSecret: 'THisIsSecret',
// appId: 'W5ikGw16dQjgWm8bGjqdAwi1IDR2XibD3XESYokH',
// appClients: [] }
// mongo console output
// { "_id" : ObjectId ("5987099f2cb916a0de80f067"),
// "appSecret" : "THisIsSecret",
// "appId" : "W5ikGw16dQjgWm8bGjqdAwi1IDR2XibD3XESYokH",
// "appClients" : [ ObjectId("59881a64dbab536016e7f970") ], "__v" : 0 }
});
clientApp.findOne({}).populate('appClients').findOne({
'appClients.token': authHeader
}).exec(function (error, client) {
if (error) {
console.log("inside dberror");
console.error(error);
return callback(error, null);
}
if (!client) {
return callback(new ApiError("Unauthorized access"), null);
}
return callback(client);
});
}
catch (exception) {
console.log("inside exception");
console.error(exception);
return callback(exception, null);
}
}
Clientapp and client schemas: ( they are in different files)
var appSchema = new Schema({
appId: {
type: String,
required: true,
unique: true
},
appSecret: {
type: String,
required: true,
unique: true
},
appClients: [{ type: Schema.Types.ObjectId, ref: 'TClient' }],
createdAt: Date,
modifiedAt: Date
});
// model
var clientApp = mongoose.model('ClientApp', appSchema);
var clientSchema = new Schema({
clientId: {
type: String,
required: true,
unique: true
},
info: {
type: String,
required: true,
},
token: {
type: String,
required: true,
unique: true
},
createdAt: Date,
modifiedAt: Date
});
// model
var tclient = mongoose.model('TClient', clientSchema);
What I'm doing wrong? Any help is appreciated
The findOne() function return only one document. After that the populate method populates the sub document value. But the next findOne() doesn't work, so you get a null response.
You could try this instead:
async function performAuth(){
let authHeader = authHeader.replace("Basic ", "");
authHeader = Buffer.from(authHeader, 'base64').toString('ascii');
const clientApp = await clientApp.find({})
.populate('appClients')
.map(data => data.find(item=>item.appClients.token === authHeader).exec();
}
Operations Performed
Get all the clientApps
Populate the appClients
Find the clientApp whose token matches the authHeader

Mongodb and Mongoose error: E11000 duplicate key error index

So I've been working on a project and I finished most of it, but then this error popped up, saying there is something that is undefined, here is the error:
E11000 duplicate key error index: build-a-voting-app.polls.$votedIp_1 dup key: { : undefined }
Here is my code for my create new mongo schema file (polls.model.js)
const mongoose = require('mongoose');
const Schema = mongoose.Schema;
const pollSchema = new Schema({
title: { type: String, unique: true, required: true },
choices: [
{
title: { type: String, required: true },
count: { type: Number, default: 0 }
}
],
votedIp: [{ type: String, unique: true }],
createdAt: {type:Date, default:Date.now()},
createdBy: String
});
const Poll = mongoose.model('polls', pollSchema);
module.exports = Poll;
Here is the function where I add the inputs
function submitVote(field, res, ip) {
Poll.findOneAndUpdate(
{ choices: { $elemMatch: { title: field } } },
{ $inc: { 'choices.$.count': 1 }, $addToSet: { 'votedIp': ip } },
{ new: true },
function (err, poll) {
if (err) throw err;
res.json({ updated: poll });
}
);
}
Here is how I first created it
var newPoll = new Poll({
title: req.body.title,
choices: choicesArr,
createdBy: req.session.user.username || req.session.user
}).save(function (err, poll) {
if (err) throw err
res.redirect('/mypolls')
});
If you want to see the full code please go to https://github.com/ElisaLuo/Freecodecamp-Build-A-Voting-App
I'm using the ip addresses for checking if the user has voted or not (I'm building a voting app), but right now, I cannot even create a new schema / poll. Does anyone know why the error happens and how I can solve it?
#Elisa l - you may want to read this - mongoose enforce unique attribute on subdocument property
However, I did manage to test with mongoose-mock and the behavior is as expected - test results below (please do check the two versions of votedIp in the test code snippets)
and as described in the MongoDb document referenced in the above link. Mongoose does not enforce the unique integrity, MongoDb does.
With the mocha test below (inserted as snippets, not to run the code but just for better readability, please ignore the messy look of the comments in the code but the permutation and combination had to be worked out!), I did manage to create the mongoose schema by adding a create method in "Poll". please note the change in the schema - votedIp: { type: String, unique: true }, you can change it to array in the test code.
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var pollSchema = new Schema({
title: { type: String, unique: true, required: true },
choices: [
{
title: { type: String, required: true },
count: { type: Number, default: 0 }
}
],
votedIp: { type: String, unique: true },
createdAt: {type:Date, default:Date.now()},
createdBy: String
});
// Below code added by RJv (ie. me :)
var NewPoll = new mongoose.Schema(pollSchema);
NewPoll.statics.create = function (params, callback) {
var newUpdate = new NewPoll(params);
newUpdate.save(function(err, result) {
callback(err, result);
});
return newUpdate;
};
var Poll = mongoose.model('Model', NewPoll);
module.exports = Poll;
var expect = require('chai').expect,mongooseMock = require('mongoose-mock'),proxyquire=require('proxyquire'),
sinon = require('sinon'), chai=require('chai'),sinonChai = require("sinon-chai");chai.use(sinonChai);
var Poll;
before(function(done){
Poll = proxyquire('./Poll', {'mongoose': mongooseMock});
done();
})
describe('model', function() {
/* beforeEach(function (done) {
Poll = proxyquire('./Poll', {'mongoose': mongooseMock});
done();
});*/
it("should be called once",function(done){
setTimeout(done, 15000);
var callback = sinon.spy();
var poll1 = Poll.create({ "title": 'jv', "choices": [{"title":"jv#gmail.com","count":"1"}],
"votedIp":"9.1.2.1","createdAt":"23/07/2017","createdBy":"Lisa"}, callback);
// Below to pass data for votedIp as an array as described in the original schema by Elisa
//"votedIp":[{"add1":"9.","add2":"1.","add3":"2.","add4":"1"}],"createdAt":"23/07/2017","createdBy":"Lisa"}, callback);
//expect(poll1.votedIp[0].add1+poll1.votedIp[0].add2+poll1.votedIp[0].add3+poll1.votedIp[0].add4).equals("9.1.2.1");
expect(poll1.save).calledOnce;
console.log(JSON.stringify(poll1));
expect(poll1.votedIp).equals("9.1.2.1");
done();
});
it('should expect same ip to get added', function(done) {
this.timeout(5000);
setTimeout(done, 15000);
var callback = sinon.spy();//mock(new Poll({ title: 'jv', choices: [{title:"jv#gmail.com","count":"1"}], votedIp:[{ad1:"9.",add2:"1.",add3:"2.",add4:"1"}],createdAt:"25/07/2017",createdBy:"Lisa"}));
var poll = Poll.create({ "title": 'jv', "choices": [{"title":"jv#gmail.com","count":"1"}],
"votedIp":"9.1.2.1","createdAt":"23/07/2017","createdBy":"Lisa"}, callback);
// var poll = Poll.create({ "title": 'jv', "choices": [{"title":"jv#gmail.com","count":"1"}],
// Below to pass data for votedIp as an array as described in the original schema by Elisa
// "votedIp":[{"add1":"9.","add2":"1.","add3":"2.","add4":"1"}],"createdAt":"25/07/2017","createdBy":"Lisa"}, callback);
// expect(poll.votedIp[0].add1+poll.votedIp[0].add2+poll.votedIp[0].add3+poll.votedIp[0].add4).equals("9.1.2.1");
expect(poll.save).calledOnce;
expect(poll.votedIp).equals("9.1.2.1");
//assert(spy.calledOnce);
done();
});
});
Are you calling submitVote multiple times in quick succession? You might be running into https://jira.mongodb.org/browse/SERVER-14322.
The suggested fix for this is to check the error and if one of the calls fails retry it.
https://docs.mongodb.com/manual/reference/method/db.collection.update/#use-unique-indexes

mongoose update model before saving

I have a model:
const wordSchema = mongoose.Schema({
author: {type: Object, default: 'unknown'},
quote: String,
source: {type: String, default: 'unknown', index: true},
rating: {type: Number, default: 0},
createdAt: {type: Date, default: Date.now},
updatedAt: {type: Date, default: Date.now},
});
Now after receiving a POST request to my server, I want to make a GET request to wikipedia, and get the author info, then append it to my model as an object, and write this model into my database.
app.post('/', function(req, res) {
let author = {};
let quote = new Word({
author: req.body.author,
quote: req.body.quote,
source: req.body.source,
rating: req.body.rating,
});
let authorName = req.body.author.replace(/ /g, '%20');
let url = 'https://en.wikipedia.org/w/api.php?action=query&format=json&titles=' + authorName + '&prop=pageimages|extracts&pithumbsize=200&exsentences=10&exintro=true';
request.get(url, (error, response, body) => {
if(error) {
return error;
}
let data = JSON.parse(body);
let pageID;
for(page in data.query.pages) {
pageID = page;
}
author = {
name: req.body.author,
thumbnail: data.query.pages[pageID].thumbnail.source,
flavorText: data.query.pages[pageID].extract,
};
});
// Save the quote
quote.pre('save', (next) => {
this.author = author;
})
quote.save(function(err, quote) {
if (err) {
res.send(err);
}
res.redirect('/words');
});
});
Now I tried updating the value with the .pre function, but I am getting
quote.pre is not a function
What would be the 'correct way of doing this and what am I doing wrong?
quote is not a Mongoose schema, but wordSchema is so it needs to be as follow:
wordSchema.pre('save', (next) => {
this.quote = whteverYouWantToAssignWith;
})
But actually you don't need it in your use case as far as I understand, you may achieve what you want as follow:
app.post('/', function(req, res) {
let authorName = req.body.author.replace(/ /g, '%20');
let url = 'https://en.wikipedia.org/w/api.php?action=query&format=json&titles=' + authorName + '&prop=pageimages|extracts&pithumbsize=200&exsentences=10&exintro=true';
request.get(url, (error, response, body) => {
if(error) {
return error;
}
let data = JSON.parse(body);
let pageID;
for(page in data.query.pages) {
pageID = page;
}
let quote = new Word({
author: {
name: req.body.author,
thumbnail: data.query.pages[pageID].thumbnail.source,
flavorText: data.query.pages[pageID].extract,
},
quote: req.body.quote,
source: req.body.source,
rating: req.body.rating,
});
quote.save(function(err, quote) {
if (err) {
res.send(err);
}
res.redirect('/words');
});
});
});

Schema declaration and subdocument

Can you tell me what i'm doing wrong ?
var ObjectId = Schema.Types.ObjectId;
var ProductSchema = new Schema({
name: { type: String, required: true },
price: { type: Number, required: true },
category : { type: String, required: true }
});
var OrderSchema = new Schema({
products: [{
product: {type: ObjectId, ref: 'Product'},
quantity: {type: Number}
}],
status: { type: String, required: true }
});
Product = mongoose.model('Product', ProductSchema);
Order = rmongoose.model('Order', OrderSchema);
OrderSchema.statics.addOrder = function (data, cb) {
// data: array of products ID
var newOrder = new Order();
data.data.forEach(function(element, index, array) {
Product.findOne({ '_id': element.id }, function (err, product) {
if (err) return handleError(err);
newOrder.products.push({product: product, quantity: element.quantity});
})
});
newOrder.status = 'waiting';
newOrder.save(function (err, order) {
if (err) cb(err, false);
console.log(order);
var response = json.stringify({
event: 'addOrder',
success: true,
data: order.id
});
cb(false, response);
});
}
When i add an order products, array is always empty but i have no error. Maybe it's the wrong to do what i want.
Data send by the client are good and the foreach and findOne work well but push seems doing nothing.
If there is no solution maybe you can try to help me to find another solution.
Thanks :)
That's because you need to wait for all the products to be found.
Try this (untested):
OrderSchema.statics.addOrder = function (data, cb) {
// data: array of products ID
var newOrder = new Order();
var productIds = [];
var quantity = [];
data.data.forEach(function(element, index, array) {
productIds.push(element.id);
quantity.push(element.quantity);
});
Product.find({ '_id' : { $in: productIds} }, function(err, products) {
if (err) return handleError(err);
products.forEach(function(product, index) {
newOrder.products.push({product: product, quantity: quantity[index]});
});
newOrder.status = 'waiting';
newOrder.save(function (err, order) {
if (err) cb(err, false);
console.log(order);
var response = json.stringify({
event: 'addOrder',
success: true,
data: order.id
});
cb(false, response);
});
});
});

Resources