Concurrent request handling in Nodejs - node.js

I have a issue of concurrent request, which modifies db.
What am I doing is.
One request fetch data for user-1 , then calculate data for user-1 modified a field-1 in record, and save.
next request fetch data for user-1 , then calculate data for user-1 modified a field-1 in record, and save.
Both request operates simultaneously. so last request update the wrong data.
function calculate() {
var needUpdate = false;
user = new UserLib(user_id);
var old_config = user.config;
if (old_config[req.id]) {
old_config[req.id].value = 0;
needUpdate = true;
}
if (req.delete == void(0) || req.delete == false) {
delete req.delete;
old_config[req.id].value = old_config[req.id].value + 1;
needUpdate = true;
}
if (needUpdate) {
return user.save();
}
return true;
}
We are getting both requests at the same time.
var express = require('express');
var app = express();
app.get('/update', function(req, res) {
res.writeHead(200, {
'Content-Type': 'text/html'
});
calculate(req);
function calculate(req) {
var needUpdate = false;
user = new UserLib(user_id);
var old_config = user.config;
if (old_config[req.id]) {
old_config[req.id].value = 0;
needUpdate = true;
}
if (req.delete == void(0) || req.delete == false) {
delete req.delete;
old_config[req.id].value = old_config[req.id].value + 1;
needUpdate = true;
}
if (needUpdate) {
user.save();
}
}
res.end('Done');
});
first reuest with following parameter {
user_id: 1,
id: 1,
value: 5,
delete: false
}
Anothere request with follwing parmter {
user_id: 1,
id: 1,
delete: true
}

If you want to operate on each request simultaneously, I'd suggest to use Bluebird.map where you can handle each request as you want, with concurrency, and a final result.
For example:
let users = ['foo', 'bar']; //fetching users you want
Bluebird.map(users, (user) => {
return user.calculate()
.then((res) => res.shouldUpdate ? user.save() : Promise.resolve())
}, {concurrency: 2})
.then((results) => {
//results is an array with both resolved promises from below
})
You may also be interested in Bluebird.join where you could calculate, and join the resulting data for more than one promise.
Second example where you fetch same user twice in the same promise:
//both are promises
Bluebird.all([fetchUser1, fetchUser2])
.spread(function(user1, user2) {
//check if should update
return user1.delete !== user2.delete ? user.delete() : null
})
.then(() => {})
Bluebird.spread documentation

Related

dynammic way to use the same code for employeeID and employeeName

I have this project in order to return "the employee's screenshots", and I set up the api to return the screenshots:
http://localhost:3000/employee/screenshot/14ll0a54kb9kkvh8?page=5
employeeId="14ll0a54kb9kkvh8".............................................,
and with this api i pass the employee Id and the postman return screenshots for this employee , and the code is at the bottom, but in any case I want to pass the "employee’s name" not the "employee's Id" to the api, so how can I add this thing to the code, "in a dynamic way", meaning without returning to write all this same code ?
I need to have two api with the same code ,the first api with id and the second with name:
http://localhost:3000/employee/screenshot/14ll0a54kb9kkvh8?page=5
http://localhost:3000/employee/screenshot/George?page=5
screenshotServices.js:
async getAll(employeeId, pageNumber, pageSize) {
// Grab images from db according to pagination data and uid
const dbImages = await ScreenshotModel
.findAndCountAll({
where: {
employeeId: employeeId
},
limit: pageSize,
offset: (pageNumber - 1) * pageSize
})
.then(screenshots => {
// console.log(dbRes);
const imagesData = [];
for (let i = 0; i < screenshots.rows.length; i++) {
imagesData.push(screenshots.rows[i]['dataValues']);
}
return imagesData;
})
.catch(dbError => {
console.log(dbError);
return null;
});
if (dbImages === null) return dbImages;
// Database returns images paths
// Now we need to get the actual images from files
// getting images paths from db response
const imagePaths = [];
for (let i = 0; i < dbImages.length; i++) {
imagePaths.push(dbImages[i]['imagePath']);
}
const directoryPath = rootDirectory + `/images/screenshots/${employeeId}`;
// Grabbing images from files
return await ScreenshotModel
.findAllInDirectoryWithSpecifiedImagePaths(directoryPath, imagePaths)
.then(readFromDirectoryResponse => {
return readFromDirectoryResponse;
})
.catch(readFromDirectoryError => {
console.log(readFromDirectoryError);
return null;
});
}
module.exports = ScreenshotService;
screenshotController.js:
const _getAll = async (req, res) => {
// Grabbing data
const employeeId = req.params['id'];
// Pagination data
const pageNumber = +req.query['page'];
const pageSize = 3;
// console.log(uid);
// console.log(pageNumber);
const screenshots = await ScreenshotService.Instance.getAll(employeeId, pageNumber,
pageSize);
if(screenshots === null)
return res.status(500).json({message: 'failed', screenshots: screenshots});
return res.status(200).json({message: 'succeeded', screenshots: screenshots});
};
module.exports = {
getAll: _getAll
};
the best solution should be separate the entry point (the endpoint) and receive different param values, then each endpoint goes to a common function.
other solution is to write a regex then you can identify if the incoming value is an id or a name.
EDIT:
screenshotController.js:
const criteriaRegex = new RegExp('^(?=.*[0-9])(?=.*[a-zA-Z])([a-zA-Z0-9]+)$'); // define this in a global place
const _getAll = async (req, res) => {
const filter = {};
// Grabbing data
const critieria = req.params['id'];
if (criteriaRegex.test(critieria)){ // if true is because the ID is alphanumeric, thus, we assume this is an ID
filter = {employeeId: critieria}
} else {
filter = {employeeName: critieria}
}
// Pagination data
const pageNumber = +req.query['page'];
const pageSize = 3;
// console.log(uid);
// console.log(pageNumber);
const screenshots = await ScreenshotService.Instance.getAll(filter, pageNumber,
pageSize);
if(screenshots === null)
return res.status(500).json({message: 'failed', screenshots: screenshots});
return res.status(200).json({message: 'succeeded', screenshots: screenshots});
};
module.exports = {
getAll: _getAll
};
screenshotServices.js:
async getAll(filter, pageNumber, pageSize) {
// Grab images from db according to pagination data and uid
const dbImages = await ScreenshotModel
.findAndCountAll({
where: {
...filter
},
limit: pageSize,
offset: (pageNumber - 1) * pageSize
})
.then(screenshots => {
// console.log(dbRes);
const imagesData = [];
for (let i = 0; i < screenshots.rows.length; i++) {
imagesData.push(screenshots.rows[i]['dataValues']);
}
return imagesData;
})
.catch(dbError => {
console.log(dbError);
return null;
});
if (dbImages === null) return dbImages;
// Database returns images paths
// Now we need to get the actual images from files
// getting images paths from db response
const imagePaths = [];
for (let i = 0; i < dbImages.length; i++) {
imagePaths.push(dbImages[i]['imagePath']);
}
const directoryPath = rootDirectory + `/images/screenshots/${employeeId}`;
// Grabbing images from files
return await ScreenshotModel
.findAllInDirectoryWithSpecifiedImagePaths(directoryPath, imagePaths)
.then(readFromDirectoryResponse => {
return readFromDirectoryResponse;
})
.catch(readFromDirectoryError => {
console.log(readFromDirectoryError);
return null;
});
}
module.exports = ScreenshotService;

NodeJS Error in if else case - Can't set headers after they are sent

I know, this is something old. There are many questions regarding to this. But none of them didn't guide me or didn't gave me the actual concept.
My case is:
if something
render something
else
render somethingAnother
Why is this logic generates this error enter image description here
After 1st execution, I'm not able to continue this process (I could continue for a while, but after some time error will come), by pressing back button of browser and then returning back to the home page. Everytime I should restart my server using node command. Why headers won't reset if I press back button of browser, how to do some header reset or something which will correct my logic.
const cv = require('opencv4nodejs');
var async = require('async');
var OrderID;
var OrderNo;
var compare = 0;
var CompanyName;
var notSimilar = 0;
const download = require('image-downloader')
const distanceThreshold = 30;
var url;
var FolderPath;
var isSimilar = 0;
var j = 0;
var image1;
var dbPath;
var filterCount = 0;
var image2;
var dbImgCount = 0;
var express = require('express');
var request = require('request');
var app = express();
app.set('view engine', 'pug')
var multer = require('multer');
var storage = multer.diskStorage({
destination: function (req, file, callback) {
callback(null, './');
},
filename: function (req, file, callback) {
callback(null, file.fieldname + '-' + Date.now());
}
});
var upload = multer({ storage : storage}).single('userPhoto');
const sql = require("mssql");
var config = {
user: '***',
password: '****',
server: '192.168.5.100\\SQLEXPRESS',
database: 'Test_MatdesignDB1',
connectionTimeout: 300000,
requestTimeout: 300000,
pool: {
idleTimeoutMillis: 300000,
max: 100
}
};
sql.connect(config).then(pool => {
return pool.request()
.query('select count(*) from OrderImageUpload; select FolderPath from OrderImageUpload;')
}).then(result => {
var a = JSON.stringify(result.recordset[0]);
dbImgCount = a.slice(4,6);
FolderPath = result.recordsets[1];
sql.close();
}).catch(err => {
console.log(err);
sql.close();
})
app.get('/',function(req,res){
res.sendFile(__dirname + "/index.html");
});
app.post('/api/photo',function(req,res){
compare = 1;
upload(req,res,function(err) {
if(err) {
console.log(err);
res.send("File uploading error");
}
else{
// console.log("Success");
image1 = req.file.filename;
var matchFeatures = ({ url, img1, img2, detector, matchFunc }) => {
// detect keypoints
const keyPoints1 = detector.detect(img1);
const keyPoints2 = detector.detect(img2);
// compute feature descriptors
const descriptors1 = detector.compute(img1, keyPoints1);
const descriptors2 = detector.compute(img2, keyPoints2);
// match the feature descriptors
const matches = matchFunc(descriptors1, descriptors2);
// only keep good matches
const bestN = 40;
const bestMatches = matches.sort(
(match1, match2) => (match1.distance - match2.distance)
).slice(0, bestN);
//console.log(bestMatches);
for(var i=0; i<bestN; i++){
if((bestMatches[i].distance) <= (distanceThreshold)){
filterCount++;
}
}
if(filterCount >= (bestN/4))
isSimilar = 1;
if(isSimilar){
notSimilar = 0;
filterCount = 0;
isSimilar = 0;
console.log("Similar images\n");
dbPath = url;
sql.close();
(async function() {
try {
let pool = await sql.connect(config)
let result1 = await pool.request()
.query("select OrderID from Test_MatdesignDB1.dbo.OrderImageUpload where FolderPath = '"+dbPath+"';")
OrderID = result1.recordset[0].OrderID;
let result2 = await pool.request()
.query('select OrderNo , CompanyName from Test_MatdesignDB1.dbo.[Order] where OrderID = '+OrderID);
OrderNo = result2.recordset[0].OrderNo;
CompanyName = result2.recordset[0].CompanyName;
res.render('similar', { title: 'Similar', CompanyName: CompanyName, OrderID: OrderID, OrderNo: OrderNo, img_path_var : dbPath }) //Render number 1 in 'if' case
} catch (err) {
console.log(err);
sql.close();
}
sql.close();
})()
sql.on('error', err => {
console.log(err);
})
}
else{
isSimilar = 0;
filterCount = 0;
notSimilar++;
if(notSimilar >= (dbImgCount ))
{
notSimilar = 0;
res.render('notSimilar', { title: 'Not Similar', message: 'No Similar Images' }) //Render number 2 in 'else' case
}
console.log("Not similar\n");
}
return cv.drawMatches(
img1,
img2,
keyPoints1,
keyPoints2,
bestMatches
);
};
for (j=0; j<dbImgCount; j++) {
(function(j) {
async.waterfall([
async function downloadIMG(done) {
try {
var options = {
url: FolderPath[j].FolderPath,
dest: '/home/ubuntu/imgCompare/DBimages/'
}
const { filename, image } = await download.image(options);
return [filename, options.url];
} catch (e) {
console.error(e)
}
},
async function featureMatching([a, MatchURL], done){
const img1 = cv.imread(image1);
url = MatchURL;;
const img = a.slice(33);
const img2 = cv.imread('./DBimages/'+img);
const orbMatchesImg = matchFeatures({
url,
img1,
img2,
detector: new cv.ORBDetector(),
matchFunc: cv.matchBruteForceHamming
});
done(null);
}
],
function (err) {});
})(j);
}
}
});
});
app.listen(5000,function(){
console.log("Working on port 5000");
});
You need to add return before rendering a view. It's happening because the view rendering is happening more than 1 time there must be a condition in your code which is letting views to render multiple times. Add this return statement:
return res.render();
You're getting this error because you're calling matchFeatures() multiple times within a for loop.
app.post('/api/photo', function (req, res) {
var matchFeatures = ({url, img1, img2, detector, matchFunc}) => {
if (isSimilar) {
res.render('similar', {
title: 'Similar',
...
}) //Render number 1 in 'if' case
} else {
res.render('notSimilar', {
title: 'Not Similar',
message: 'No Similar Images'
}) //Render number 2 in 'else' case
}
};
for (j = 0; j < dbImgCount; j++) {
async function featureMatching() {
const orbMatchesImg = matchFeatures({ // since you're calling it multiple times here
url, // matchFeatures() will attempt to send
img1, // responses multiple times
img2,
detector: new cv.ORBDetector(),
matchFunc: cv.matchBruteForceHamming
});
}
}
});
To fix this, You need to consolidate all these responses and send to client only once.
I figured out the error. I didn't reset the variable notSimilar at the entry point.
Done resetting of notSimilar as below, no error! Thanks Everyone.
app.post('/api/photo',function(req,res){
notSimilar = 0;

The ultimate way to prevent duplication in Parse Server once and for all

One of the biggest issue we face now with parse-server is duplication. Although we have implemented a Parse cloud code to prevent such event through beforeSave and afterSave methods at the same time added external middleware to check for existing object before saving still we face duplication over and over specially on concurrent operations.
Here is our code to prevent duplication for a specific class:
Parse.Cloud.beforeSave("Category", function(request, response) {
var newCategory = request.object;
var name = newCategory.get("name");
var query = new Parse.Query("Category");
query.equalTo("name", name);
query.first({
success: function(results) {
if(results) {
if (!request.object.isNew()) { // allow updates
response.success();
} else {
response.error({errorCode:400,errorMsg:"Category already exist"});
}
} else {
response.success();
}
},
error: function(error) {
response.success();
}
});
});
Parse.Cloud.afterSave("Category", function(request) {
var query = new Parse.Query("Category");
query.equalTo("name", request.object.get("name"));
query.ascending("createdAt");
query.find({
success:function(results) {
if (results && results.length > 1) {
for(var i = (results.length - 1); i > 0 ; i--) {
results[i].destroy();
}
}
else {
// No duplicates
}
},
error:function(error) {
}
});
});
This code above is able to prevent some duplicate but most still goes through, example:
What is the "ultimate way" to prevent duplication with Parse server?
You can always create a unique index in mongodb for the field that should be unique in your document.
This way any save that conflicts with that index, will be aborted
Maybe you should write something with Promises like :
Parse.Cloud.beforeSave("Category", function (request, response) {
return new Promise((resolve, reject) => {
var query = new Parse.Query("Category");
query.equalTo("name", "Dummy");
return query.first().then(function (results) {
resolve(); // or reject()
});
})
});
Parse.Cloud.beforeSave("Category", async (request) => {
(...)
await results = query.first();
// then your logic here
response.success();
response.error({ errorCode: 400, errorMsg: "Category already exist" })
})
Here is my Solution:
Parse.Cloud.beforeSave( 'ClassName', async ( request ) => {
const columnName = 'columnName'
const className = 'ClassName'
if( request.object.isNew() ) {
var newCategory = request.object
var name = newCategory.get( columnName )
var query = new Parse.Query( className )
query.equalTo( columnName, name )
const results = await query.count()
if( results === 0 ) {
// no response.success needed
// https://github.com/parse-community/parse-server/blob/alpha/3.0.0.md
} else {
throw 'Is not unique';
}
}
} )

Unhandled promise rejection firestore

I don't quite have the promises down I make. Anyway, I am reading a list of rows in a mysql table, then upon writing into Firestore I want to update a count of a "related" record. Basically aggregate counts.
Here is the start:
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
var data = JSON.parse(body);
//console.log(data);
var allObjects = data.resource;
var obj = {};
for (j = 0; j < allObjects.length; j++) {
var records = [];
var record = allObjects[j];
//FireStore
var db = admin.firestore();
var docRef = db.collection(toCollection).doc(record.nfCode);
var rec = docRef.set(record);
addCounts(docRef);
}
res.render('index', {title: 'Load Table'});
offset = offset + limit
} else {
return res.send(body);//
//res.render('index', { title: 'Error' });
}
});
Then here is addCount:
function addCounts(docRef) {
// In a transaction, update the aggregate totals
var db = admin.firestore();
return db.runTransaction(transaction => {
transaction.get(docRef).then(res => {
var brandRef = db.collection('brands').doc(res.data().brandNFCode);
var transaction = db.runTransaction(t => {
return t.get(brandRef)
.then(doc => {
if (res.data().glutenFreeYN == "Y") {
var glutenFreeCount = doc.data().glutenFreeCount + 1;
var setWithOptions = transaction.set(brand, {
glutenFreeProductCount: glutenFreeProductCount
}, { merge: true });
return setWithOptions;
}
});
})
.then(result => {
console.log('Transaction success', result);
// return nil
})
.catch(err => {
// console.log('Transaction failure:', err);
return nil
});
})
})
}
Is there a better way to do this? And where is my error coming from?
Auth error:Error: socket hang up
(node:90050) UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 2): Error: Getting metadata from plugin failed with error: socket hang up
There are a few things wrong with your addCounts function that may be causing this:
function addCounts(docRef) {
// In a transaction, update the aggregate totals
var db = admin.firestore();
return db.runTransaction(transaction => {
return transaction.get(docRef).then(res => {
if(res.data().glutenFreeYN == "Y"){
var glutenFreeProductCount = res.data().glutenFreeProductCount + 1;
}
if(res.data().vegetarianYN == "Y"){
var vegetarianProductCount = res.data().vegetarianProductCount + 1;
}
if(res.data().dairyFreeYN == "Y"){
var dairyFreeProductCount = res.data().dairyFreeProductCount + 1;
}
if(res.data().organicYN == "Y"){
var organicProductCount = res.data().organicProductCount + 1;
}
// LOOK HERE!!! You had brand.set()
// this version uses the transaction
var setWithOptions = transaction.set(brand, {
glutenFreeProductCount: glutenFreeProductCount,
organicProductCount: organicProductCount,
vegetarianProductCount: vegetarianProductCount,
dairyFreeProductCount: dairyFreeProductCount
}, { merge: true });
// LOOK HERE!! Make sure to return the
// set operation
return setWithOptions;
});
// I removed the .catch() here as it will obscure
// errors. You can catch the result of addCounts.
})
};
Was running into this issue and spent over an hour trying to figure it out. Make sure your system time is synced properly. My time was correct, but my time zone wasn't so the system UTC time was out of sync.
To correct on Windows 10, go to Settings -> Time & Language -> Date and Time -> Sync Now

How to send a response only after a query has been executed in loopback

I have a remote method in loopback like:
Alerts.getAlertDetails = function (alertId, options, cb) {
var response = {};
var userId = options.accessToken.userId;
Alerts.app.models.MobileUserAlertRelation.find({where: {userId: userId, alertId: alertId, isDeleted: -1}, include: {relation: 'alerts', scope: {include: ['alertTypes'], where: {status: 1}}}}, function (err, alertRel) {
if (alertRel.length > 0 && alertRel[0].alerts()) {
response.code = 200;
response.status = "success";
response.data = {};
if (alertRel[0].alertId) {
response.data.alertId = alertRel[0].alertId;
}
if (alertRel[0].readStatus) {
response.data.readStatus = alertRel[0].readStatus;
}
if (alertRel[0].receivedOn) {
response.data.alertReceivedOn = alertRel[0].receivedOn;
}
var alertData = alertRel[0].alerts();
if (alertData.title) {
response.data.alertTitle = alertData.title;
}
if (alertData.message) {
response.data.alertShortMessage = alertData.message;
}
if (alertData.extraMessage) {
response.data.alertMessage = alertData.extraMessage;
}
if (alertData.priority) {
response.data.alertPriority = alertData.priority;
}
if (alertData.validUntil) {
response.data.alertExpiresOn = alertData.validUntil;
}
if (alertData.images && alertData.images.length > 0) {
response.data.alertImages = [];
for (var image in alertData.images) {
if (alertData.images.hasOwnProperty(image)) {
response.data.alertImages.push(constants.ALERT_IMAGE_URL + '/' + alertData.images[image]);
}
}
}
if (alertData.alertTypes() && alertData.alertTypes().alertTypeName) {
response.data.alertType = alertData.alertTypes().alertTypeName;
}
if (alertData.alertLocations && alertData.alertLocations > 0) {
response.data.alertLocations = [];
response.data.policeDepartments = [];
response.data.hospitals = [];
response.data.fireDepartments = [];
var locations = alertData.alertLocations;
for (var locKey in locations) {
if (locations.hasOwnProperty(locKey)) {
if (locations[locKey].data) {
response.data.alertLocations.push(locations[locKey].data);
console.log(locations[locKey].data);
if (locations[locKey].data.type) {
var locationType = locations[locKey].data.type;
if (locationType === "Polygon") {
var coordinates = locations[locKey].data.coordinates[0];
var polygonCenter = getPolygonCenter(coordinates);
console.log(polygonCenter);
}
}
}
}
}
}
cb(null, response);
} else {
response.code = 404;
response.status = 'error';
response.message = 'Alert not found.';
cb(null, response);
}
})
};
But when I call this method through api, response is received without data added from the complex code part. I know that callback will be called asynchronously here and so that cb(response) will be called before the complex code is executed completely. How can i send response only after the complex part is completed and data is correctly added to response from that data. I cannot move cb(response) inside the complex part as data is being pushed in for loop.
I have heard of promises, can it be used here, if so, how could it be done?
Someone please help!!
The problem is because of fetching relation in if.
The relation method is an async.
Alerts.getAlertDetails = function (alertId, options, cb) {
var response = {};
var userId = options.accessToken.userId;
Alerts.app.models.MobileUserAlertRelation.find({where: {userId: userId, alertId: alertId, isDeleted: -1}, include: {relation: 'alerts', scope: {include: ['alertTypes'], where: {status: 1}}}}, function (err, alertRel) {
if(alertRel.length < 1){
return handleError();
}
alertRel[0].alerts(handleResponse);
function handleResponse(err, alertRelAlert){
if(err) return handleError();
if (alertRelAlert) {
//all that code in question if if section
}else {
return handleError();
}
}
function handleError(){
response.code = 404;
response.status = 'error';
response.message = 'Alert not found.';
cb(null, response);
}
});
}

Resources