I am developing a RESTful API using NodeJS and Express.
I noticed that incoming requests sometimes lack of some expected variables, which cause the program to crash, saying it couldn't set the value of a variable, to an 'undefined' value - as no value arrived with the request.
Example:
The application is expecting variableY, but instead variableX is being sent:
formData: { variableX: 'valueX' }
The program is expecting to receive variableY, with the following code:
const checkVariables = Joi.validate({
variableY: req.body.variableY,
}, schema);
The application crashes with the following error:
TypeError: Cannot read property 'variableY' of undefined
I thought about a few ways to handle that, including declaration of variables upon application initiation and using them along, using try-catch.
Another way will be to use if-else, if-chaining, or case-switch, but as you understood of course I am looking for the cleanest way to achieve that.
Any ideas?
Thank you.
** EDIT **
Progressed and managed to achieve the result using the object only. Once trying to reach any of it's inner fields the error will be thrown anyway, example:
if(req.body.variableY == undefined){console.log('The expected variable is undefined');} //true
When the validation addresses a field inside the 'undefined' object:
if(req.body.variableY.dataId == undefined){console.log('The expected variable is undefined');} //crashes
The following error is being thrown again:
TypeError: Cannot read property 'variableX' of undefined
After doing some more digging around, found this Stackoverflow thread:
How to check if object property exists with a variable holding the property name?
Tried using hasOwnProperty, but the same kind of error is being thrown:
TypeError: Cannot read property 'hasOwnProperty' of undefined
Tried wrapping variable declaration using try-catch, still didn't work:
try{
var variableX = req.body.variableX
var variableXDataId = req.body.variableX.dataId
}
catch(e){
res.status(400).send('Wrong request error: Please check your request variables and try again');
}
As this is a really basic validation that should be addressed by most of the RESTful APIs (validating that you get the expected incoming variables inside the request, so the program won't crash by having errors it can't handle - what is the common solution for such problems (expected / unexpected request validation)?
Thank you.
You can take another approach, check req.body before you reach checkVariables:
let body = req.body;
// data - your req.body
// requiredKeys - is an array of strings , [ key1, key2 ... keyN] | string[]
const setKeys = ( data, requiredKeys )=>{
if( !typeof requiredKeys.length ){
requiredKeys = [];
}
if(requiredKeys.length) requiredKeys.forEach( k =>{
k = k.replace(/\+/g,'/');
let keysList = [];
if( /\/+/g.test(k)){
keysList = k.split('/');
}else{
keysList = [k];
}
let [firstKey, ...rest] = keysList;
if( typeof data[firstKey] === 'undefined' ){
data[firstKey] = {};
}
if( rest.length ){
data[firstKey] = setKeys(data[firstKey], [rest.join('/')] );
}
})
return data;
}
let checkedData= setKeys(body, ['variableT','variableP/noname/emptyObj','custom/object/does/not/exist/but/it/will/be/created/here']);
const checkVariables = Joi.validate(checkedData, schema);
UPDATE
Below you will find an working example on how things should work during a /(let's say /usersStatus/:id ) request:
const express = require('express')
const app = express()
const port = 3000
const setKeys = (data, requiredKeys) => {
if (!typeof requiredKeys.length) {
requiredKeys = [];
}
if (requiredKeys.length) requiredKeys.forEach(k => {
k = k.replace(/\+/g, '/');
let keysList = [];
if (/\/+/g.test(k)) {
keysList = k.split('/');
} else {
keysList = [k];
}
let [firstKey, ...rest] = keysList;
if (typeof data[firstKey] === 'undefined') {
data[firstKey] = {};
}
if (rest.length) {
data[firstKey] = setKeys(data[firstKey], [rest.join('/')]);
}
})
return data;
}
/**
* Mock some data
*/
const getUserData = (req, res, next) => {
if (typeof req.body === 'undefined') {
req.body = {};
}
req.body = {
variableY: {
someName: 23
},
variableZ: {
name: 3,
type: {
id: 5,
typeName: 'something',
tags: ['a', 'b', 'c']
}
}
};
console.log('Middleware 1 getUserData');
next();
}
/**
* 1. Setup our middleware for checking keys
* "requiredKeys" is an array of strings
*/
const middlewareSetKeys = (requiredKeys, wrappedMiddleware) => {
return (req, res, next) => {
console.log('Middleware 2 middlewareSetKeys');
if (typeof req.body === "undefined") {
console.log('Leaving Middleware 2 since we don\'t have req.body');
next();
}
/**
* Update "req.body" with keys that we want to have available
* in our next middleware
*/
req.body = setKeys(req.body, requiredKeys);
if (typeof wrappedMiddleware === 'function') {
return wrappedMiddleware.call(this, req, res, next);
} else {
next();
}
}
}
/**
* 2. Let's assume a "user status" situation
* 2.1. We need userInfo from database
* 2.2. Some info won't be retrieved, unless the user accesed some parts of the website to trigger some mechanisms that allows those fields to be exposed, therefore the lack of keys
* 2.3. But we know those keys/objects, and we still want to be present so our code won't crash.
*/
// lets call our getUserData
app.get(
'/', // this path is for some userInfo
getUserData, // this returns userInfo and appends it to `req.data`
middlewareSetKeys([
'userActivity/daily/jobs', // these won't exist in getUserData because the user is lazy and he didn't apply for any JOBS
'userStatus/active/two-weeks-ago', // these won't exist in getUserData because the user joined two days ago. BUT WE STILL NEED IT coz reazons.
]), // We set our desired-later-to-use keys
(req, res, next) => {
/**
* 3. Now our req.body will have our keys
* even if they didn't exist in the getUserData middleware
*/
console.log('Middleware 3 Your middleware');
console.log(req.body);
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify(req.body, null, 2))
})
app.listen(port, () => console.log(`Example app listening on port ${port}!`))
you can use express validator https://www.npmjs.com/package/express-validator
to validate incoming request.Then add this to your controller where a,b,c ,d are parameters you want to valaidate
const nonEmptyFields = ['a', 'b', 'c', 'd'];
nonEmptyFields.forEach(field => req.assert(field, `${field} cannot be blank`).notEmpty());
const errors = req.validationErrors();
if (errors) {
return res.status(400).send(errors);
}
for validating a field inside a field you can try doing this
typeof(req.body && req.body.name !== undefined)
A solution will be to set a default empty object to replace undefined at a parent level:
// checking for body.variableX.variableZ with object destructuring ES6
const {body = {}} = request;
const {variableX = {}, variableY} = body;
const {variableZ} = variableX.variableZ;
// or prior ES6
var body = request.body || {};
var variableX = body.variableX || {};
var variableY = variableX.variableY;
// or in a statement
var variableY = request.body && request.body.variableX ? request.body.variableX.variableY : undefined;
Based on that you can create your own function like getValue(request, 'body.variableX.variableY') to return null if any parent or the end value is undefined:
// asumes the value in the path is either object or undefined
function getValue(rootObj, path = '') {
const parts = key.split('.');
let value = rootObj || {};
let part;
while ((part = parts.shift()) && value !== null) {
value = value[part] || null;
}
return value;
};
Related
I am building my first web application with node v12.18.1 and express v4.17.1. Since the start of development, I have the same error on all routes: when i quickly click a link multiple times the server crashes with this error: screenshot of the error. It can be fixed in the front-end by disabling events on user input after a click, but I prefer to know what is wrong with my code.
Route of the index page :
let express = require('express');
let router = express.Router();
let controller_index = require("../controller/controller_index.js")
router.get('/', controller_index.get_index);
router.get('/rubrique/:category', controller_index.get_category);
module.exports = router;
Controller of the index page :
const Query = require("../lib/dbs.js");
const ObjectId = require('mongodb').ObjectId;
const utils = require('../lib/utils');
exports.get_index = async (req, res, next) => {
try {
let user = await Query.findOne("users", "_id", ObjectId(req.user_id));
let notification;
if (user) {
notification = await Query.findSortToArray("notifications", "for_user", ObjectId(user._id));
notification.count = notification.filter((notif => !notif.hasOwnProperty("readedAt"))).length;
}
if (!req.query.q) {
let page = req.query.page > 1 ? (req.query.page * 10) - 10 : req.query.page <= 0 ? undefined : 0;
let current_page = req.query.page ? Number(req.query.page) : 1;
let [countDocuments, docs] = await Promise.all([Query.countAll("articles"), Query.findAll(page)]);
let nb_pages = Math.ceil(countDocuments / 10);
if (!docs.length && current_page !== 1) {
next();
}
else {
docs = await documents_processing(docs, user, req);
res.render("../views/index.pug", { docs: docs, user: user, notification: notification, nb_pages: nb_pages, current_page: current_page })
};
}
else if (req.query.q) {
let page = req.query.page > 1 ? (req.query.page * 10) - 10 : req.query.page <= 0 ? undefined : 0;
let current_page = req.query.page ? Number(req.query.page) : 1;
let [countDocuments, docs] = await Promise.all([Query.countAllBySearch("articles", req.query.q), Query.findAllBySearch(req.query.q, page)]);
let nb_pages = Math.ceil(countDocuments / 10);
if (!docs.length && current_page !== 1) {
next();
}
else {
docs = await documents_processing(docs, user, req);
res.render("../views/index.pug", { docs: docs, search: req.query.q, user: user, notification: notification, nb_pages: nb_pages, current_page: current_page })
};
};
}
catch (err) {
console.error(err);
return next(err);
};
};
Example of static function of the query object :
const connect = require("../index.js")
module.exports = class Query {
static async findOne(collection, field, item) {
const result = await connect.client.db("blog_db").collection(collection).findOne({ [`${field}`]: item })
return result;
};
static async findOneAndUpdateOrInsertOnUser(collection, field, itemToSearch, updateItem) {
const result = await connect.client.db("blog_db").collection(collection).findOneAndUpdate({ [`${field}`]: itemToSearch }, { $set: updateItem }, { upsert: true, returnOriginal: false });
return result;
};
static async findSortToArray(collection, field, item) {
const results = await connect.client.db("blog_db").collection(collection).find({ [`${field}`]: item }).sort({ date: -1 }).toArray()
return results;
};
};
I'm fairly new to programming so any advice is welcome, thank you in advance!
----- EDIT -----
Kind of solution :
I have found people who have talked about this error on node v12 and newer, with a downgrade to v10 the issue was resolved without any clear explanation yet.
The error in the screenshot you've shared shows that the error is "Callback called multiple times", but I don't see anywhere obvious in the code you've shared where this is happening. As you're saying this bug only happens when multiple requests are made rapidly one after the other, it suggests there might be a global variable which is being shared between requests, which is something that should be avoided.
To debug the error you're seeing I would recommend commenting out all of the code in the get_index function and gradually uncommenting it in small chunks until you see the error happen again. You will probably want to do the same with the code that is called by the get_index controller function e.g. documents_processing, as the issue might possibly lie there.
Express only support callback-style, and you're using async function to handle the logic.
Basically, with async function, you call a function without waiting for it to resolve the logics. Hence, it creates too many callbacks in the event loop when that route has a large amount of concurrent coming requests.
function asyncWrapper(fn) {
return (req, res, next) => {
return Promise.resolve(fn(req))
.then((result) => res.send(result))
.catch((err) => next(err))
}
};
router.get('/', asyncWrapper(controller_index.get_index));
I'm trying to create a function that generates a discount voucher, and I'm getting an error as "TypeError: res.status is not a function at the controller."
here is my discount Controller :
function coupongenerator () {
var coupon = '';
var possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
for (var i = 0; i < 10; i++) {
coupon += possible.charAt(Math.floor(Math.random() * possible.length));
}
return coupon;
}
exports.createCoupon = () => {
let isExistDiscount = false;
do {
let myDiscountCode = coupongenerator()
let newDiscountCode = new Coupon({
code: myDiscountCode,
isPercent: true,
amount:30 ,
expireDate: '',
isActive: true
});
newDiscountCode.save(function (err,res) {
if (err) {
if (err.name === 'MongoError' && err.code === 11000) {
// Duplicate code detected
isExistDiscount = true;
}
}
res.status(200).send('yes');
});
} while (isExistDiscount);
}
The "res" object you are trying to access is actually the response coming after saving the Mongo document in callback function. res.status works with HTTP res object in express which generally will be a function parameter of a controller. Just check if your controller variable identifier and the callback identifier is same, which will shadow the controller variable. Hence the scope of res is the res in callback param which do not have a function called status in it. If that's the reason change one of the identifier names.
I have an app which sends a form schema with the data for page rendering.
The form schema comes from a require call to a configuration - this is in javascript object notation. Depending on the user's permission level, the function massageSchema then for example removes protected fields from the schema, prior to the schema being sent.
This all works. However, if I log out and log in as a different user, the schema relevant to the previous user is sent.
If I stop/start the node instance, the correct schema then gets sent.
So I've deduced that this appears to be a caching issue, but I have no clue as to how to address it.
The router code:
router.get('/:table/:id', authUser, resolveTableName, authMethodTable, function(req, res, next) {
getTableModule(req.params.table)
.then(mod => {
// Massage the schema
mod.formSchema = massageSchema(mod, req.session.user);
...
db.one( sql, [ req.params.table, res.locals.table.idAttribute, req.params.id ])
.then( row => {
res.render("record", {
data: row,
user: req.session.user,
table: req.params.table,
module: mod,
referer: req.get('Referer')
})
....
The massageSchema function:
module.exports = function(mod, user) {
var rv = {};
var orig = mod.formSchema ? mod.formSchema : mod.schema;
// Remove unallowed fields
for(var i in orig) {
if(orig[i].display) {
if(orig[i].display == 'admin' && (user.role == 'admin' || user.role == 'master')) {
rv[i] = orig[i];
} else if(orig[i].display == 'editor' &&
(user.role == 'editor' || user.role == 'admin' || user.role == 'master')) {
rv[i] = orig[i];
}
} else {
rv[i] = orig[i];
}
}
return rv;
};
Why is this happening? What to do?
I'm guessing mod is part of some module.exports?. In JavaScript, objects are always passed by reference, including module.exports mod.formSchema = massageSchema(mod, req.session.user) is actually modifying the module's exported object.
try let schema = massageSchema(mod, req.session.user) instead
We're working on a cloud function that allows us to keep our bigquery and firebase database in sync. The function triggers when a place is created/updated/deleted.
Based on the trigger action (create/update/delete) we add a property called big_query_active to signal if the object exists or not. Same goes for the date.
Our current problem is that the call to big query sometimes returns an error. So that would mean that the data is not in sync anymore. How can this be prevented?
'use strict';
// Default imports.
const functions = require('firebase-functions');
const bigQuery = require('#google-cloud/bigquery');
// If you want to change the nodes to listen to REMEMBER TO change the constants below.
// The 'id' field is AUTOMATICALLY added to the values, so you CANNOT add it.
const ROOT_NODE = 'places';
const VALUES = [
'country_id',
'category_id',
'name',
'active',
'archived'
];
// This function listens to the supplied root node, but on child added/removed/changed.
// When an object is inserted/deleted/updated the appropriate action will be taken.
exports.children = functions.database.ref(ROOT_NODE + '/{id}').onWrite(event => {
const query = bigQuery();
const dataset = query.dataset('stampwallet');
const table = dataset.table(ROOT_NODE);
if (!event.data.exists() && !event.data.previous.exists()) {
return;
}
const item = event.data.exists() ? event.data.val() : event.data.previous.val();
const data = {};
data['id'] = event.params.id;
for (let index = 0; index < VALUES.length; index++) {
const key = VALUES[index];
data[key] = item[key] !== undefined ? item[key] : null;
}
data['big_query_date'] = new Date().getTime() / 1000;
data['big_query_active'] = event.data.exists();
return table.insert(data).then(() => {
return true;
}).catch((error) => {
if (error.name === 'PartialFailureError') {
console.log('A PartialFailureError happened while uploading to BigQuery...');
} else {
console.log(JSON.stringify(error));
console.log('Random error happened while uploading to BigQuery...');
}
});
});
This is the error that we (sometimes) receive
{"code":"ECONNRESET","errno":"ECONNRESET","syscall":"read"}
How could it be prevented that the data goes out of sync? Or is there a way to retry so that it always succeeds?
hello i have a question regarding the foodme express example over github:
code:
var express = require('express');
var fs = require('fs');
var open = require('open');
var RestaurantRecord = require('./model').Restaurant;
var MemoryStorage = require('./storage').Memory;
var API_URL = '/api/restaurant';
var API_URL_ID = API_URL + '/:id';
var API_URL_ORDER = '/api/order';
var removeMenuItems = function(restaurant) {
var clone = {};
Object.getOwnPropertyNames(restaurant).forEach(function(key) {
if (key !== 'menuItems') {
clone[key] = restaurant[key];
}
});
return clone;
};
exports.start = function(PORT, STATIC_DIR, DATA_FILE, TEST_DIR) {
var app = express();
var storage = new MemoryStorage();
// log requests
app.use(express.logger('dev'));
// serve static files for demo client
app.use(express.static(STATIC_DIR));
// parse body into req.body
app.use(express.bodyParser());
// API
app.get(API_URL, function(req, res, next) {
res.send(200, storage.getAll().map(removeMenuItems));
});
i don't understand where is the api folder. it doesn't exist and i don't understand how information is going in and out from there. i can't find it.
can someone please explain this to me?
another question:
there is a resource for the restaurant
foodMeApp.factory('Restaurant', function($resource) {
return $resource('/api/restaurant/:id', {id: '#id'});
});
and in the restaurant controller there is a query:
var allRestaurants = Restaurant.query(filterAndSortRestaurants);
and the following lines:
$scope.$watch('filter', filterAndSortRestaurants, true);
function filterAndSortRestaurants() {
$scope.restaurants = [];
// filter
angular.forEach(allRestaurants, function(item, key) {
if (filter.price && filter.price !== item.price) {
return;
}
if (filter.rating && filter.rating !== item.rating) {
return;
}
if (filter.cuisine.length && filter.cuisine.indexOf(item.cuisine) === -1) {
return;
}
$scope.restaurants.push(item);
});
// sort
$scope.restaurants.sort(function(a, b) {
if (a[filter.sortBy] > b[filter.sortBy]) {
return filter.sortAsc ? 1 : -1;
}
if (a[filter.sortBy] < b[filter.sortBy]) {
return filter.sortAsc ? -1 : 1;
}
return 0;
});
};
the things that isn't clear to me is:
how is that we are giving the query just a function without even activating it.
as i understand we should have passed the query somthing like:
{id: $routeParams.restaurantId}
but we only passed a reference to a function. that doesn't make any sense.
could someone elaborate on this?
thanks again.
var API_URL = '/api/restaurant';
var API_URL_ID = API_URL + '/:id';
var API_URL_ORDER = '/api/order';
These lines are just defining string constants that are plugged into Express further down. They're not a folder.
app.get(API_URL, function(req, res, next) {
res.send(200, storage.getAll().map(removeMenuItems));
});
So this function call to app.get(API_URL... is telling Express "Look out for GET requests that are pointed at the URL (your app's domain)/api/restaurant, and execute this function to handle such a request."
"api" is not a folder.
Every requests will pass through the app.get method.
This method will respond to the routes /api/restaurant as defined in the API_URL variable.