Is it possible to define a unique request Id that is included in each log statement without handing the logger to each method/function call?
Technologies in use: NodeJS, Express, Winston
Edited
Finally, I have created a library that makes all the work.
https://github.com/davicente/express-logger-unique-req-id
It is a wrapper of Winston library, so you can use it the same way.
Let me know if it helps you
We had this same problem in several projects, and I couldn't finde any complete solution for this question. We are using same technologies (Node.js, Express.js and Winston for logs)
I found a solution to this using a couple of libraries and wrapping Winston library:
- node-uuid for creating unique identificators for each request
- continuation-local-storage for sharing this IDs among different modules without sending req object in all the calls.
First I need to create and set the unique identificator with each request. I do it in the middleware:
var uuid = require('node-uuid');
var createNamespace = require('continuation-local-storage').createNamespace;
var myRequest = createNamespace('my request');
// Run the context for each request. Assign a unique identifier to each request
app.use(function(req, res, next) {
myRequest.run(function() {
myRequest.set('reqId', uuid.v1());
next();
});
});
After that I had to wrap Winston library, recovering the id from the context and adding to the message of the log:
var winston = require('winston');
var getNamespace = require('continuation-local-storage').getNamespace;
// Wrap Winston logger to print reqId in each log
var formatMessage = function(message) {
var myRequest = getNamespace('my request');
message = myRequest && myRequest.get('reqId') ? message + " reqId: " + myRequest.get('reqId') : message;
return message;
};
var logger = {
log: function(level, message) {
winstonLogger.log(level, formatMessage(message));
},
error: function(message) {
winstonLogger.error(formatMessage(message));
},
warn: function(message) {
winstonLogger.warn(formatMessage(message));
},
verbose: function(message) {
winstonLogger.verbose(formatMessage(message));
},
info: function(message) {
winstonLogger.info(formatMessage(message));
},
debug: function(message) {
winstonLogger.debug(formatMessage(message));
},
silly: function(message) {
winstonLogger.silly(formatMessage(message));
}
};
module.exports = logger;
I think it was a little bit complex, so I decided to write it down in a post. You can get more information from there: Express.js: Logging info with global unique request ID – Node.js
I hope this helps with your problem.
This answer has a problem: the counter goes back to 0 every time the node process is restarted. Turns out there is fairly simple to work around. You simply add an express middleware that tags each request called with a UUID using the uuid package.
For uuid Pre-2.0
const uuid = require('uuid');
const app = express();
app.use(function (req, next) {
req.id = uuid.v4();
next();
});
For uuid 3.0+
const uuidv4 = require('uuid/v4');
const app = express();
app.use(function (req, next) {
req.id = uuidv4();
next();
});
At the very beginning of your request handling add something like the following (or put it in its own file):
var makeID = (function() {
var index = 0;
return function() {
return index++;
}
})();
app.use(function(req, res, next) {
req.id = makeID()
next()
})
This will give every request a unique (sequential) id. Do not allow people to identify themselves with it, only use it internally!
When logging in Winston, you can enter metadata, to be displayed after the message (in the form ${name}=${value}), which looks like this
app.use(function(req, res) {
winston.log('info', 'Test Log Message', { id: req.id });
res.end("Done.")
});
Hope this is helpful.
Related
I'm getting my callback from my dao and into my service and in my service i have iterated the list of object to be sent to my dao.
My service code
var dao = require('./dao');
var async = require('async');
exports.addUser = function(obj,callback) {
async.forEachOf(obj,function(value,key,callback){
dao.addUser(value,function(data){
callback(data);
})
})
}
this callback is not going to my control layer
Control layer
var express = require('express');
var app = express();
var service = require('./service');
app.use(express.urlencoded({ extended: false }));
app.use(express.json());
app.post('/addPerson',function(req,res){
var obj = req.body;
console.log(obj);
console.log("......",obj.name);
console.log("......",obj.age);
service.addUser(obj,function(data) {
console.log("---->",data);
res.json(data);
})
})
var server = app.listen(8080,function(){});
i need to send the data back to the browser
There are two problems with your existing code
1. By referring callback you are calling callback of the asyn.forEachOf
2. Trying to call ur original callback (which intern does res.json(data)). Here you are trying to send multiple response which is not possible.
Try using async.mapValues instead
exports.addUser = function(obj,callback) {
async.mapValues(obj,function(value,key,cb){
dao.addUser(value,function(data){
cb(null, data);
})
}, function(err, result) {
// result is now a map of results for each key
callback(result);
});
}
Is it possible to trace Node.js applications the same way it's possible to trace applications that use threads by assigning a request id and using it in log output? There are third-party tools that can do it (NewRelic, Rollbar), but how do they work? How does one implement a similar functionality? I can do something like this
function foo(x, logger) {
logger.info("Running foo")
}
function bar(a, b, logger) {
logger.info(`Running bar with ${a} and ${b}`)
}
app.get('/', (req, res) => {
const requestId = uuid()
const logger = {
info: (msg) => console.log(`${requestId} ${msg}`)
}
foo(1, logger)
bar(2, 3, logger)
}
But that requires explicitly passing logger object. What I want to do is to something like this in downstream functions
function foo(x) {
const logger = require('logger')
logger.info('Running foo') // => "[ce089d84-3679-42ae-b05c-2d0a071062f0] Running foo"
}
You can do it with domain and express for example. You can test this script, just save it in a file run npm i express uniqid then node index.js and open browser at localhost:8080/test, just reload it and you will see the id changing.
const app = require('express')();
const uniqid = require('uniqid');
app.use((req, res, next) => {
const nDomain = require('domain').create();
nDomain.id = uniqid();
nDomain.add(req);
nDomain.add(res);
nDomain.run(next);
nDomain.on('error', next);
});
app.get('/test', function(req, res) {
console.log(process.domain.id, 'id');
});
const server = app.listen(8080);
This way, just create a logger in a package which uses process.domain.id when logging. We use something similar at work to track which request triggered a log.
Beware, domain is pending deprecation. Please read the doc https://nodejs.org/api/domain.html
I've posted an approximation of my node web application below. The original problem I had is that I want to be able to let the client know on post to createentity whether the insert was successful and the id of the inserted entity. However, connection.query having a callback rather than running synchronously, I can't use the entityservice how I'd expect in another language, ie simply returning the result synchronously. There are several solutions, and I'm curious which is the best/common practice re node.js or if there is another I'm not thinking of.
Passing res down to the service, and responding within a callback; seems poor practice
Similarly, passing functions to execute after success/failure to the service; also seems poor practice
Returning a promise from the service and setting res based on resolution/failure; seems like services shouldn't return promises but I'm new to node
Some better method using appropriate features of node.js of which I'm unaware
trying to change the service such that it runs synchronously and just returning the result Other questions/answers have made me leery that this is possible or wise
structure the application some other way
something else?
//app.js
var express = require('express');
var bodyParser = require('body-parser')
var path = require('path');
var EntityService = require('./entityService.js');
var app = express();
var urlencodedParser = bodyParser.urlencoded({ extended: true })
app.post('/createentity', urlencodedParser, function(req, res){
EntityService.createEntity(req.body.name);
res.status(200).json(null);
});
app.listen(3000);
//entityService.js
var mysql = require('mysql');
EntityService = function(){
var connection = mysql.createConnection({
host : CONNECTION_IP,
user : 'root',
password : 'password',
database : 'entitydb'
});
connection.connect();
this.createEntity = function(name){
var record = {name: 'name'};
var query = connection.query('INSERT INTO entity set ?', record, function(error, results, fields ){
//want to return the results.insertId from the service
});
}
}
module.exports = new EntityService();
The correct approach here is option 3 - have your service return a Promise
this.createEntity = name => new Promise((resolve, reject) => {
const query = connection.query('...', { name }, (err, results) => {
if (err) return reject(err);
return resolve(results.map(r => r.insertId));
});
})
If you're on the latest version of Node, I'd go with the asynch/await syntax.
Your service returns a promise,then your calling code can do:
app.post('/createentity', urlencodedParser, async function(req, res){
const entity = await EntityService.createEntity(req.body.name);
res.status(200).json(entity);
});
I am building a cross system admin app, which will be used as an admin tool for multiple backend systems. The app is built on top of Mean.js.
I have setup a /proxy route using "express-http-proxy" to send all sub-routes to their respective backend system endpoints. However, I need to have each request authenticated within my admin app and then decorated with the targeted backendSystem credentials before the "express-http-proxy" can continue. Here's an example of my /proxy route...
app.use('/proxy', users.requiresLogin, expressHttpProxy(config.backendSystem.host, {
forwardPath: function (req) {
return '/1.0' + require('url').parse(req.url).path;
},
decorateRequest: function (req) {
req.headers['content-type'] = 'application/json';
req.headers['backend-system-id'] = config.backendSystem.id;
req.headers['backend-system-key'] = config.backendSystem.key;
return req;
}
}));
NOTE:
Currently the backendSystem credentials are stored based on the environment my admin app is ran in. However, in the future the backendSystem credentials will be specified by the user, and this /proxy route will differently than what is currently shown.
THE ISSUE:
Proxy routes that require data within the request body don't work.
e.g. POST /comments {"user": user_id, "text": "rabble rabble rabble"}
WHAT I'VE FOUND:
bodyParser.json() and "express-https-proxy" don't play nice. I've confirmed this by removing bodyParser.json() from express.js.
However, this isn't a full solution since almost all of my other routes need bodyParser.json, e.g. /auth/signin.
Does anyone have a clean way that I can make a route exception for my /proxy route so that bodyParser.json won't be called for it?
As far as I understand, the root of problem is so:
if you were reading a POST request by pure node, you should be using a code like this
if (req.method == 'POST') {
console.log("POST");
var body = '';
req.on('data', function (data) {
body += data;
console.log("Partial body: " + body);
});
req.on('end', function () {
console.log("Body: " + body);
});
res.writeHead(200, {'Content-Type': 'text/html'});
res.end('post received');
}
in other words, you need to use the req.on('data') & req.on('end') events.
but the problem is,that you can use this code only once. after the 'end' is called, the request is consumed.
so then you use bodyParser , it consumes the request, and the proxy have nothing to do with it.
actually, in my opinion, the proxy wait for the 'data' event to appear , but it will newer happen, so the code halts.
The solution:
you need to 're-enable' the events. I used this code and it works for me.
var express = require('express');
var bodyParser = require('body-parser');
var http = require('http');
//call for proxy package
var devRest = require('dev-rest-proxy');
//init express (as default)
var users = require('./routes/users');
var app = express();
app.use(bodyParser.json());
//set the proxy listening port
app.set('port', 8080);
//process the POST request
app.post('/users/*', function(req, res) {
//just print the body. do some logic with it
console.log("req.body: ",req.body);
//remove listeners set by bodyParser
req.removeAllListeners('data');
req.removeAllListeners('end');
//add new listeners for the proxy to use
process.nextTick(function () {
if(req.body) {
req.emit('data', JSON.stringify(req.body));
}
req.emit('end');
});
//forward the request to another server
devRest.proxy(req,res, 'localhost', 3000);
});
//start the proxy server
http.createServer(app).listen(app.get('port'), function(){
console.log("Express server listening on port " + app.get('port'));
});
module.exports = app;
the solution found on schumacher-m post (github of nodejitsu)
I was able to resolve my issue by adding a regex that excluded my /proxy route to where bodyParser.json was being added within express.js. I found that from this answer
While this approach doesn't scale well, it solved my immediate issue.
I get it works by converting the data into query string using 3rd party query-string as follows:
proxyReqBodyDecorator: function(bodyContent, srcReq) {
return (queryString.stringify(bodyContent));
}
Have tried JSON.stringify but not working, need the data in the following format
array_field=val1&array_field=val2&array_field=val3......
To modify the request body, do this with the latest express-http-proxy v1.6.2:
const express = require('express');
const proxy = require('express-http-proxy');
const bodyParser = require('body-parser');
const conf = {
proxyHost: 'some.example.net:9200',
proxyOptions: {
proxyReqBodyDecorator: modifyRequestBody,
preserveHostHdr: true,
parseReqBody: true
},
port: 8073
};
var app = express();
app.use('/proxy', proxy(conf.proxyHost, conf.proxyOptions));
function modifyRequestBody(body, srcReq) {
if(srcReq.method.match(/^(GET|POST)$/i)) {
try {
// convert buffer to string, then to object
var str = Buffer.from(body).toString('utf-8');
var reqBody = JSON.parse(str);
if(someCondition)) {
reqBody.addStuff = 'whatever';
body = reqBody; // return modified body as object
}
} catch(error) {
console.log('- error: ' + JSON.stringify(error));
}
}
return body; // return original buffer, or modified object
}
app.listen(conf.port, function () {
log('app listening on port ' + conf.port);
});
You can fill the proxyReq.bodyContent inside the decorateRequest method with the JSON-ed data from originalReq.body to be correctly POST'ed:
app.use('/proxy', users.requiresLogin, expressHttpProxy(config.backendSystem.host, {
...
...
decorateRequest: function (proxyReq, originalReq) {
...
...
if (originalReq.body) {
proxyReq.bodyContent = JSON.stringify(originalReq.body);
}
return proxyReq;
}
...
...
}));
I am using express to make a web app in node.js. This is a simplification of what I have:
var express = require('express');
var jade = require('jade');
var http = require("http");
var app = express();
var server = http.createServer(app);
app.get('/', function(req, res) {
// Prepare the context
res.render('home.jade', context);
});
app.post('/category', function(req, res) {
// Process the data received in req.body
res.redirect('/');
});
My problem is the following:
If I find that the data sent in /category doesn't validate, I would like pass some additional context to the / page. How could I do this? Redirect doesn't seem to allow any kind of extra parameter.
There are a few ways of passing data around to different routes. The most correct answer is, of course, query strings. You'll need to ensure that the values are properly encodeURIComponent and decodeURIComponent.
app.get('/category', function(req, res) {
var string = encodeURIComponent('something that would break');
res.redirect('/?valid=' + string);
});
You can snag that in your other route by getting the parameters sent by using req.query.
app.get('/', function(req, res) {
var passedVariable = req.query.valid;
// Do something with variable
});
For more dynamic way you can use the url core module to generate the query string for you:
const url = require('url');
app.get('/category', function(req, res) {
res.redirect(url.format({
pathname:"/",
query: {
"a": 1,
"b": 2,
"valid":"your string here"
}
}));
});
So if you want to redirect all req query string variables you can simply do
res.redirect(url.format({
pathname:"/",
query:req.query,
});
});
And if you are using Node >= 7.x you can also use the querystring core module
const querystring = require('querystring');
app.get('/category', function(req, res) {
const query = querystring.stringify({
"a": 1,
"b": 2,
"valid":"your string here"
});
res.redirect('/?' + query);
});
Another way of doing it is by setting something up in the session. You can read how to set it up here, but to set and access variables is something like this:
app.get('/category', function(req, res) {
req.session.valid = true;
res.redirect('/');
});
And later on after the redirect...
app.get('/', function(req, res) {
var passedVariable = req.session.valid;
req.session.valid = null; // resets session variable
// Do something
});
There is also the option of using an old feature of Express, req.flash. Doing so in newer versions of Express will require you to use another library. Essentially it allows you to set up variables that will show up and reset the next time you go to a page. It's handy for showing errors to users, but again it's been removed by default. EDIT: Found a library that adds this functionality.
Hopefully that will give you a general idea how to pass information around in an Express application.
The easiest way I have found to pass data between routeHandlers to use next() no need to mess with redirect or sessions.
Optionally you could just call your homeCtrl(req,res) instead of next() and just pass the req and res
var express = require('express');
var jade = require('jade');
var http = require("http");
var app = express();
var server = http.createServer(app);
/////////////
// Routing //
/////////////
// Move route middleware into named
// functions
function homeCtrl(req, res) {
// Prepare the context
var context = req.dataProcessed;
res.render('home.jade', context);
}
function categoryCtrl(req, res, next) {
// Process the data received in req.body
// instead of res.redirect('/');
req.dataProcessed = somethingYouDid;
return next();
// optionally - Same effect
// accept no need to define homeCtrl
// as the last piece of middleware
// return homeCtrl(req, res, next);
}
app.get('/', homeCtrl);
app.post('/category', categoryCtrl, homeCtrl);
I had to find another solution because none of the provided solutions actually met my requirements, for the following reasons:
Query strings: You may not want to use query strings because the URLs could be shared by your users, and sometimes the query parameters do not make sense for a different user. For example, an error such as ?error=sessionExpired should never be displayed to another user by accident.
req.session: You may not want to use req.session because you need the express-session dependency for this, which includes setting up a session store (such as MongoDB), which you may not need at all, or maybe you are already using a custom session store solution.
next(): You may not want to use next() or next("router") because this essentially just renders your new page under the original URL, it's not really a redirect to the new URL, more like a forward/rewrite, which may not be acceptable.
So this is my fourth solution that doesn't suffer from any of the previous issues. Basically it involves using a temporary cookie, for which you will have to first install cookie-parser. Obviously this means it will only work where cookies are enabled, and with a limited amount of data.
Implementation example:
var cookieParser = require("cookie-parser");
app.use(cookieParser());
app.get("/", function(req, res) {
var context = req.cookies["context"];
res.clearCookie("context", { httpOnly: true });
res.render("home.jade", context); // Here context is just a string, you will have to provide a valid context for your template engine
});
app.post("/category", function(req, res) {
res.cookie("context", "myContext", { httpOnly: true });
res.redirect("/");
}
use app.set & app.get
Setting data
router.get(
"/facebook/callback",
passport.authenticate("facebook"),
(req, res) => {
req.app.set('user', res.req.user)
return res.redirect("/sign");
}
);
Getting data
router.get("/sign", (req, res) => {
console.log('sign', req.app.get('user'))
});
we can use express-session to send the required data
when you initialise the app
const express = require('express');
const app = express();
const session = require('express-session');
app.use(session({secret: 'mySecret', resave: false, saveUninitialized: false}));
so before redirection just save the context for the session
app.post('/category', function(req, res) {
// add your context here
req.session.context ='your context here' ;
res.redirect('/');
});
Now you can get the context anywhere for the session. it can get just by req.session.context
app.get('/', function(req, res) {
// So prepare the context
var context=req.session.context;
res.render('home.jade', context);
});
Here s what I suggest without using any other dependency , just node and express, use app.locals, here s an example :
app.get("/", function(req, res) {
var context = req.app.locals.specialContext;
req.app.locals.specialContext = null;
res.render("home.jade", context);
// or if you are using ejs
res.render("home", {context: context});
});
function middleware(req, res, next) {
req.app.locals.specialContext = * your context goes here *
res.redirect("/");
}
You can pass small bits of key/value pair data via the query string:
res.redirect('/?error=denied');
And javascript on the home page can access that and adjust its behavior accordingly.
Note that if you don't mind /category staying as the URL in the browser address bar, you can just render directly instead of redirecting. IMHO many times people use redirects because older web frameworks made directly responding difficult, but it's easy in express:
app.post('/category', function(req, res) {
// Process the data received in req.body
res.render('home.jade', {error: 'denied'});
});
As #Dropped.on.Caprica commented, using AJAX eliminates the URL changing concern.
Update 2021:
i tried url.format and querystring and both of them are deprecated, instead we can use URLSearchParams
const {URLSearchParams} = require('url')
app.get('/category', (req, res) =>{
const pathname = '/?'
const components ={
a:"a",
b:"b"
}
const urlParameters = new URLSearchParams(components)
res.redirect(pathname + urlParameters)
})
I use a very simple but efficient technique
in my app.js ( my entry point )
I define a variable like
let authUser = {};
Then I assign to it from my route page ( like after successful login )
authUser = matchedUser
It May be not the best approach but it fits my needs.
app.get('/category', function(req, res) {
var string = query
res.redirect('/?valid=' + string);
});
in the ejs you can directly use valid:
<% var k = valid %>