This question already has answers here:
Expressjs raw body
(12 answers)
Closed 5 months ago.
When I use Express, and my code is:
app.use(express.bodyParser());
How would I get the raw request body?
Edit 2: Release 1.15.2 of the body parser module introduces raw mode, which returns the body as a Buffer. By default, it also automatically handles deflate and gzip decompression. Example usage:
var bodyParser = require('body-parser');
app.use(bodyParser.raw(options));
app.get(path, function(req, res) {
// req.body is a Buffer object
});
By default, the options object has the following default options:
var options = {
inflate: true,
limit: '100kb',
type: 'application/octet-stream'
};
If you want your raw parser to parse other MIME types other than application/octet-stream, you will need to change it here. It will also support wildcard matching such as */* or */application.
Note: The following answer is for versions before Express 4, where middleware was still bundled with the framework. The modern equivalent is the body-parser module, which must be installed separately.
The rawBody property in Express was once available, but removed since version 1.5.1. To get the raw request body, you have to put in some middleware before using the bodyParser. You can also read a GitHub discussion about it here.
app.use(function(req, res, next) {
req.rawBody = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
req.rawBody += chunk;
});
req.on('end', function() {
next();
});
});
app.use(express.bodyParser());
That middleware will read from the actual data stream, and store it in the rawBody property of the request. You can then access the raw body like this:
app.post('/', function(req, res) {
// do something with req.rawBody
// use req.body for the parsed body
});
Edit: It seems that this method and bodyParser refuse to coexist, because one will consume the request stream before the other, leading to whichever one is second to never fire end, thus never calling next(), and hanging your application.
The simplest solution would most likely be to modify the source of bodyParser, which you would find on line 57 of Connect's JSON parser. This is what the modified version would look like.
var buf = '';
req.setEncoding('utf8');
req.on('data', function(chunk){ buf += chunk });
req.on('end', function() {
req.rawBody = buf;
var first = buf.trim()[0];
...
});
You would find the file at this location:
/node_modules/express/node_modules/connect/lib/middleware/json.js.
I got a solution that plays nice with bodyParser, using the verify callback in bodyParser. In this code, I am using it to get a sha1 of the content and also getting the raw body.
app.use(bodyParser.json({
verify: function(req, res, buf, encoding) {
// sha1 content
var hash = crypto.createHash('sha1');
hash.update(buf);
req.hasha = hash.digest('hex');
console.log("hash", req.hasha);
// get rawBody
req.rawBody = buf.toString();
console.log("rawBody", req.rawBody);
}
}));
I am new in Node.js and express.js (started yesterday, literally!) so I'd like to hear comments on this solution.
This solution worked for me:
var rawBodySaver = function (req, res, buf, encoding) {
if (buf && buf.length) {
req.rawBody = buf.toString(encoding || 'utf8');
}
}
app.use(bodyParser.json({ verify: rawBodySaver }));
app.use(bodyParser.urlencoded({ verify: rawBodySaver, extended: true }));
app.use(bodyParser.raw({ verify: rawBodySaver, type: '*/*' }));
When I use solution with req.on('data', function(chunk) { }); it not working on chunked request body.
BE CAREFUL with those other answers as they will not play properly with bodyParser if you're looking to also support json, urlencoded, etc. To get it to work with bodyParser you should condition your handler to only register on the Content-Type header(s) you care about, just like bodyParser itself does.
To get the raw body content of a request with Content-Type: "text/plain" into req.rawBody you can do:
app.use(function(req, res, next) {
var contentType = req.headers['content-type'] || ''
, mime = contentType.split(';')[0];
if (mime != 'text/plain') {
return next();
}
var data = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
next();
});
});
This is a variation on hexacyanide's answer above. This middleware also handles the 'data' event but does not wait for the data to be consumed before calling 'next'. This way both this middleware and bodyParser may coexist, consuming the stream in parallel.
app.use(function(req, res, next) {
req.rawBody = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
req.rawBody += chunk;
});
next();
});
app.use(express.bodyParser());
In 2022
The best way to get raw body in every API is to convert buffer into a string.
app.use(
express.json({
limit: '5mb',
verify: (req, res, buf) => {
req.rawBody = buf.toString();
},
})
);
// Change the way body-parser is used
const bodyParser = require('body-parser');
var rawBodySaver = function (req, res, buf, encoding) {
if (buf && buf.length) {
req.rawBody = buf.toString(encoding || 'utf8');
}
}
app.use(bodyParser.json({ verify: rawBodySaver, extended: true }));
// Now we can access raw-body any where in out application as follows
request.rawBody;
Use
app.use(express.json());
or
app.use(express.text());
or
app.use(express.urlencoded());
Depending upon your raw format
I have resolved the adding to the header property.
{"Content-Type": "application/json"}
Use body-parser Parse the body with what it will be:
app.use(bodyParser.text());
app.use(bodyParser.urlencoded());
app.use(bodyParser.raw());
app.use(bodyParser.json());
ie. If you are supposed to get raw text file, run .text().
Thats what body-parser currently supports
Related
I have the following code:
var getRequest=function(options){
var body='';
var req = http.get(options,function(response){
response.on('data', function(chunk) {
body+=chunk;
}).on('end', function(chunk) {
console.log("body");
return JSON.parse(body);
}).on('error',function(error){
console.log("error: "+error.getMessage());
})
})
return req;
};
What I am trying to do is pass a JSON object of http options,eg:
var options={
host:'localhost',
port:'8080',
method:'GET',
path:'/stuff'
};
and send back a parsed response. However, I can't get this to work and I think its because of the nested function and my misunderstanding of how they work.
Please could someone advise me on how to get the function to return the result of JSON.parse(body) to getRequest?
TypeError: Converting circular structure to JSON
thanks.
Easy way to do it is with a body-parser middleware:
Install body-parser with npm (npm install body-parser --save)
Use it like this
var bodyParser = require('body-parser');
var app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
//Now you got access to the body object in request.body
app.get('/stuff', function(request, response) {
response.send(request.body);
}
If you want to handle the body packages yourself, here is a sample of working code.
var data = [];
request.on('data', function(chunck) {
data.push(chunck);
}).on('end', function() {
data = Buffer.concat(data).toString();
response.send(JSON.parse(data));
}).on('error', function(error) {
console.log(error);
response.status(400).send(error);
});
I wrote an Express middleware to retrieve the raw body from the request, and I set it before body-parser middleware.
My custom middleware is calling req.setEncoding('utf8'), but this causes the following body-parser error:
Error: stream encoding should not be set
at readStream (/node_modules/body-parser/node_modules/raw-body/index.js:211:17)
at getRawBody (/node_modules/body-parser/node_modules/raw-body/index.js:106:12)
at read (/node_modules/body-parser/lib/read.js:76:3)
at jsonParser (/node_modules/body-parser/lib/types/json.js:127:5)
Here is my code:
var express = require('express');
var bodyParser = require('body-parser')
function myMiddleware() {
return function(req, res, next) {
req.rawBody = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
req.rawBody += chunk;
});
req.on('end', function() {
next();
});
}
}
var app = express();
app.use(myMiddleware());
app.use(bodyParser.json());
var listener = app.listen(3000, function() {
});
app.get('/webhook/', function (req, res) {
res.sendStatus(200);
});
Is there a way to unset the encoding? Is there another way to retrieve the raw body, but still use body-parser after it?
It turns out that body-parser has a verify option to call a function when the request body has been read. The function receives the body as a buffer.
Here is an example:
var express = require('express');
var bodyParser = require('body-parser')
function verifyRequest(req, res, buf, encoding) {
// The raw body is contained in 'buf'
console.log( buf.toString( encoding ) );
};
var app = express();
var listener = app.listen(3000);
// Hook 'verifyRequest' with body-parser here.
app.use(bodyParser.json({ verify: verifyRequest }))
app.post('/webhook/', function (req, res) {
res.status(200).send("done!");
});
You are calling next() inside "done", which means the stream has already been consumed. Instead, set up the handler for "data" then pass the request along using next(). The "done" event is likely being handled inside bodyParser so after it executes you have access to req.rawBody. If this was not the case you would add another middleware that calls next() inside a req.on('done') to hold the rest from processing until you have all the data.
// custom middleware - req, res, next must be arguments on the top level function
function myMiddleware(req, res, next) {
req.rawBody = '';
req.on('data', function(chunk) {
req.rawBody += chunk;
});
// call next() outside of 'end' after setting 'data' handler
next();
}
// your middleware
app.use(myMiddleware);
// bodyparser
app.use(bodyParser.json())
// test that it worked
function afterMiddleware(req, res, next) {
console.log(req.rawBody);
next();
}
app.use(afterMiddleware);
If you need to access the raw body you might also want to look into bodyParser.raw(). This will put the raw body into req.body, same as bodyParse.json() but can be made to run conditionally based on the content type - check out options.type.
I recommend a different approach, since your current approach actually consumes the message and makes it impossible for body-parser to read it (and there are a bunch of edge case bugs that spring up by calling next synchronously):
app.use(bodyParser.json());
app.use(bodyParser.text({type: '*/*'}));
This will read any application/json request as JSON, and everything else as text.
If you must have the JSON object in addition to the text, I recommend parsing it yourself:
app.use(bodyParser.text({type: '*/*'}));
app.use(myMiddleware);
function myMiddleware(req, res, next) {
req.rawBody = req.body;
if(req.headers['content-type'] === 'application/json') {
req.body = JSON.parse(req.body);
}
next();
}
I need to be able to use the http request body in my request proxy application and then again in the actual web service. I am using restreamer to 'reset' the stream (and even wrote the middleware myself with no change). The web service receives the body just fine, but because end is never emitted, I cannot continue with the request.
Testing with postman, sending a raw body, with content type set. Any suggestions would be greatly appreciated. Thanks!
var express = require('express')
, bodyParser = require('body-parser')
, http = require('http')
, restreamer = require('connect-restreamer')
, httpProxy = require('http-proxy')
var app = express();
app.use(function (req, res, next) {
var body = '';
req.on('data', function (chunk) {
body += chunk.toString('utf8');
});
req.on('end', function (chunk) {
req.body = JSON.parse(body)
next();
});
});
app.use(restreamer());
var proxy = httpProxy.createServer();
app.all('*', function (req, res) {
proxy.web(req, res, {
target: 'http://localhost:8001'
});
});
http.createServer(app).listen(8000);
app2 = express();
app2.use(function (req, res, next) {
var body = '';
req.on('data', function (chunk) {
body += chunk.toString('utf8');
});
req.on('end', function (chunk) {
req.body = JSON.parse(body)
next();
});
});
app2.all('*', function (req, res) {
res.send(req.body)
});
http.createServer(app2).listen(8001);
Using the request library in my application, it worked:
var request = require('request')
request.post({
url: 'http://localhost:8000',
json: {content: 123, type: "greeting", access_token: "here i am"}
},function(err, res,data){
console.log('return:' ,err, data)
});
But using curl with a file containing the same message, it would not work:
curl -X POST -d #data.json http://localhost:8000 --header "Content-Type:application/json"
I compared the request objects against each other and found a few differences and when I got to the request header for content-length, I found that editing it the "correct" length would end the steam (and the web server would send a response).
I will make the modifications needed and commit to connect-restreamer module.
I need to get the raw post data for a specific endpoint in a node/express app. I do:
app.use('/paypal/ipn',function(req, res, next) {
var postData='';
req.on('data', function(chunk) {
postData += chunk;
});
req.on('end', function() {
req.rawBody = postData;
console.log('ended buffering. result: ' + req.rawBody);
next();
});
});
What happens is that I get the console.log output in the console and then nothing happens. After a minute or so I see the server returns 200 - probably a timeout. It's like the next() command never executes, or executes and stales.
When I comment out everything, and simply call next():
app.use('/paypal/ipn',function(req, res, next) {
/*
var postData='';
req.on('data', function(chunk) {
postData += chunk;
});
req.on('end', function() {
req.rawBody = postData;
console.log('ended buffering. result: ' + req.rawBody);
next();
});
*/
next();
});
Everything works, that is the endpoint is called (of course the request doesn't contain the rawBody).
So it seems like I'm doing something wrong the way I buffer the rawBody? Something that causes next() not to work?
Call next() without waiting for 'end', to prevent bodyParser and your code from interfering.
app.use(function(req, res, next) {
req.rawBody = '';
req.on('data', function(chunk) {
req.rawBody += chunk;
});
next();
});
app.use(express.bodyParser());
Kudos to https://stackoverflow.com/a/21222540/1891397
I had similar problem, checkout this code (rawBody):
/*
for some very strange reason, while a IE8/IE9 use a XDomain for CORS requests, express.js bodyParser fail to parse
*/
app.use(function(req, res, next) {
if(req.method !== 'POST') {
next();
return;
}
var data = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
req.failSafeBody = queryParser(data);
next();
});
});
//se above
//app.use(express.bodyParser());
app.use(function(req, res, next){
if(req.method === 'POST' && (!req.body || Object.keys(req.body).length<1) ) {
req.body = req.failSafeBody;
}
next();
});
app.use(express.methodOverride());
A simple change would show why next isn't run:
app.use('/paypal/ipn',function(req, res, next) {
var postData='';
req.on('data', function(chunk) {
postData += chunk;
});
req.on('end', function() {
req.rawBody = postData;
console.log('ended buffering. result: ' + req.rawBody);
console.log(next);
});
});
It is likely that the next callback simply doesn't exist and so fails silently. If it DOES exist, then you just have to enter it and debug it line by line.
I got same troubles like that.
So, after no good answer found on google, I spent more hours to fix it.
You should be noticed that on middleware of connect node_module, it completed reading data on stream when you call express.urlencoded(), so I find a best way to fix that problems:
app.use(express.urlencoded());
app.use(function(req, res, next){
if (req.method !== 'POST' || JSON.stringify(req.body) !== JSON.stringify({})){
next()
return
}
var data = ''
req.on('data', function(chunk){
data += chunk
})
req.on('end', function(){
req.rawBody = data
next()
})
})
How it works?
First, we completed reading stream by calling urlencoded() and we got req.body by function returned.
After that, in hack function rawBody, we check if method is not 'POST' (GET ?) or req.body is not equals {} that means urlencoded() parsed data sucessfully, we dont need to do anything (or if you still need rawBody at this point, you simply join() req.body with '&') so we simple call next() and return for breaking function
(sr my bad English, I really want to share this exp, I think you will need it like me ;) )
How can I access raw body of request object given to me by expressjs?
var express = require('./node_modules/express');
var app = express.createServer();
app.post('/', function(req, res)
{
console.log(req.body); //says 'undefined'
});
app.listen(80);
Something like this should work:
var express = require('./node_modules/express');
var app = express.createServer();
app.use (function(req, res, next) {
var data='';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.body = data;
next();
});
});
app.post('/', function(req, res)
{
console.log(req.body);
});
app.listen(80);
Using the bodyParser.text() middleware will put the text body in req.body.
app.use(bodyParser.text({type: '*/*'}));
If you want to limit processing the text body to certain routes or post content types, you can do that too.
app.use('/routes/to/save/text/body/*', bodyParser.text({type: 'text/plain'})); //this type is actually the default
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));
If you want a raw Buffer, you can use bodyParse.raw().
app.use(bodyParser.raw({type: '*/*'}));
Note: this answer was tested against node v0.12.7, express 4.13.2, and body-parser 1.13.3.
Put the following middleware before bodyParser middleware. It'll collect raw body data in request.rawBody and won't interfere with bodyParser.
app.use(function(req, res, next) {
var data = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
next();
});
});
app.use(express.bodyParser());
Default express does not buffer data unless you add middleware to do so. The simple solution is to follow the example in #Stewe's answer below, which would just concatenate all of the data yourself. e.g.
var concat = require('concat-stream');
app.use(function(req, res, next){
req.pipe(concat(function(data){
req.body = data;
next();
}));
});
The downside of this is that you have now moved all of the POST body content into RAM as a contiguous chunk, which may not be necessary. The other option, which is worth considering but depends on how much data you need to process in the post body, would be to process the data as a stream instead.
For example, with XML you could use an XML parser that supports parsing XML as it comes in as chunks. One such parser would be XML Stream. You do something like this:
var XmlStream = require('xml-stream');
app.post('/', function(req, res) {
req.setEncoding('utf8');
var xml = new XmlStream(req);
xml.on('updateElement: sometag', function(element) {
// DO some processing on the tag
});
xml.on('end', function() {
res.end();
});
});
app.use(bodyParser.json({
verify: function (req, res, buf, encoding) {
req.rawBody = buf;
}
}));
app.use(bodyParser.urlencoded({
extended: false,
verify: function (req, res, buf, encoding) {
req.rawBody = buf;
}
}));
So, it seems like Express's bodyParser only parses the incoming data, if the content-type is set to either of the following:
application/x-www-form-urlencoded
application/json
multipart/form-data
In all other cases, it does not even bother reading the data.
You can change line no. 92 of express/node_modules/connect/lib/middleware/bodyParser.js from
} else {
next();
}
To:
} else {
var data='';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
next();
});
}
And then, read req.rawBody from your code.
If you want the body as a buffer:
var rawParser = function(req, res, next) {
var chunks = [];
req.on('data', function(chunk) {
chunks.push(chunk)
});
req.on('end', function() {
req.body = Buffer.concat(chunks);
next();
});
}
or
var rawParser = bodyParser.raw({type: '*/*'});
and then:
app.put('/:name', rawParser, function(req, res) {
console.log('isBuffer:', Buffer.isBuffer(req.body));
})
or for all routes:
app.use(bodyParser.raw({type: '*/*'}));
It seems this has become a lot easier now!
The body-parser module is able to parse raw and text data now, which makes the task a one-liner:
app.use(bodyParser.text({type: 'text/plain'}))
OR
app.use(bodyParser.raw({type: 'application/binary'}))
Both lines simply fill the body property, so get the text with res.body.
bodyParser.text() will give you the UTF8 string while bodyParser.raw() will give you the raw data.
This is the full code for text/plain data:
var express = require('express')
var bodyParser = require('body-parser')
var app = express()
app.use(bodyParser.text({type: 'text/plain'}))
app.post('/', function (req, res, next) {
console.log('body:\n' + req.body)
res.json({msg: 'success!'})
next()
})
See here for the full documentation:
https://www.npmjs.com/package/body-parser
I used express 4.16 and body-parser 1.18
If you are having trouble with the above solutions interfering with normal post requests, something like this might help:
app.use (function(req, res, next) {
req.rawBody = '';
req.setEncoding('utf8');
req.on('data', function(chunk) { req.rawBody += chunk });
});
More info & source: https://github.com/visionmedia/express/issues/897#issuecomment-3314823
All the answers seems outdated, if anyone still struggling with this then express has built-in Express raw middeware.
This middleware is available in Express v4.16.0 onwards.
This is a built-in middleware function in Express. It parses incoming requests with JSON payloads and is based on body-parser.
var express = require("express");
var app = express();
app.use(express.raw({ type: "*/*" }))
app.post("/", (req, res) => {
// req.body = JSON.parse(req.body); // To parse JSON if needed (in-case)
console.log(req.body);
res.end();
});
app.listen(3000, (err) => {
if(!err) console.log("App running!!")
});
BE CAREFUL with those other answers as they will not play properly with bodyParser if you're looking to also support json, urlencoded, etc. To get it to work with bodyParser you should condition your handler to only register on the Content-Type header(s) you care about, just like bodyParser itself does.
To get the raw body content of a request with Content-Type: "text/xml" into req.rawBody you can do:
app.use(function(req, res, next) {
var contentType = req.headers['content-type'] || ''
, mime = contentType.split(';')[0];
if (mime != 'text/xml') {
return next();
}
var data = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
next();
});
});
When sending the request be sure to add this header:
'Content-Type': 'application/json'