I was using bodyParser.json() as a middleware with express and I recently replaced it by this code :
//gets any json object and put them in req.body
app.use((req, res, next) => {
let lang = req.acceptsLanguages()[0];
const decoder = new StringDecoder('UTF-8');
req.on('data', function(data) {
try {
req.body = JSON.parse(decoder.write(data));
next()
} catch (ex) {
res.status(400).send(translator(lang, 'entry.error.input.malformed'));
}
});
});
Do I have to use bodyParser and add the library to my project ? or my custom code is enough to parse and inject raw data as Json object in req??
Your custom middleware doesn't quite seem right, i would use
app.use((req, res, next) => {
let lang = req.acceptsLanguages()[0];
let data = '';
req.on('data', chunk => data += chunk);
req.on('end', () => {
try {
req.body = JSON.parse(data);
next()
} catch (ex) {
res.status(400).send(translator(lang, 'entry.error.input.malformed'));
}
});
});
And yes this is enough, if you expect, all the requests coming into the server to be of JSON type. bodyparser basically does the same, except that, it handles a lot other cases
Related
I'm trying to retrieve data from KEEPA about Amazon's products.
I'm straggling to receive the data in proper JSON format, as KEEPA sending the data as gzip.
I tried to used 'decompressResponse' module which helped to get the data as JSON but it was received multiple times on each call.
As the code appears below I'm just getting a huge Gibberish to my console.
Let me know what am I missing here, or if you have a better suggestion please let me know.
Thanks
const express = require("express");
const https = require("https");
const bodyParser = require("body-parser");
const app = express();
app.use(bodyParser.urlencoded({extended: true}));
app.get("/", function(req, res) {
res.sendFile(__dirname + "/index.html");
});
app.post("/", function(req, res) {
const query = req.body.asinId;
const apiKey = "MY_API_KEY";
const url = "https://api.keepa.com/product?key=" + apiKey + "&domain=1&asin=" + query;
const options = {
methode: "GET",
headers: {
"Content-Type": "application/json;charset=UTF-8",
"Accept-Encoding":"gzip"
}
}
https.get(url,options,function(res) {
console.log(res.statusCode);
console.log(res.headers);
var data;
res.on("data", function(chunk){
if(data){
data = chunk;
} else {
data += chunk;
}
console.log(data);
});
});
res.send("server is running");
});
app.listen(3000, function() {
console.log("server is running on port 3000");
});
Have you tried using the built-in zlib module with gunzip()?
zlib.gunzip(data, (error, buff) => {
if (error != null) {
// An error occured while unzipping the .gz file.
} else {
// Use the buff which contains the unzipped JSON.
console.log(buff)
}
});
Full example with your code: https://www.napkin.io/n/7c6bc48d989b4727
well the output function was wrong .. the correct one below
https.get(url,options,function(response) {
response = decompressResponse(response);
console.log(res.statusCode);
console.log(res.headers);
let data = '';
response.on("data", function(chunk){
data += chunk;
});
response.on("end",function(){
console.log(data);
});
});
I have a small api I have built using Node.js and express.
I am trying to create a logger and I need log the request body AND response body.
app.use((req, res) => {
console.log(req);
res.on("finish", () => {
console.log(res);
});
});
"express": "^4.16.3",
However, i am not able to find the body in the req or res object. Please tell me how i can get them. thanks.
For res.body try the following snippet:
const endMiddleware = (req, res, next) => {
const defaultWrite = res.write;
const defaultEnd = res.end;
const chunks = [];
res.write = (...restArgs) => {
chunks.push(new Buffer(restArgs[0]));
defaultWrite.apply(res, restArgs);
};
res.end = (...restArgs) => {
if (restArgs[0]) {
chunks.push(new Buffer(restArgs[0]));
}
const body = Buffer.concat(chunks).toString('utf8');
console.log(body);
defaultEnd.apply(res, restArgs);
};
next();
};
app.use(endMiddleware)
// test
// HTTP GET /
res.status(200).send({ isAlive: true });
You need body-parser that will create body object for you in your request. To do that
npm install body-parser
var bodyParser = require('body-parser')//add this
app.use(bodyParser())//add this before any route or before using req.body
app.use((req, res) => {
console.log(req.body); // this is what you want
res.on("finish", () => {
console.log(res);
});
});
Ran into this problem but didn't like the solutions. An easy way is to simply wrap the original res.send or res.json with your logger.
Put this as middleware before your routes.
app.use(function responseLogger(req, res, next) {
const originalSendFunc = res.send.bind(res);
res.send = function(body) {
console.log(body); // do whatever here
return originalSendFunc(body);
};
next();
});
https://github.com/expressjs/express/blob/master/lib/response.js
res.send has signature of function(body) { return this; }
Here is a working example using the built in PassThrough stream. Remember to use the express.json() built in middleware to enable request body parsing.
After that, you need to intercept all writes to the response stream. Writes will happen on calling write or end, so replace those functions and capture the arguments in a separate stream.
Use res.on('finish', ...) to gather all the written data into a Buffer using Buffer.concat and print it.
const express = require('express');
const { PassThrough } = require('stream')
const app = express();
app.use(express.json());
app.use((req, res, next) => {
const defaultWrite = res.write.bind(res);
const defaultEnd = res.end.bind(res);
const ps = new PassThrough();
const chunks = [];
ps.on('data', data => chunks.push(data));
res.write = (...args) => {
ps.write(...args);
defaultWrite(...args);
}
res.end = (...args) => {
ps.end(...args);
defaultEnd(...args);
}
res.on('finish', () => {
console.log("req.body", req.body);
console.log("res.body", Buffer.concat(chunks).toString());
})
next();
})
app.use('/', (req, res) => {
res.send("Hello");
});
app.listen(3000);
install npm install body-parser
and use this snippet,
var express = require('express')
var bodyParser = require('body-parser')
var app = express()
// create application/json parser
var jsonParser = bodyParser.json()
to get json response
app.use(jsonParser, function (req, res) {
console.log(req.body); // or console.log(res.body);
})
There is ready made module https://www.npmjs.com/package/morgan-body
const express = require('express')
const morganBody = require("morgan-body")
const bodyParser = require("body-parser")
const app = express()
const port = 8888
// must parse body before morganBody as body will be logged
app.use(bodyParser.json());
// hook morganBody to express app
morganBody(app, {logAllReqHeader:true, maxBodyLength:5000});
app.get('/', (req, res) => {
res.send('Hello World!')
})
app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`)
})
Hi was looking for same as complete log of request and response as middleware in express js. Found the solution as well w
/*Added by vikram parihar for log */
const moment = require('moment');
const rfs = require("rotating-file-stream");
const geoip = require('geoip-lite');
const { PassThrough } = require('stream')
let path = require('path');
const accessLogStream = rfs.createStream('access.log', {
interval: '1M', // rotate daily
compress: true,
path: path.join(__dirname, '../../log')
});
module.exports = function (req, res, next) {
try {
let geo = geoip.lookup(req.ip);
let country = geo ? geo.country : "Unknown";
let region = geo ? geo.region : "Unknown";
let log = {
"time": moment().format('YYYY/MM/DD HH:mm:ss'),
"host": req.hostname,
"ip": req.ip,
"originalUrl": req.originalUrl,
"geo": {
"browser": req.headers["user-agent"],
"Language": req.headers["accept-language"],
"Country": country,
"Region": region,
},
"method": req.method,
"path": req.path,
"url": req.url,
"body": req.body,
"params": req.params,
"query": req.query,
"response": {
"body": res.body
}
};
const defaultWrite = res.write.bind(res);
const defaultEnd = res.end.bind(res);
const ps = new PassThrough();
const chunks = [];
ps.on('data', data => chunks.push(data));
res.write = (...args) => {
ps.write(...args);
defaultWrite(...args);
}
res.end = (...args) => {
ps.end(...args);
defaultEnd(...args);
}
res.on('finish', () => {
log.response.body = Buffer.concat(chunks).toString()
accessLogStream.write(JSON.stringify(log) + "\n");
})
} catch (error) {
console.log(error)
next(error)
}
next();
}
I'm implementing a simple app that simply sends some GET request to a nodeJS-express endpoint. This endpoint is returning just this:
router.get('/', (request, response) => {
response.status(500).send("Error message");
}
My app is using request to send the request like this:
request.get(BASE_URL)
.on("error", err => {
console.log(`ERROR: ${err}`)
})
.on("response", res => {
console.log(`RESPONSE: ${res.statusCode} - ${res.body}`)
});
But body is always undefined. I have tried to use:
.on("response", res => {
res.on("data", () => {
console.log(`RESPONSE: ${res.statusCode} - ${data}`)
}
});
But the data is some byte array. I just want the string message, where it is and how can I get it without using the callback syntax?
If You are using express you will need to inject the body-parser middleware for parsing the request body.
const bodyParser = require('body-parser');
app.use(bodyParser.json());
app.post('/login', (req, res) => {
console.log(req.body); //data
})
I wrote an Express middleware to retrieve the raw body from the request, and I set it before body-parser middleware.
My custom middleware is calling req.setEncoding('utf8'), but this causes the following body-parser error:
Error: stream encoding should not be set
at readStream (/node_modules/body-parser/node_modules/raw-body/index.js:211:17)
at getRawBody (/node_modules/body-parser/node_modules/raw-body/index.js:106:12)
at read (/node_modules/body-parser/lib/read.js:76:3)
at jsonParser (/node_modules/body-parser/lib/types/json.js:127:5)
Here is my code:
var express = require('express');
var bodyParser = require('body-parser')
function myMiddleware() {
return function(req, res, next) {
req.rawBody = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
req.rawBody += chunk;
});
req.on('end', function() {
next();
});
}
}
var app = express();
app.use(myMiddleware());
app.use(bodyParser.json());
var listener = app.listen(3000, function() {
});
app.get('/webhook/', function (req, res) {
res.sendStatus(200);
});
Is there a way to unset the encoding? Is there another way to retrieve the raw body, but still use body-parser after it?
It turns out that body-parser has a verify option to call a function when the request body has been read. The function receives the body as a buffer.
Here is an example:
var express = require('express');
var bodyParser = require('body-parser')
function verifyRequest(req, res, buf, encoding) {
// The raw body is contained in 'buf'
console.log( buf.toString( encoding ) );
};
var app = express();
var listener = app.listen(3000);
// Hook 'verifyRequest' with body-parser here.
app.use(bodyParser.json({ verify: verifyRequest }))
app.post('/webhook/', function (req, res) {
res.status(200).send("done!");
});
You are calling next() inside "done", which means the stream has already been consumed. Instead, set up the handler for "data" then pass the request along using next(). The "done" event is likely being handled inside bodyParser so after it executes you have access to req.rawBody. If this was not the case you would add another middleware that calls next() inside a req.on('done') to hold the rest from processing until you have all the data.
// custom middleware - req, res, next must be arguments on the top level function
function myMiddleware(req, res, next) {
req.rawBody = '';
req.on('data', function(chunk) {
req.rawBody += chunk;
});
// call next() outside of 'end' after setting 'data' handler
next();
}
// your middleware
app.use(myMiddleware);
// bodyparser
app.use(bodyParser.json())
// test that it worked
function afterMiddleware(req, res, next) {
console.log(req.rawBody);
next();
}
app.use(afterMiddleware);
If you need to access the raw body you might also want to look into bodyParser.raw(). This will put the raw body into req.body, same as bodyParse.json() but can be made to run conditionally based on the content type - check out options.type.
I recommend a different approach, since your current approach actually consumes the message and makes it impossible for body-parser to read it (and there are a bunch of edge case bugs that spring up by calling next synchronously):
app.use(bodyParser.json());
app.use(bodyParser.text({type: '*/*'}));
This will read any application/json request as JSON, and everything else as text.
If you must have the JSON object in addition to the text, I recommend parsing it yourself:
app.use(bodyParser.text({type: '*/*'}));
app.use(myMiddleware);
function myMiddleware(req, res, next) {
req.rawBody = req.body;
if(req.headers['content-type'] === 'application/json') {
req.body = JSON.parse(req.body);
}
next();
}
How can I access raw body of request object given to me by expressjs?
var express = require('./node_modules/express');
var app = express.createServer();
app.post('/', function(req, res)
{
console.log(req.body); //says 'undefined'
});
app.listen(80);
Something like this should work:
var express = require('./node_modules/express');
var app = express.createServer();
app.use (function(req, res, next) {
var data='';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.body = data;
next();
});
});
app.post('/', function(req, res)
{
console.log(req.body);
});
app.listen(80);
Using the bodyParser.text() middleware will put the text body in req.body.
app.use(bodyParser.text({type: '*/*'}));
If you want to limit processing the text body to certain routes or post content types, you can do that too.
app.use('/routes/to/save/text/body/*', bodyParser.text({type: 'text/plain'})); //this type is actually the default
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));
If you want a raw Buffer, you can use bodyParse.raw().
app.use(bodyParser.raw({type: '*/*'}));
Note: this answer was tested against node v0.12.7, express 4.13.2, and body-parser 1.13.3.
Put the following middleware before bodyParser middleware. It'll collect raw body data in request.rawBody and won't interfere with bodyParser.
app.use(function(req, res, next) {
var data = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
next();
});
});
app.use(express.bodyParser());
Default express does not buffer data unless you add middleware to do so. The simple solution is to follow the example in #Stewe's answer below, which would just concatenate all of the data yourself. e.g.
var concat = require('concat-stream');
app.use(function(req, res, next){
req.pipe(concat(function(data){
req.body = data;
next();
}));
});
The downside of this is that you have now moved all of the POST body content into RAM as a contiguous chunk, which may not be necessary. The other option, which is worth considering but depends on how much data you need to process in the post body, would be to process the data as a stream instead.
For example, with XML you could use an XML parser that supports parsing XML as it comes in as chunks. One such parser would be XML Stream. You do something like this:
var XmlStream = require('xml-stream');
app.post('/', function(req, res) {
req.setEncoding('utf8');
var xml = new XmlStream(req);
xml.on('updateElement: sometag', function(element) {
// DO some processing on the tag
});
xml.on('end', function() {
res.end();
});
});
app.use(bodyParser.json({
verify: function (req, res, buf, encoding) {
req.rawBody = buf;
}
}));
app.use(bodyParser.urlencoded({
extended: false,
verify: function (req, res, buf, encoding) {
req.rawBody = buf;
}
}));
So, it seems like Express's bodyParser only parses the incoming data, if the content-type is set to either of the following:
application/x-www-form-urlencoded
application/json
multipart/form-data
In all other cases, it does not even bother reading the data.
You can change line no. 92 of express/node_modules/connect/lib/middleware/bodyParser.js from
} else {
next();
}
To:
} else {
var data='';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
next();
});
}
And then, read req.rawBody from your code.
If you want the body as a buffer:
var rawParser = function(req, res, next) {
var chunks = [];
req.on('data', function(chunk) {
chunks.push(chunk)
});
req.on('end', function() {
req.body = Buffer.concat(chunks);
next();
});
}
or
var rawParser = bodyParser.raw({type: '*/*'});
and then:
app.put('/:name', rawParser, function(req, res) {
console.log('isBuffer:', Buffer.isBuffer(req.body));
})
or for all routes:
app.use(bodyParser.raw({type: '*/*'}));
It seems this has become a lot easier now!
The body-parser module is able to parse raw and text data now, which makes the task a one-liner:
app.use(bodyParser.text({type: 'text/plain'}))
OR
app.use(bodyParser.raw({type: 'application/binary'}))
Both lines simply fill the body property, so get the text with res.body.
bodyParser.text() will give you the UTF8 string while bodyParser.raw() will give you the raw data.
This is the full code for text/plain data:
var express = require('express')
var bodyParser = require('body-parser')
var app = express()
app.use(bodyParser.text({type: 'text/plain'}))
app.post('/', function (req, res, next) {
console.log('body:\n' + req.body)
res.json({msg: 'success!'})
next()
})
See here for the full documentation:
https://www.npmjs.com/package/body-parser
I used express 4.16 and body-parser 1.18
If you are having trouble with the above solutions interfering with normal post requests, something like this might help:
app.use (function(req, res, next) {
req.rawBody = '';
req.setEncoding('utf8');
req.on('data', function(chunk) { req.rawBody += chunk });
});
More info & source: https://github.com/visionmedia/express/issues/897#issuecomment-3314823
All the answers seems outdated, if anyone still struggling with this then express has built-in Express raw middeware.
This middleware is available in Express v4.16.0 onwards.
This is a built-in middleware function in Express. It parses incoming requests with JSON payloads and is based on body-parser.
var express = require("express");
var app = express();
app.use(express.raw({ type: "*/*" }))
app.post("/", (req, res) => {
// req.body = JSON.parse(req.body); // To parse JSON if needed (in-case)
console.log(req.body);
res.end();
});
app.listen(3000, (err) => {
if(!err) console.log("App running!!")
});
BE CAREFUL with those other answers as they will not play properly with bodyParser if you're looking to also support json, urlencoded, etc. To get it to work with bodyParser you should condition your handler to only register on the Content-Type header(s) you care about, just like bodyParser itself does.
To get the raw body content of a request with Content-Type: "text/xml" into req.rawBody you can do:
app.use(function(req, res, next) {
var contentType = req.headers['content-type'] || ''
, mime = contentType.split(';')[0];
if (mime != 'text/xml') {
return next();
}
var data = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
next();
});
});
When sending the request be sure to add this header:
'Content-Type': 'application/json'