Writing a general purpose http get using node and express - node.js

I have the following code:
var getRequest=function(options){
var body='';
var req = http.get(options,function(response){
response.on('data', function(chunk) {
body+=chunk;
}).on('end', function(chunk) {
console.log("body");
return JSON.parse(body);
}).on('error',function(error){
console.log("error: "+error.getMessage());
})
})
return req;
};
What I am trying to do is pass a JSON object of http options,eg:
var options={
host:'localhost',
port:'8080',
method:'GET',
path:'/stuff'
};
and send back a parsed response. However, I can't get this to work and I think its because of the nested function and my misunderstanding of how they work.
Please could someone advise me on how to get the function to return the result of JSON.parse(body) to getRequest?
TypeError: Converting circular structure to JSON
thanks.

Easy way to do it is with a body-parser middleware:
Install body-parser with npm (npm install body-parser --save)
Use it like this
var bodyParser = require('body-parser');
var app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
//Now you got access to the body object in request.body
app.get('/stuff', function(request, response) {
response.send(request.body);
}
If you want to handle the body packages yourself, here is a sample of working code.
var data = [];
request.on('data', function(chunck) {
data.push(chunck);
}).on('end', function() {
data = Buffer.concat(data).toString();
response.send(JSON.parse(data));
}).on('error', function(error) {
console.log(error);
response.status(400).send(error);
});

Related

how to receive gzip data ? Node.js

I'm trying to retrieve data from KEEPA about Amazon's products.
I'm straggling to receive the data in proper JSON format, as KEEPA sending the data as gzip.
I tried to used 'decompressResponse' module which helped to get the data as JSON but it was received multiple times on each call.
As the code appears below I'm just getting a huge Gibberish to my console.
Let me know what am I missing here, or if you have a better suggestion please let me know.
Thanks
const express = require("express");
const https = require("https");
const bodyParser = require("body-parser");
const app = express();
app.use(bodyParser.urlencoded({extended: true}));
app.get("/", function(req, res) {
res.sendFile(__dirname + "/index.html");
});
app.post("/", function(req, res) {
const query = req.body.asinId;
const apiKey = "MY_API_KEY";
const url = "https://api.keepa.com/product?key=" + apiKey + "&domain=1&asin=" + query;
const options = {
methode: "GET",
headers: {
"Content-Type": "application/json;charset=UTF-8",
"Accept-Encoding":"gzip"
}
}
https.get(url,options,function(res) {
console.log(res.statusCode);
console.log(res.headers);
var data;
res.on("data", function(chunk){
if(data){
data = chunk;
} else {
data += chunk;
}
console.log(data);
});
});
res.send("server is running");
});
app.listen(3000, function() {
console.log("server is running on port 3000");
});
Have you tried using the built-in zlib module with gunzip()?
zlib.gunzip(data, (error, buff) => {
if (error != null) {
// An error occured while unzipping the .gz file.
} else {
// Use the buff which contains the unzipped JSON.
console.log(buff)
}
});
Full example with your code: https://www.napkin.io/n/7c6bc48d989b4727
well the output function was wrong .. the correct one below
https.get(url,options,function(response) {
response = decompressResponse(response);
console.log(res.statusCode);
console.log(res.headers);
let data = '';
response.on("data", function(chunk){
data += chunk;
});
response.on("end",function(){
console.log(data);
});
});

SyntaxError unexpected end of json input Node Js

I'm trying to retrieve data from Keepa using express js and HTTP module,
Keepa sending all data as gzip.
I was challenged to get the data properly and I got a previous error-Unexpected token in JSON at position 0,
So I have installed 'decompress-response' module which resolved this issue but now I'm getting half of the JSON data and then a new syntax error appears - unexpected end of json input Node Js
I'm trying to figure what am I missing here.. hope you can help me.
const express = require("express");
const https = require("https");
const decompressResponse = require("decompress-response");
const bodyParser = require("body-parser");
const app = express();
app.use(bodyParser.urlencoded({extended: true}));
app.get("/", function(req, res){
res.sendFile(__dirname + "/index.html");
});
app.post("/", function(req, res){
console.log(req.body.asinId);
const query = req.body.asinId;
const apiKey = "MY_API_KEY";
const url = "https://api.keepa.com/product?key="+ apiKey +"&domain=1&asin="+ query;
https.get(url, function(response){
response = decompressResponse(response);
console.log(response.statusCode);
console.log(response.headers);
var data;
response.on("data", function(chunk) {
if (!data) {
data = chunk;
} else {
data += chunk;
}
const asinData = JSON.parse(data);
console.log(asinData);
res.send();
});
});
});
Please try to print the response before "response = decompressResponse(response);". And let me know what you get there.

Node.js: Unable to compress response using express

I am using express version 4.13.4 and the following code for my app.js. I have tried changing the location of app.use(compression()) which did not show any effect. when I run the application I saw no evidence of compression in viewing the chrome dev tools response headers i.e it doesn't have the gzip content-encoding header.
I am new to node js.I want to gzip compress my response to browser. Please help me fix this issue.
var compression = require('compression')
var express = require('express');
var http = require('http');
var app = express();
app.use(compression());
var settings = {
UiServerPort: 8080,
ApiServerHost: "localhost",
ApiServerPort: 12121
};
app.use('/ui', express.static('ui'));
app.all('/api/*', function (req, res) {
var options = {
host: settings.ApiServerHost,
port: settings.ApiServerPort,
path: req.url.substring(4),
method: 'POST'
};
var requestData = '';
req.on('data', function (data) { requestData += data; });
req.on('end', function () {
var request = http.request(options, function (response) {
var responseData = '';
res.flush();
response.on('data', function (data) { responseData += data; });
response.on('end', function () {
res.statusCode = response.statusCode;
res.write(responseData);
res.end();
});
});
request.write(requestData);
request.end();
});
});
app.listen(settings.UiServerPort)
you saw " Vary Accept-Encoding " ?? if you don't use compression , this won't show. and I paste your code ,but It can't run.
Instead of
app.use(compression())
You should add this piece of code :
app.use(compression({filter: shouldCompress}))
function shouldCompress (req, res) {
if (req.headers['x-no-compression']) {
// don't compress responses with this request header
return false
}
// fallback to standard filter function
return compression.filter(req, res)
}
PS : it works for me.

TypeError: undefined is not a function in node js

I am trying to get data from server and show it on web page but
I am getting a error at end of file
response.end(body);
TypeError: undefined is not a function
i don't get what the error is please help
My Nodejs Code
var express=require("express");
var app=express();
var request = require('request');
var passport = require('passport')
var LocalStrategy = require('passport-local').Strategy;
var bodyParser = require("body-parser");
app.use(bodyParser.urlencoded({ extended: false }));
app.get('/service',function(req,res){
var id = req.query.name;
console.log('hi');
console.log(id);
request.post({
headers: {
//set header if needed
},
url : 'http://example.com:8080/rest/api/2/issue/id',
json : {
userName:'username',
password:'password'
}
},
function(error, response, body){
if(error) {
console.log(error);
} else {
response.end(body);
}
});
});
console.log('running');
app.listen(8082);
You want to use the res variable, which is the response object passed into your Express request handler:
res.end(body);
response is an object passed by the request library that you're using, which is a different thing.

Expressjs raw body

How can I access raw body of request object given to me by expressjs?
var express = require('./node_modules/express');
var app = express.createServer();
app.post('/', function(req, res)
{
console.log(req.body); //says 'undefined'
});
app.listen(80);
Something like this should work:
var express = require('./node_modules/express');
var app = express.createServer();
app.use (function(req, res, next) {
var data='';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.body = data;
next();
});
});
app.post('/', function(req, res)
{
console.log(req.body);
});
app.listen(80);
Using the bodyParser.text() middleware will put the text body in req.body.
app.use(bodyParser.text({type: '*/*'}));
If you want to limit processing the text body to certain routes or post content types, you can do that too.
app.use('/routes/to/save/text/body/*', bodyParser.text({type: 'text/plain'})); //this type is actually the default
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));
If you want a raw Buffer, you can use bodyParse.raw().
app.use(bodyParser.raw({type: '*/*'}));
Note: this answer was tested against node v0.12.7, express 4.13.2, and body-parser 1.13.3.
Put the following middleware before bodyParser middleware. It'll collect raw body data in request.rawBody and won't interfere with bodyParser.
app.use(function(req, res, next) {
var data = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
next();
});
});
app.use(express.bodyParser());
Default express does not buffer data unless you add middleware to do so. The simple solution is to follow the example in #Stewe's answer below, which would just concatenate all of the data yourself. e.g.
var concat = require('concat-stream');
app.use(function(req, res, next){
req.pipe(concat(function(data){
req.body = data;
next();
}));
});
The downside of this is that you have now moved all of the POST body content into RAM as a contiguous chunk, which may not be necessary. The other option, which is worth considering but depends on how much data you need to process in the post body, would be to process the data as a stream instead.
For example, with XML you could use an XML parser that supports parsing XML as it comes in as chunks. One such parser would be XML Stream. You do something like this:
var XmlStream = require('xml-stream');
app.post('/', function(req, res) {
req.setEncoding('utf8');
var xml = new XmlStream(req);
xml.on('updateElement: sometag', function(element) {
// DO some processing on the tag
});
xml.on('end', function() {
res.end();
});
});
app.use(bodyParser.json({
verify: function (req, res, buf, encoding) {
req.rawBody = buf;
}
}));
app.use(bodyParser.urlencoded({
extended: false,
verify: function (req, res, buf, encoding) {
req.rawBody = buf;
}
}));
So, it seems like Express's bodyParser only parses the incoming data, if the content-type is set to either of the following:
application/x-www-form-urlencoded
application/json
multipart/form-data
In all other cases, it does not even bother reading the data.
You can change line no. 92 of express/node_modules/connect/lib/middleware/bodyParser.js from
} else {
next();
}
To:
} else {
var data='';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
next();
});
}
And then, read req.rawBody from your code.
If you want the body as a buffer:
var rawParser = function(req, res, next) {
var chunks = [];
req.on('data', function(chunk) {
chunks.push(chunk)
});
req.on('end', function() {
req.body = Buffer.concat(chunks);
next();
});
}
or
var rawParser = bodyParser.raw({type: '*/*'});
and then:
app.put('/:name', rawParser, function(req, res) {
console.log('isBuffer:', Buffer.isBuffer(req.body));
})
or for all routes:
app.use(bodyParser.raw({type: '*/*'}));
It seems this has become a lot easier now!
The body-parser module is able to parse raw and text data now, which makes the task a one-liner:
app.use(bodyParser.text({type: 'text/plain'}))
OR
app.use(bodyParser.raw({type: 'application/binary'}))
Both lines simply fill the body property, so get the text with res.body.
bodyParser.text() will give you the UTF8 string while bodyParser.raw() will give you the raw data.
This is the full code for text/plain data:
var express = require('express')
var bodyParser = require('body-parser')
var app = express()
app.use(bodyParser.text({type: 'text/plain'}))
app.post('/', function (req, res, next) {
console.log('body:\n' + req.body)
res.json({msg: 'success!'})
next()
})
See here for the full documentation:
https://www.npmjs.com/package/body-parser
I used express 4.16 and body-parser 1.18
If you are having trouble with the above solutions interfering with normal post requests, something like this might help:
app.use (function(req, res, next) {
req.rawBody = '';
req.setEncoding('utf8');
req.on('data', function(chunk) { req.rawBody += chunk });
});
More info & source: https://github.com/visionmedia/express/issues/897#issuecomment-3314823
All the answers seems outdated, if anyone still struggling with this then express has built-in Express raw middeware.
This middleware is available in Express v4.16.0 onwards.
This is a built-in middleware function in Express. It parses incoming requests with JSON payloads and is based on body-parser.
var express = require("express");
var app = express();
app.use(express.raw({ type: "*/*" }))
app.post("/", (req, res) => {
// req.body = JSON.parse(req.body); // To parse JSON if needed (in-case)
console.log(req.body);
res.end();
});
app.listen(3000, (err) => {
if(!err) console.log("App running!!")
});
BE CAREFUL with those other answers as they will not play properly with bodyParser if you're looking to also support json, urlencoded, etc. To get it to work with bodyParser you should condition your handler to only register on the Content-Type header(s) you care about, just like bodyParser itself does.
To get the raw body content of a request with Content-Type: "text/xml" into req.rawBody you can do:
app.use(function(req, res, next) {
var contentType = req.headers['content-type'] || ''
, mime = contentType.split(';')[0];
if (mime != 'text/xml') {
return next();
}
var data = '';
req.setEncoding('utf8');
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
req.rawBody = data;
next();
});
});
When sending the request be sure to add this header:
'Content-Type': 'application/json'

Resources