I have a small proxy for certain requests in Express. Using the request library, I have fairly concise code:
app.use('/api', function(req, res) {
var url = rewriteUrl(req.url);
var newReq = request(url, function(error) {
if (error) {
logError(error);
}
});
req.pipe(newReq).pipe(res);
});
My problem is that the response from the API server contains a bunch of unwanted headers that I want to remove. How can I remove the headers from the response of newReq before piping it to res?
mscdex's answer did work for me, but I found a way that I think is slightly cleaner. In my original code, I had this line:
req.pipe(newReq).pipe(res);
I replaced that with these lines:
req.pipe(newReq).on('response', function(res) {
delete res.headers['user-agent'];
// ...
}).pipe(res);
With the request module, there currently isn't a way (AFAIK) to have a callback and not buffer the server response. So here is how you might do it with the built-in http.request:
app.use('/api', function(req, res) {
var url = rewriteUrl(req.url);
var newReq = http.request(url, function(newRes) {
var headers = newRes.headers;
// modify `headers` here ...
res.writeHead(newRes.statusCode, headers);
newRes.pipe(res);
}).on('error', function(err) {
res.statusCode = 500;
res.end();
});
req.pipe(newReq);
});
It is easy with request.
request("https://example.com/image.png")
.on("response", remoteRes => {
// You can add/remove/modify headers here
remoteRes.headers["content-disposition"] = "attachment; filename=awesome.png";
})
.pipe(res);
There is more elegant way to modify/remove headers by setting a pipe filter as follows:
const req = request.get(url);
req.pipefilter = function(response, dest) {
// remove headers
for(const h in response.headers) {
dest.removeHeader(h);
}
// or modify
dest.setHeader('Content-Type', 'text/html')
}
req.pipe(resp)
Related
We are using navigator.sendBeacon function to send data to Koa server, in which we are using bodyparser.
If we not wrapped data into form then by default this function send data as request payload. How I can able to access this data on Koa server?
Example -
navigator.sendBeacon('http://localhost:3000/cookies/', 'test=payload')
At server, request body is blank.
Considering that
Koa does not parse request body, so you need to use either koa-bodyparser or koa-body,
koa-bodyparser by default has only json and form parsing enabled,
From your screenshot, it is clear that navigator.sendBeacon set the Content-Type to text,
You need to change the Koa server code, so that it parses text data.
Example:
const Koa = require('koa'),
bodyParser = require('koa-bodyparser'),
app = (module.exports = new Koa());
app.use(bodyParser({ enableTypes: ['json', 'text'] }));
app.use(async (ctx) => {
// ctx.request.body should contain the parsed data.
console.log('Received data:', ctx.request.body);
ctx.body = ctx.request.body;
});
if (!module.parent) app.listen(3000);
Tested with
koa 2.7.0,
koa-bodyparser 4.2.1.
Although koa doesn't parse request body and for some reason you don't want to use koa-bodyparser you can still use the raw http to collect the body from request object.
app.use(async (ctx) => {
try {
// notice that I'm wrapping event emitter in a `promise`
ctx.body = await new Promise((resolve, reject) => {
let data = '';
// this is same as your raw `http.request.on('data', () => ()`
ctx.req.on('data', chunk => {
data += chunk;
};
ctx.req.on('error', err => {
reject(err);
};
ctx.req.on('end', () => {
resolve(data);
};
});
} catch (e) {
console.error(e);
}
});
I have an app and I would like to use it as a proxy. Code looks like this
I am using request npm.
app.all('*', function(req, res){
console.log(req.url); //lower case
console.log(req.url.substr(0, 5));
var alteredRequest;
if (req.url.substr(0,5) == '/pro/') {
var requestedURL = req.url.substr(5);
console.log(requestedURL);
if( (requestedURL.substr(0, 7) != 'http://') || (requestedURL.substr(0, 8) != 'https://') ){
alteredRequest = "http://" + requestedURL;
}
request(alteredRequest, function(error, response, body){
console.log('error: ', error); // Print the error if one occurred
console.log('statusCode: ', response && response.statusCode); // Print the response status code if a response was received
res.send(response);
})
}
})
When I call res.send(response) at the end I get the response as a body, and all the headers exc are taken to the html body logged to the screen. Rather than that, I would like to send it as a real response with all the headers and cookies. How do I achieve it? thank you for helping.
To do more demonstration, the response (from google.com) starts like this, but it's on the screen! not the real headers and statusCode:
{"statusCode":200,"body":"<!doctype html><html itemscope=\"\" itemtype=\"http://schema.org/WebPage\" lang=\"tr\"><head><meta content=\"text/html; charset=UTF-8\" http-equiv=\"Content-Type\"><meta content=\"/images/branding/googleg/1x/googleg_standard_color_128dp.png\" itemprop=\"image\"><title>Google</title><script nonce=\"fzz0lJtfXqp5SuC6mvodsw==\">(function(){window.google=
it's a really bad way to do redirects. You should be using
res.redirect and req.protocol to do that
if (req.url.substr(0,5) == '/pro/') is perhaps the most hideous way of catching a route I've ever seen. This is what app.get('/', ...) is designed for.
But to answer your question, here's a working example
const express = require("express");
const request = require("request");
const app = express();
app.all('*', (req, res) => {
const newUrl = 'http://www.google.com'; //replace with your url altering code
return request(newUrl)
.on('response', function(response) {
console.log(response.statusCode) // 200
console.log(response.headers['content-type'])
})
.on('error', function(err) {
console.log(err)
})
.pipe(res);
});
app.listen(9000);
I need to modify the request body asynchronously. Something along the lines of this:
proxy.on('proxyReq', function(proxyReq, req, res, options) {
if(req.body) {
new Promise(function(resolve){
setTimeout(function() { // wait for the db to return
'use strict';
req.body.text += 'test';
let bodyData = JSON.stringify(req.body);
proxyReq.setHeader('Content-Type','application/json');
proxyReq.setHeader('Content-Length', Buffer.byteLength(bodyData));
// stream the content
proxyReq.write(bodyData);
resolve();
},1);
});
}
});
When I run this I get the error saying cannot modfiy headers once they have been set. Which makes sense.
How can I halt the sending of the request until I'm ready? I've looked at removing various listeners from proxyReq without success..
By looking at the source code #-) it seems like it's not really possible because the proxyReq event is sent and then the code moves on.
If it would instead wait for a promise, it would be possible (if you'd return that promise as well).
A minimal fork on this lib could be for example:
// Enable developers to modify the proxyReq before headers are sent
proxyReq.on('socket', function(socket) {
if(server) { server.emit('proxyReq', proxyReq, req, res, options); }
});
(proxyReq.proxyWait || Promise.resolve())
.then( ... // rest of the code inside the callback
And then
proxy.on('proxyReq', function(proxyReq, req, res, options) {
if(req.body) {
proxyReq.proxyWait = new Promise(function(resolve){
setTimeout(function() { ...
But depending on your use case, there might be other solutions as well. For example, consider if it's really necessary that you use this proxy library. It You could alternatively use http directly, where you have all the control on the events and callbacks.
You can set selfHandleResponse: true inside the HttpProxy.createProxyServer. This then allows (and forces) you to handle the proxyRes manually!
const proxy = HttpProxy.createProxyServer({selfHandleResponse: true});
proxy.on('proxyRes', async (proxyReq, req, res, options) => {
if (proxyReq.statusCode === 404) {
req.logger.debug('Proxy Request Returned 404');
const something = await doSomething(proxyReq);
return res.json(something);
}
return x;// return original proxy response
});
I came here looking for the solution to a slightly different problem: Modifying the request headers (not body) before proxying.
I post this here in case that it is helpful to others. And maybe the code can be adapted to also modify the request body.
const http = require('http');
const httpProxy = require('http-proxy');
var proxy = httpProxy.createProxyServer({});
var server = http.createServer(function(req, res) {
console.log(`${req.url} - sleeping 1s...`);
setTimeout(() => {
console.log(`${req.url} - processing request`);
req.headers['x-example-req-async'] = '456';
proxy.web(req, res, {
target: 'http://127.0.0.1:80'
});
}, 1000);
});
server.listen(5050);
I am trying to implement the ._read function of a readable stream, a problem happens when ._read is called and there isn't data, the documentation says that I can push('') until more data comes, and I should only return false when the stream will never have more data.
https://nodejs.org/api/stream.html#stream_readable_read_size_1
But it also says that if I need to do that then something is terribly wrong with my design.
https://nodejs.org/api/stream.html#stream_stream_push
But I can't find an alternative to that.
code:
var http = require('http');
var https = require('https');
var Readable = require('stream').Readable;
var router = require('express').Router();
var buffer = [];
router.post('/', function(clientRequest, clientResponse) {
var delayedMSStream = new Readable;
delayedMSStream._read = function() {
var a=buffer.shift();
if(typeof a === 'undefined'){
this.push('');
return true;
}
else {
this.push(a);
if(a===null) {
return false;
}
return true;
}
};
//I need to get a url from example.com
https.request({hostname:'example.com'}, function(exampleResponse){
data='';
exampleResponse.on('data',function(chunk){data+=chunk});
exampleResponse.on('end',function(){
var MSRequestOptions = {hostname: data, method: 'POST'};
var MSRequest = https.request(MSRequestOptions, function(MSResponse){
MSResponse.on('end', function () {
console.log("MSResponse.on(end)");//>>>
});//end MSResponse.on(end)
}); //end MSRequest
delayedMSStream.pipe(MSRequest);
});
});
clientRequest.on('data', function (chunk) {
buffer.push(chunk);
});
clientRequest.on('end', function () {//when done streaming audio
buffer.push(null);
});
});//end router.post('/')
explanation:
client sends a POST request streaming audio to my server, my server requests a url from example.com, when example.com responds with the url, my server streams the audio to it.
What's a smarter way to do it?
So if I undertstand the code correctly, you:
receive a request,
make your own request to a remote endpoint and fetch a URL
make a new request to that URL and pipe that to original response.
There are ways to do this other then yours, and even your way would look cleaner to me if you just improve the naming a bit. Also, splitting the huge request into a few functions with smaller responsibility scopes might help.
I would make the endpoint this way:
let http = require('http');
let https = require('https');
let Readable = require('stream').Readable;
let router = require('express').Router();
let buffer = [];
/**
* Gets some data from a remote host. Calls back when done.
* We cannot pipe this directly into your stream chain as we need the complete data to get the end result.
*/
function getHostname(cb) {
https.request({
hostname: 'example.com'
}, function(response) {
let data = '';
response.on('error', err => cb(err)); // shortened for brewity
response.on('data', function(chunk) {
data = data + chunk;
});
response.on('end', function() {
// we're done here.
cb(null, data.toString());
});
});
}
router.post('/', function(request, response) {
// first let's get that url.
getHostname(function(err, hostname) {
if (err) { return response.status(500).end(); }
// now make that other request which we can stream.
https.request({
hostname: hostname,
method: 'POST'
}, function(dataStream) {
dataStream.pipe(response);
});
});
});
Now, as said in the comments, with streams2, you don't have to manage your streams. With node versions pre 0.10 you have had to listen to 'read', 'data' etc events, with newer node versions, it's handled. Furthermore, you don't even need it here, streams are smart enough to handle backpressure on their own.
in my project I have to do a request to upcDatabase.com, I amworking with nodeJS, I get the answer from the server but I do not how to extractthe data this are the important part of my code:
module.exports = function (http,upc){
var upc_ApiKey = "XXX",
url = "http://upcdatabase.org/api/json/"+upc_ApiKey+'/'+upc;
http.get(url,function(resp){
// my code to read the response
I do not get any error, but the resp is a big Json and I do not know where to find the data
I would recommend you using the superagent module. It provides much more functionality than the built-in http request and it will automatically parse the response for you.
request
.get(url)
.end(function(err, res) {
if (res.ok) {
// Her ethe res object will be already parsed. For example if
// the server returns Content-Type: application/json
// res will be a javascript object that you can query for the properties
console.log(res);
} else {
// oops, some error occurred with the request
// you can check the err parameter or the res.text
}
});
You could achieve the same with the built-in http module but with much more code:
var opts = url.parse(url);
opts.method = "GET";
var req = http.request(opts, function (res) {
var result = "";
res.setEncoding("utf8");
res.on("data", function (data) {
result += data;
});
if (res.statusCode === 200) {
res.on("end", function () {
// Here you could use the result object
// If it is a JSON object you might need to JSON.parse the string
// in order to get an easy to use js object
});
} else {
// The server didn't return 200 status code
}
});
req.on("error", function (err) {
// Some serious error occurred during the request
});
// This will send the actual request
req.end();