nodejs node-http-proxy setup with cache - node.js

I'm trying to setup node-http-proxy module with caching
node-http-proxy module. I've managed to configure node-http-proxy to do what I need to do in terms of proxying calls, but I would like to find a way to cache some of these calls.
My current code is as follows (omitted some config bootstrapping):
var http = require('http');
var https = require('https');
var httpProxy = require('http-proxy');
var proxy = httpProxy.createProxyServer({});
var fs = require('fs');
var handler = function(req, res) {
proxy.web(req, res, {target: 'http://localhost:9000'});
};
var server = http.createServer(handler).listen(config.children.http.port, function() {
console.log('Listening on port %d', server.address().port);
});
var secure = https.createServer(config.children.https.options, handler).listen(config.children.https.port, function() {
console.log('Listening on port %d', secure.address().port);
});
Inside the handler function, I would like to be able to somehow capture what the proxy is reading from "target" and stream it into a fs.createWriteStream('/somepath') before piping it out to res. Then I would modify my function to do look to something along the line:
var handler = function(req, res) {
var path = '/somepath';
fs.exists(path, function(exists) {
if(exists) {
console.log('is file');
fs.createReadStream(path).pipe(res);
} else {
console.log('proxying');
// Here I need to find a way to write into path
proxy.web(req, res, {target: 'http://localhost:9000'});
}
});
};
Does anyone know how to do this ?

The answer to the question ended up being very simple:
var handler = function(req, res, next) {
var path = '/tmp/file';
fs.exists(path, function(exists) {
if(exists) {
fs.createReadStream(path).pipe(res);
} else {
proxy.on('proxyRes', function(proxyRes, req, res) {
proxyRes.pipe(fs.createWriteStream(path));
});
proxy.web(req, res, {target: 'http://localhost:9000'});
}
});
};

Related

Node.js HTTP Server Routing

below node.js code is using express to route to different urls, how can I do the same with http instead of express?
var express = require('express');
var app = express();
app.use(express.static('public'));
app.get('/', function (req, res) {
res.send('Welcome Home');
});
app.get('/tcs', function (req, res) {
res.send('HI RCSer');
});
// Handle 404 - Keep this as a last route
app.use(function(req, res, next) {
res.status(404);
res.send('404: File Not Found');
});
app.listen(8080, function () {
console.log('Example app listening on port 8080!');
});
Here's a quick example. Obviously, there are many ways of doing this, and this is probably not the most scalable and efficient way, but it will hopefully give you an idea.
const http = require('http');
const fs = require('fs');
const server = http.createServer((req, res) => {
req.on('error', err => {
console.error(err);
// Handle error...
res.statusCode = 400;
res.end('400: Bad Request');
return;
});
res.on('error', err => {
console.error(err);
// Handle error...
});
fs.readFile('./public' + req.url, (err, data) => {
if (err) {
if (req.url === '/' && req.method === 'GET') {
res.end('Welcome Home');
} else if (req.url === '/tcs' && req.method === 'GET') {
res.end('HI RCSer');
} else {
res.statusCode = 404;
res.end('404: File Not Found');
}
} else {
// NOTE: The file name could be parsed to determine the
// appropriate data type to return. This is just a quick
// example.
res.setHeader('Content-Type', 'application/octet-stream');
res.end(data);
}
});
});
server.listen(8080, () => {
console.log('Example app listening on port 8080!');
});
Try the code below . It is a pretty basic example
var http = require('http');
//create a server object:
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'}); // http header
var url = req.url;
if(url ==='/about'){
res.write('<h1>about us page<h1>'); //write a response
res.end(); //end the response
}else if(url ==='/contact'){
res.write('<h1>contact us page<h1>'); //write a response
res.end(); //end the response
}else{
res.write('<h1>Hello World!<h1>'); //write a response
res.end(); //end the response
}
}).listen(3000, function(){
console.log("server start at port 3000"); //the server object listens on port 3000
});
express can also be used by http directly.
From: https://expressjs.com/en/4x/api.html#app.listen
The app returned by express() is in fact a JavaScript Function, designed to be passed to Node’s HTTP servers as a callback to handle requests.
var http = require('http')
var express = require('express')
var app = express()
http.createServer(app).listen(80)
With this, you can still make use of express for routing, while keeping native http support.

httpdispatcher returning no or blank response

I have the following simple code for a nodejs server....
var http = require('http');
var port = 1337;
var dispatcher = require('httpdispatcher');
dispatcher.setStaticDirname(__dirname);
dispatcher.setStatic('');
dispatcher.onGet("/page1", function (req, res) {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.end('Page One');
});
var server = http.createServer().listen(port);
server.on('request', function (req, res) {
console.log('GOT');
dispatcher.dispatch(req, res);
});
console.log('Listening on port %s', port);
when I goto http://localhost:1337/index.html it is showing up correctly but when I do http://localhost:1337/page1 nothing happens...how can I get it to function properly...
You need to define your custom dispatcher events, in this case "/page1", before setting the static dispatcher. Otherwise static hogs every possible path remaining to check against the file system.
dispatcher.onGet("/page1", function (req, res) {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.end('Page One');
});
dispatcher.setStaticDirname(__dirname);
dispatcher.setStatic('');
Your revised code will look like this:
var http = require('http');
var port = 1337;
var dispatcher = require('httpdispatcher');
dispatcher.onGet("/page1", function (req, res) {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.end('Page One');
});
dispatcher.setStaticDirname(__dirname);
dispatcher.setStatic('');
var server = http.createServer().listen(port);
server.on('request', function (req, res) {
console.log('GOT');
dispatcher.dispatch(req, res);
});
console.log('Listening on port %s', port);
Tested and works

NODE.JS: how to proxy to a host with the variable target DEPENDING on the data received in the req?

I need to use the node-http-proxy library to proxy to a host with the variable target DEPENDING on the data received in the req, let's suppose that I need to find a tag and basing on this tag route to a host. My code is:
var http = require('http'),
httpProxy = require('http-proxy'),
proxy = httpProxy.createProxyServer({});
var miTarget='web:CountryName';
var inicio = '<'+miTarget+'>';
var fin = '</'+miTarget+'>';
var server = http.createServer(function(req, res) {
var miTag = '';
req.on('data', function (data) {
miTag = data.toString().split(inicio)[1].split(fin)[0];
});
req.on('end', function () {
console.log('miTag!!!!!!:'+miTag);
if(miTag=='Portugal') {
proxy.web(req, res, {target: 'http://www.webservicex.net'});
}
else {
proxy.web(req, res, {target: 'http://localhost:1337'});
}
});
});
console.log("listening on port 80")
server.listen(80);
This code is not working... Can anyone help me to sort out this problem?
The most important for me is to execute thie proxy.web() AFTER the data is received miTag in the req: req.on('data', function (data) {}
I did something similar once for a little personal project. You can use res.locals as a safe place to store per-request context (much safer than using a global).
app.use('/proxy', function(req, res, next) {
// Do whatever you need to do to create targetUrl (a URL object)
var targetUrl = whatever(req);
// Save the targetUrl path so we can access it in the proxyReq handler
res.locals.path = targetUrl.path;
// Initiate the proxy
var options = {
prependPath: false,
target: targetUrl.format(),
changeOrigin: true,
hostRewrite: targetUrl.host,
protocolRewrite: targetUrl.protocol.replace(/:$/, '')
};
proxy.web(req, res, options);
});
Then setup a proxyReq handler that uses the saved targetUrl path.
proxy.on('proxyReq', function(proxyReq, req, res, options) {
if (res.locals.path) {
proxyReq.path = res.locals.path;
}
});

How to configure express js 4 to serve some pages in http and others in https?

Is there any way to configure a node js application with express js 4 to serve some pages under http protocol and other, those which need more security, in https?
I describe my problem: I'm developing a shop online and I want to display certain pages, like the products list or the product detail views under http, and others which I think need more security, like login or the shopping cart views, under https protocol.
I have tried the express-force-ssl module, but it isn't working. The following code snippet is not from my app (which is too dirty) it is just an example which alos doesn't work for me:
var express = require('express');
var forceSSL = require('express-force-ssl');
var fs = require('fs');
var http = require('http');
var https = require('https');
var ssl_options = {
key: fs.readFileSync('./server-private-key.pem'),
cert: fs.readFileSync('./server-certificate.pem'),
ca: fs.readFileSync('./server-certificate-signing-request.pem')
};
var app = express();
var server = http.createServer(app);
var secureServer = https.createServer(ssl_options, app);
app.use(forceSSL);
app.get('/', function (req, res, next) {
res.send('Hello')
});
app.get('/user/:name', function (req, res, next) {
var user = req.params.name;
res.send('Hello ' + user + '')
});
app.get('/login', forceSSL, function (req, res, next) {
res.send('Hello<br/>Goodbye')
});
app.get('/logout', forceSSL, function (req, res, next) {
res.send('Hello')
});
secureServer.listen(443)
server.listen(8085)
console.log('server started');
The result is that when I launch the application, with url http://localhost:8085, the server automatically redirects it to https://localhost and serves all pages in https protocol.
What I want is to start on http://localhost:8085, navigate to http://localhost/user/userA, then from it go to https://localhost/login and, if click on "Hello" link, I would like to be redirected to http://localhost:8085.
Is there any missing code to get the behavior I want or even any other way to reach it without express-force-ssl module?
I have asked to the author of express-force-ssl module and he has told me that the redirect behavior works as expected. Here is the post.
But diving a little more in its code I've created a custom plugin to solve my problem. Here is the code:
var parseUrl = require('url').parse;
var isSecure = function (req) {
if (req.secure) {
return true;
}
else if (
req.get('X-Forwarded-Proto') &&
req.get('X-Forwarded-Proto').toLowerCase &&
req.get('X-Forwarded-Proto').toLowerCase() === 'https') {
return true;
}
return false;
};
exports = module.exports = function (req, res, next) {
if (isSecure(req)) {
if (req.method === "GET") {
var httpPort = req.app.get('httpPort') || 80;
var fullUrl = parseUrl(req.protocol + '://' + req.header('Host') + req.originalUrl);
res.redirect('http://' + fullUrl.hostname + ':' + httpPort + req.originalUrl);
}
else {
next();
}
}
else {
next();
}
};
It's very similar to force-ssl file but here we manage the opposite action, i.e., here I redirect to http when a route is forced to it. So it's needed to add the function to every route we want to see under http protocol:
var express = require('express');
var forceSSL = require('express-force-ssl');
var fs = require('fs');
var http = require('http');
var https = require('https');
var useHttp = require('./useHttp');
var ssl_options = {
key: fs.readFileSync('./server-private-key.pem'),
cert: fs.readFileSync('./server-certificate.pem')
};
var app = express();
var server = http.createServer(app);
var secureServer = https.createServer(ssl_options, app);
app.get('/', useHttp, function (req, res, next) {
res.send('Hello')
});
app.get('/user/:name', useHttp, function (req, res, next) {
var user = req.params.name;
res.send('Hello ' + user + '')
});
app.get('/login', forceSSL, function (req, res, next) {
res.send('Hello<br/>Goodbye')
});
app.get('/logout', forceSSL, function (req, res, next) {
res.send('Hello')
});
app.set('httpsPort', 9090);
app.set('httpPort', 8085);
secureServer.listen(9090)
server.listen(8085)
console.log('server started');
As you can see I need now to specify in all routes which protocol use: useHttp for http or forceSSL for https.
Although I'm not comfortable at all with this solution because I have to specify in all routes which kind of protocol I want. But at least it works. So I would be very pleased if someone finds any other solution, for isntance, adding in middleware layer a function to manage all http requests and just redirect to https when it is specified with forceSSL. By the moment, this works.

No response using express proxy route

I've written a small proxy with nodejs, express and htt-proxy. It works well for serving local files but fails when it comes to proxy to external api:
var express = require('express'),
app = express.createServer(),
httpProxy = require('http-proxy');
app.use(express.bodyParser());
app.listen(process.env.PORT || 1235);
var proxy = new httpProxy.RoutingProxy();
app.get('/', function(req, res) {
res.sendfile(__dirname + '/index.html');
});
app.get('/js/*', function(req, res) {
res.sendfile(__dirname + req.url);
});
app.get('/css/*', function(req, res) {
res.sendfile(__dirname + req.url);
});
app.all('/*', function(req, res) {
req.url = 'v1/public/yql?q=show%20tables&format=json&callback=';
proxy.proxyRequest(req, res, {
host: 'query.yahooapis.com', //yahoo is just an example to verify its not the apis fault
port: 8080
});
});
The problem is that there is no response from the yahoo api, maybe there is an response but i dont came up in the browser.
Even simpler with pipe and request-Package
var request = require('request');
app.use('/api', function(req, res) {
var url = apiUrl + req.url;
req.pipe(request(url)).pipe(res);
});
It pipes the whole request to the API and pipes the response back to the requestor. This also handles POST/PUT/DELETE and all other requests \o/
If you also care about query string you should pipe it as well
req.pipe(request({ qs:req.query, uri: url })).pipe(res);
Maybe your code is different when you're testing, but I'm querying the same URL as in your code sample using the following:
http://query.yahooapis.com:8080/v1/public/yql?q=show%20tables&format=json&callback=
and I get nothing back. My guess is you want to change port to 80 (from 8080) -- it works when I change it like so:
http://query.yahooapis.com:80/v1/public/yql?q=show%20tables&format=json&callback=
So that means it should be:
proxy.proxyRequest(req, res, {
host: 'query.yahooapis.com', //yahoo is just an example to verify its not the apis fault
port: 80
});
Maybe I use http-proxy in a wrong way. Using restler does what I want:
var express = require('express'),
app = express.createServer(),
restler = require('restler');
app.use(express.bodyParser());
app.listen( 1234);
app.get('/', function(req, res) {
console.log(__dirname + '/index.html')
res.sendfile(__dirname + '/index.html');
});
app.get('/js/*', function(req, res) {
res.sendfile(__dirname + req.url);
});
app.get('/css/*', function(req, res) {
res.sendfile(__dirname + req.url);
});
app.all('/*', function(req, res) {
restler.get('http://myUrl.com:80/app_test.php/api' + req.url, {
}).on('complete', function (data) {
console.log(data)
res.json(data)
});
});
I ended up using http-proxy-middleware.
The code looks something like this:
var express = require("express");
var proxy = require("http-proxy-middleware");
const theProxy = proxy({
target: "query.yahooapis.com",
changeOrigin: true,
});
app.use("/", theProxy);
app.listen(process.env.PORT || 3002);
That's what I've been using for a while. Can handle both JSON and binary requests.
app.use('/api', (req, res, next) => {
const redirectUrl = config.api_server + req.url.slice(1);
const redirectedRequest = request({
url: redirectUrl,
method: req.method,
body: req.readable ? undefined : req.body,
json: req.readable ? false : true,
qs: req.query,
// Pass redirect back to the browser
followRedirect: false
});
if (req.readable) {
// Handles all the streamable data (e.g. image uploads)
req.pipe(redirectedRequest).pipe(res);
} else {
// Handles everything else
redirectedRequest.pipe(res);
}
});

Resources