I am trying to create a http caching proxy server using node.js , where i could forward to any webpages and cached them on my local disk !
The following is my first attempt code :
var http = require('http'),
url = require('url'),
sys = require('url');
var fs = require('fs');
var port = "9010";
// function notFound
function notFound(response){
response.writeHead(404, "text/plain");
response.end("404 : File not Found");
}
//create simple http server with browser requet and browser response
http.createServer(function(b_request, b_response){
//Parse the browser request'url
var b_url = url.parse(b_request.url, true);
if(!b_url.query || !b_url.query.url) return notFound(b_response);
//Read and parse url parameter (/?url=p_url)
var p_url = url.parse(b_url.query.url);
//Initialize Http client
var p_client = http.createClient(p_url.port || 80, p_url.hostname);
//Send request
var p_request = p_client.request('GET', p_url.pathname || "/", {
host: p_url.hostname
});
p_request.end();
//Listen for response
p_request.addListener('response', function(p_response){
//Pass through headers
b_response.writeHead(p_response.statusCode, p_response.headers);
//Pass through data
p_response.addListener('data', function(chunk){
b_response.write(chunk);
});
//End request
p_response.addListener('end', function(){
b_response.end();
});
});
}).listen(port);
console.log("Server running at http://127.0.0.1:" +port + "/");
i want to use any cached library for my app suchas : Node-static(https://github.com/cloudhead/node-static), Static cache, ....
if website that i visited is working fine , my app will forward to it . If not my app will get and return me data that cached on my disk .
is there any solutions for this works ?
thank !
Related
Here is what i want:
I have an url "10.0.0.1/0000000/abctest/filedirectory/filename",
and with this url, I can read the file accordingly
Thus, i create a server with "http.createServer" and binded at 8001 port, and set nginx that whenever it read /0000000/abctest/, it will redirect to port 8001
After i added the nginx reverse proxy, I try to write my server code, and here is my http server code
var port = 8001;
var http = require('http');
var fs = require('fs');
var DefaultFileDir = '/home/ubuntu/Server/Proj/static';
var UrlPrefix = "/0000000/abctest"
var hostname = "127.0.0.1"
//var host = hostname+UrlPrefix
var server= http.createServer(function(req,res){
var url = req.url;
console.log(url);
var file = DefaultFileDir + url;
fs.readFile(file, function(err,data){
if(err){
console.log(err);
res.writeHeader(404,{
'content-type' : 'text/html;charset="utf-8"'
});
res.write('<h1>404</h1>');
res.end();
}else{
if(url.substr(url.length-3, 3) == ".gz"){
res.writeHeader(200,{
'Content-Encoding' : 'gzip',
'content-type' : 'application/octet-stream'
});
}else{
res.writeHeader(200,{
'content-type' : 'text/html;charset="utf-8"'
});
}
res.write(data);
res.end();
}
});
}).listen(port,hostname);
console.log("Static file server running at\n => http://localhost:" + port + "/\nCTRL + C to shutdown");
And the problem come out:
it dont know how to ignore the url prefix "/0000000/abctest"
Error: ENOENT: no such file or directory, open '/home/ubuntu/xxxxx/static/0000000/abctest/folder001/index.html'
However my actual file path shall be '/home/ubuntu/xxxxx/static/folder001/index.html'
is that http.createserver cannot ignore the url prefix? May i seek for some hints from that.
http://nginx.org/en/docs/http/ngx_http_rewrite_module.html
I finally bypass the problem by rewriting the url with the use of nginx reverse proxy
I'm just starting with node.js and express and I'm doing a simple HTTPS server. I've been working with nginx for some time and when I make an HTTP request to an HTTPS endpoint I get a "400 Bad Request" error. However, when using node.js the request never finishes.
How can I intercept an HTTP request in Express to be able to generate the "400 Bad Request" response?
This is my code:
var express = require('express');
var https = require('https');
var fs = require('fs');
var port = process.env.PORT || 8080;
var tls_options = {
key: fs.readFileSync('certs/server.key'),
cert: fs.readFileSync('certs/server.crt'),
ca: fs.readFileSync('certs/ca.crt'),
requestCert: true,
};
var app = express();
var router = express.Router();
router.get('/', function(req, res) {
res.json({ message: 'Checkpoint!!' });
});
app.use('/', router);
var secureServer = https.createServer(tls_options, app);
secureServer.listen(port);
console.log('Listening on port ' + port);
Until now the only thing I've been able to use is getting a 'connection' event every time a request arrives to the server:
secureServer.on('connection', function (stream) {
console.log('someone connected!');
});
Done. In fact, an HTTP request to an HTTPS socket ends after the default 120secs TLS handsahke timeout. This way I can end the request without waiting. I include the solution I used just for future references if anything needs the same functionality.
var secureServer = https.createServer(options, app);
secureServer.on('connection', function(socket) {
socket.on('data', function(data) {
var first_line = data.toString().split('\r\n')[0];
var pattern = /\bhttp\/1\.[01]$\b/i;
if (pattern.test(first_line)) {
var headers = {};
headers['Date'] = new Date().toUTCString();
headers['Connection'] = 'close';
var headers_string = '';
for (var name in headers) {
headers_string = headers_string + '\r\n' + name + ': ' + headers[name];
}
socket.end('HTTP/1.1 400 Bad Request' + headers_string);
}
});
There isn't a way of starting both HTTP and HTTPS servers on the same port. What most people do is either:
Start two servers (one HTTP and one HTTPS) on different ports, and redirect the HTTP traffic to HTTPS. Using Express it would mean the additional code:
// create two ports, one for HTTP and one for HTTPS
var port = process.env.PORT || 8080;
var httpsPort = 8081;
// redirect all HTTP requests to HTTPS
app.use(function(req, res, next) {
var hostname;
if (!req.secure) {
hostname = req.get("host").split(":")[0];
return res.redirect(["https://", hostname, ":", httpsPort, req.url].join(""));
}
next();
});
app.listen(port); // listen on HTTP
https.createServer(tls_options, app).listen(httpsPort); // listen on HTTPS
Or they use nginx or apache to handle outside connections (both HTTP and HTTPS) and redirect traffic to the Node server (which can then just run on HTTP).
I am learning Node.JS and this is the most commonly available example of server by Node.JS
// Load the http module to create an http server.
var http = require('http');
// Configure our HTTP server to respond with Hello World to all requests.
var server = http.createServer(function (request, response) {
// var name=request.getParameter('name');
// console.log(name);
console.log('res: ' + JSON.stringify(response.body));
response.writeHead(200, {"Content-Type": "text/plain"});
response.end("Hello World\n");
});
// Listen on port 8000, IP defaults to 127.0.0.1
server.listen(8000);
Now when I am executing this from console it works fine, and from browser also it works fine, by hitting the URL: localhost:8000
But now I also want to send some parameters to this server, so I tried localhost:8000/?name=John and few more URL's but none of them work, Can anyone help me?
Thanks in advance!!
try:
var url = require('url');
var name = url.parse(request.url, true).query['name'];
Node's HTTP API is rather low-level compared to other frameworks/environments that you might be familiar with, so pleasantries like a getParameter() method don't exist out of the box.
You can get the query-string from the request's url, which you can then parse:
var http = require('http');
var url = require('url');
var server = http.createServer(function (request, response) {
var parsedUrl = url.parse(request.url, true);
var query = parsedUrl.query;
console.log(query.name);
// ...
});
I have a very simple javascript using Node. The purpose of the script is to:
Open a listening socket
Add a handler for URL /test with HTTP function GET
When /test is requested another URL located on another external web site should be fetched. This web site is encoded with ISO-8859-1
The data returned from the external website should be packaged in a JSON structure and returned to the requesting client encoded with UTF-8
So far I have created the following code:
var Buffer = require('buffer').Buffer;
var iconv = require('iconv-lite');
var urllib = require('url');
var restify = require('restify');
var server = restify.createServer();
server.use(restify.bodyParser());
server.get('/test', test);
server.listen(8080, function() {
console.log('%s listening at %s', server.name, server.url);
});
function test(req, res, next) {
console.log('TEST');
var httpClient = restify.createStringClient({ url: "http://dl.dropboxusercontent.com" });
httpClient.get("/u/815962/iso-8859-1.html", function(cerr, creq, cres, cdata) {
cdata = iconv.decode(cdata, 'iso-8859-1');
res.send(200, {"Data": cdata});
});
}
I have set up a test document used in the code above. The test document is in ISO-8859-1 encoding and has the national letters "ÅÄÖåäö" inside it. When returned to the client, if read like UTF-8 I receive "ýýýýýý"
It really seem that this is a bug in Restify. The following example shows different results using Restify and Request lib:
var request = require('request');
var iconv = require('iconv');
var restify = require('restify');
var ic = new iconv.Iconv('iso-8859-1', 'utf-8');
request.get({ url: 'http://dl.dropboxusercontent.com/u/815962/iso-8859-1.html', encoding: null, }, function(err, res, body) {
var buf = ic.convert(body);
var utf8String = buf.toString('utf-8');
console.log(utf8String);
});
var httpClient = restify.createStringClient({ url: "http://dl.dropboxusercontent.com" });
httpClient.get("/u/815962/iso-8859-1.html", function(cerr, creq, cres, cdata) {
var buf = ic.convert(cdata);
var utf8String = buf.toString('utf-8');
console.log(utf8String);
});
I have browsed the code of Restify on github trying to find the issue, but I can't
Trying to learn more about node.js by making a simple http proxy server. The use scenario is simple: user -> proxy -> server -> proxy -> user
The following code works until the last step. Couldn't find way to pipe connector's output back to the user.
#!/usr/bin/env node
var
url = require('url'),
http = require('http'),
acceptor = http.createServer().listen(3128);
acceptor.on('request', function(request, response) {
console.log('request ' + request.url);
request.pause();
var options = url.parse(request.url);
options.headers = request.headers;
options.method = request.method;
options.agent = false;
var connector = http.request(options);
request.pipe(connector);
request.resume();
// connector.pipe(response); // doesn't work
// connector.pipe(request); // doesn't work either
});
Using tcpflow I see the incoming request from the browser, then the outgoing proxy request, then the server response back to the proxy. Somehow i couldn't manage to retransmit the response back to the browser.
What is the proper way to implement this logic with pipes?
you dont have to 'pause', just 'pipe' is ok
var connector = http.request(options, function(res) {
res.pipe(response, {end:true});//tell 'response' end=true
});
request.pipe(connector, {end:true});
http request will not finish until you tell it is 'end';
OK. Got it.
UPDATE: NB! As reported in the comments, this example doesn't work anymore. Most probably due to the Streams2 API change (node 0.9+)
Piping back to the client has to happen inside connector's callback as follows:
#!/usr/bin/env node
var
url = require('url'),
http = require('http'),
acceptor = http.createServer().listen(3128);
acceptor.on('request', function(request, response) {
console.log('request ' + request.url);
request.pause();
var options = url.parse(request.url);
options.headers = request.headers;
options.method = request.method;
options.agent = false;
var connector = http.request(options, function(serverResponse) {
serverResponse.pause();
response.writeHeader(serverResponse.statusCode, serverResponse.headers);
serverResponse.pipe(response);
serverResponse.resume();
});
request.pipe(connector);
request.resume();
});
I used the examples from this post to proxy http/s requests. Faced with the problem that cookies were lost somewhere.
So to fix that you need to handle headers from the proxy response.
Below the working example:
const http = require('http');
const acceptor = http.createServer().listen(3128);
acceptor.on('request', function(request, response) {
const req = service.request(options, function(res) {
response.writeHead(res.statusCode, res.headers);
return res.pipe(response, {end: true});
});
request.pipe(req, {end: true});
});