I am trying to send http (GET and POST requests) to various sites to retrieve response header/body information, and need to vary whether I am using a proxy or not. After previously failing to use restler library for this purpose, I have switched to node 'http' and 'https' libraries to perform these tasks. Code as follows:
/******/
//Http client libraries
var http = require("http");
var https = require("https");
//Request URL
var url = 'http://whatismyip.akamai.com';
//var url = 'https://en.wikipedia.org/wiki/Main_Page';
//Proxy config
var USE_PROXY = false;
var PROXY_PROTOCOL= 'http';
var PROXY_HOST= '127.0.0.1';
var PROXY_PORT= 6401;
/******/
//Define request options
var protocol = (USE_PROXY ? (PROXY_PROTOCOL == 'https' ? https : http) : (getProtocolFromUrl(url) == 'https' ? https : http));
var host = (USE_PROXY ? PROXY_HOST : getDomainFromUrl(url));
var port = (USE_PROXY ? PROXY_PORT : (getProtocolFromUrl(url) == 'https' ? 443 : 80));
var path = (USE_PROXY ? url : getPathFromUrl(url));
console.log('options.host = ' + host);
console.log('options.port = ' + port);
console.log('options.path = ' + path);
console.log('\n');
//Make the request
protocol.get({host: host, port: port, path: path}, function(res) {
console.log('x-client-ip header: ' + res.headers['x-client-ip']);
console.log('Status code: ' + res.statusCode);
if ([301,302,303].indexOf(res.statusCode) > -1){
console.log('location header : ' + res.headers["location"]);
console.log('Redirect url: ' + combineUrls(res.headers["location"],url));
} else {
res.on('data', function (data) {
console.log('Response body: ' + data.toString());
});
}
}).on('error', function(err) {
console.log(err);
});
In the above, the host/port/path specified in the request options are determined from the URL and the proxy configuration set (as is whether to use http or https library). To check that the correct proxy settings are used, I am checking the request IP read by the host server as follows:
http protocol: Response body from http://whatismyip.akamai.com/
https protocol: x-client-ip header from https://en.wikipedia.org/wiki/Main_Page
With no proxy set, these are the results:
1.
options.host = whatismyip.akamai.com
options.port = 80
options.path = /
x-client-ip header: undefined
Status code: 200
Response body: **.**.**.** (public IP)
options.host = en.wikipedia.org
options.port = 443
options.path = /wiki/Main_Page
x-client-ip header: **.**.**.** (public IP)
Status code: 200
Response body: <!DOCTYPE html> //etc...
However, if I try to use the proxy (which is routing the requests via a tethered mobile device and using its mobile network), I get the following results:
1.
options.host = 127.0.0.1
options.port = 6401
options.path = http://whatismyip.akamai.com
x-client-ip header: undefined
Status code: 400
Response body: <HTML><HEAD>
<TITLE>Invalid URL</TITLE>
</HEAD><BODY>
<H1>Invalid URL</H1>
The requested URL "http:/whatismyip.akamai.com", is invalid.<p>
Reference #9.64fcff5a.1605358006.0
</BODY></HTML>
options.host = 127.0.0.1
options.port = 6401
options.path = https://en.wikipedia.org/wiki/Main_Page
x-client-ip header: **.**.**.** (Proxy IP)
Status code: 301
location header : https://en.wikipedia.org/wiki/Main_Page
Redirect url: https://en.wikipedia.org/wiki/Main_Page
So there seem to be 2 issues when using the proxy settings - in the first request, the response code is http400 and the body suggests that the path provided is encrypted. What is strange is that sometimes I can get it to work if I rewrite the code and hardcode the settings in, e.g.:
http.get({
host: '127.0.0.1',
port: 6401,
path: 'http://whatismyip.akamai.com/'
}, function (res) {
console.log('Status code: ' + res.statusCode);
res.on('data', function (data) {
console.log('Response body: ' + data.toString());
});
});
will return:
Status code: 200
Response body: **.**.**.** (Proxy IP)
So I am currently at a loss to explain why it fails in the earlier code.
In the second case with the https protocol, the response code of 301 is returned, but the redirect location matches the original URL, so there must be an issue with the way the path is presented when the request is forwarded to the host server.
*** Edit ***
As suggested, tried using the other request libraries. Example with axios below:
axios.get('https://en.wikipedia.org/wiki/Main_Page',{
proxy: {
host: '127.0.0.1',
port: 6401
}
}).then(function (res) {
console.log(res.status);
console.log(res.headers['x-client-ip']);
//console.log(res.data);
}).catch(function (err) {
console.log(err);
});
The Public IP was returned instead of the Proxy IP in this instance.
Related
I'm doing research in proxies in nodejs. I came across something that blew my mind. In one of the options for a http.request connection, the source code showed this as the options object
const options = {
port: 1337,
host: '127.0.0.1',
method: 'CONNECT',
path: 'www.google.com:80'
};
This was a part of a far bigger code which was the whole tunneling system. But can someone just explain how the options above work? The whole code is below
const http = require('http');
const net = require('net');
const { URL } = require('url');
// Create an HTTP tunneling proxy
const proxy = http.createServer((req, res) => {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.end('okay');
});
proxy.on('connect', (req, clientSocket, head) => {
// Connect to an origin server
const { port, hostname } = new URL(`http://${req.url}`);
const serverSocket = net.connect(port || 80, hostname, () => {
clientSocket.write('HTTP/1.1 200 Connection Established\r\n' +
'Proxy-agent: Node.js-Proxy\r\n' +
'\r\n');
serverSocket.write(head);
serverSocket.pipe(clientSocket);
clientSocket.pipe(serverSocket);
});
});
// Now that proxy is running
proxy.listen(1337, '127.0.0.1', () => {
// Make a request to a tunneling proxy
const options = {
port: 1337,
host: '127.0.0.1',
method: 'CONNECT',
path: 'www.google.com:80'
};
const req = http.request(options);
req.end();
req.on('connect', (res, socket, head) => {
console.log('got connected!');
// Make a request over an HTTP tunnel
socket.write('GET / HTTP/1.1\r\n' +
'Host: www.google.com:80\r\n' +
'Connection: close\r\n' +
'\r\n');
socket.on('data', (chunk) => {
console.log(chunk.toString());
});
socket.on('end', () => {
proxy.close();
});
});
});
Source: https://nodejs.org/api/http.html#http_event_connect
You probably have never used a network that requires you to configure a HTTP proxy. Most networks these days configure their firewall to allow HTTP traffic. This means most people these days have never needed to use a HTTP proxy to access the web.
A long-long time ago when I first started using the internet (around 1994) a lot of networks don't allow transparent internet access. Your PC does not have any connection to the outside world. But sysadmins would install a HTTP proxy that you can connect to. Your PC would only have access to the LAN (which the proxy is a part of) and only the HTTP proxy would have access to the internet.
Here's an example of how you'd configure Windows to use a HTTP proxy:
If you configure your PC as above, then when you connect to www.google.com your browser would connect to the host proxy.example.com on port 8080 and then request it to fetch data from www.google.com.
As for why it calls the requested resource path it's because it is sent in the "path" part of the packet.
For example, a normal GET request for getting this page looks something like this:
GET /questions/60498963 HTTP/1.1
Host: stackoverflow.com
And the string after GET and before protocol version is normally called the path:
.---------- this is normally called
| the "path"
v
GET /questions/60498963 HTTP/1.1
Host: stackoverflow.com
When making a proxy request the HTTP header looks like this:
CONNECT stackoverflow.com/questions/60498963 HTTP/1.1
So the url including the domain name is sent to the proxy in the part of the packet usually used to send file path.
Note that all this has nothing to do with Node.js. This is just basic networking (no programming languages involved).
I would like to re-send any POST request from my server (1234 port) to another server (another.server.com:80). Note: Post requests are soap calls.
This is my code:
var http = require('http');
var LISTEN_PORT = 1234;
var HOST = 'another.server.com';
var PORT = 80;
http.createServer(onRequest).listen(LISTEN_PORT);
function onRequest(client_req, client_res) {
var options = {
hostname: HOST,
port: PORT,
path: client_req.url,
method: 'POST'
};
var proxy = http.request(options, function (res) {
res.pipe(client_res, {
end: true
});
});
client_req.pipe(proxy, {
end: true
});
}
But it does not work.
I need to send my client HTTPS requests through an intranet proxy to a server.
I use both https and request+global-tunnel and neither solutions seem to work.
The similar code with 'http' works. Is there other settings I missed?
The code failed with an error:
REQUEST:
problem with request: tunneling socket could not be established, cause=socket hang up
HTTPS:
events.js:72
throw er; // Unhandled 'error' event
^
Error: socket hang up
at SecurePair.error (tls.js:1011:23)
at EncryptedStream.CryptoStream._done (tls.js:703:22)
at CleartextStream.read [as _read] (tls.js:499:24)
The code is the simple https test.
var http = require("https");
var options = {
host: "proxy.myplace.com",
port: 912,
path: "https://www.google.com",
headers: {
Host: "www.google.com"
}
};
http.get(options, function(res) {
console.log(res);
res.pipe(process.stdout);
});
You probably want to establish a TLS encrypted connection between your node app and target destination through a proxy.
In order to do this you need to send a CONNECT request with the target destination host name and port. The proxy will create a TCP connection to the target host and then simply forwards packs between you and the target destination.
I highly recommend using the request client. This package simplifies the process and handling of making HTTP/S requests.
Example code using request client:
var request = require('request');
request({
url: 'https://www.google.com',
proxy: 'http://97.77.104.22:3128'
}, function (error, response, body) {
if (error) {
console.log(error);
} else {
console.log(response);
}
});
Example code using no external dependencies:
var http = require('http'),
tls = require('tls');
var req = http.request({
host: '97.77.104.22',
port: 3128,
method: 'CONNECT',
path: 'twitter.com:443'
});
req.on('connect', function (res, socket, head) {
var tlsConnection = tls.connect({
host: 'twitter.com',
socket: socket
}, function () {
tlsConnection.write('GET / HTTP/1.1\r\nHost: twitter.com\r\n\r\n');
});
tlsConnection.on('data', function (data) {
console.log(data.toString());
});
});
req.end();
If I send curl -k "https://abc.com?querystring"
-k option to turn off curl's verification of the certificate
How to do this in nodejs If I want to make a GET request?
How to override all http GET request do it in the same way?
Thank you for your support.
Set the rejectUnauthorized option to false.
var https = require('https');
var req = https.request({
hostname: 'example.com',
port: 443,
path: '/',
method: 'GET',
rejectUnauthorized: false
}, function() { ... });
Check the following code:
var http = require('http');
var target = {
host : 'localhost',
port : 3000,
path : 'URL'
//pfx: Certificate, Private key and CA certificates to use for SSL. Default is null.
//cert: Public x509 certificate to use. Default null.
};
var Req_options = {
host: target.host,
port: target.port,
path: target.path,
agent: false,
method: 'GET'
};
callback = function(response) {
var str = ''
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(str);
});
}
var req = http.request(Req_options, callback);
req.end();
Updated as per comments:
In the above code, I have changed the https & target only as follows:
var https = require('https');
var target = {
host : 'www.google.co.in',
port : 443,
path : '/'
};
The output is as follows:
</html>
.........
.........
.........
attachEvent&&window.attachEvent("onload",n);google.timers.load.t.prt=e=(new Date).getTime();})();
</script></body></html>
For more information, check this Node.js API docs
I am trying to make a simple request using http.get. But I need to make this request through Squid proxy. Here's my code:
var http = require('http');
var username = 'username';
var password = 'password';
var host = "proxy_host";
var auth = 'Basic ' + new Buffer(username + ':' + password).toString('base64');
var options = {
host: host,
port: 3128,
path: "http://www.google.com",
authorization: auth,
headers: {
Host: 'www.google.com'
}
};
var req = http.request(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
});
req.end();
My username and password is correct according to the ops team that set up the proxy. The problem is that I keep getting a 407 - authorization required status coming back.
Is there something wrong with the way I am making the request? Or does Squid proxy need to be configured?
Thanks in advance.
You should include auth in the headers section of options:
var options = {
host: host,
port: 3128,
path: "http://www.google.com",
headers: {
'Proxy-Authorization': auth,
Host: 'www.google.com'
}
};