I am trying to make a simple request using http.get. But I need to make this request through Squid proxy. Here's my code:
var http = require('http');
var username = 'username';
var password = 'password';
var host = "proxy_host";
var auth = 'Basic ' + new Buffer(username + ':' + password).toString('base64');
var options = {
host: host,
port: 3128,
path: "http://www.google.com",
authorization: auth,
headers: {
Host: 'www.google.com'
}
};
var req = http.request(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
});
req.end();
My username and password is correct according to the ops team that set up the proxy. The problem is that I keep getting a 407 - authorization required status coming back.
Is there something wrong with the way I am making the request? Or does Squid proxy need to be configured?
Thanks in advance.
You should include auth in the headers section of options:
var options = {
host: host,
port: 3128,
path: "http://www.google.com",
headers: {
'Proxy-Authorization': auth,
Host: 'www.google.com'
}
};
Related
I am trying to send http (GET and POST requests) to various sites to retrieve response header/body information, and need to vary whether I am using a proxy or not. After previously failing to use restler library for this purpose, I have switched to node 'http' and 'https' libraries to perform these tasks. Code as follows:
/******/
//Http client libraries
var http = require("http");
var https = require("https");
//Request URL
var url = 'http://whatismyip.akamai.com';
//var url = 'https://en.wikipedia.org/wiki/Main_Page';
//Proxy config
var USE_PROXY = false;
var PROXY_PROTOCOL= 'http';
var PROXY_HOST= '127.0.0.1';
var PROXY_PORT= 6401;
/******/
//Define request options
var protocol = (USE_PROXY ? (PROXY_PROTOCOL == 'https' ? https : http) : (getProtocolFromUrl(url) == 'https' ? https : http));
var host = (USE_PROXY ? PROXY_HOST : getDomainFromUrl(url));
var port = (USE_PROXY ? PROXY_PORT : (getProtocolFromUrl(url) == 'https' ? 443 : 80));
var path = (USE_PROXY ? url : getPathFromUrl(url));
console.log('options.host = ' + host);
console.log('options.port = ' + port);
console.log('options.path = ' + path);
console.log('\n');
//Make the request
protocol.get({host: host, port: port, path: path}, function(res) {
console.log('x-client-ip header: ' + res.headers['x-client-ip']);
console.log('Status code: ' + res.statusCode);
if ([301,302,303].indexOf(res.statusCode) > -1){
console.log('location header : ' + res.headers["location"]);
console.log('Redirect url: ' + combineUrls(res.headers["location"],url));
} else {
res.on('data', function (data) {
console.log('Response body: ' + data.toString());
});
}
}).on('error', function(err) {
console.log(err);
});
In the above, the host/port/path specified in the request options are determined from the URL and the proxy configuration set (as is whether to use http or https library). To check that the correct proxy settings are used, I am checking the request IP read by the host server as follows:
http protocol: Response body from http://whatismyip.akamai.com/
https protocol: x-client-ip header from https://en.wikipedia.org/wiki/Main_Page
With no proxy set, these are the results:
1.
options.host = whatismyip.akamai.com
options.port = 80
options.path = /
x-client-ip header: undefined
Status code: 200
Response body: **.**.**.** (public IP)
options.host = en.wikipedia.org
options.port = 443
options.path = /wiki/Main_Page
x-client-ip header: **.**.**.** (public IP)
Status code: 200
Response body: <!DOCTYPE html> //etc...
However, if I try to use the proxy (which is routing the requests via a tethered mobile device and using its mobile network), I get the following results:
1.
options.host = 127.0.0.1
options.port = 6401
options.path = http://whatismyip.akamai.com
x-client-ip header: undefined
Status code: 400
Response body: <HTML><HEAD>
<TITLE>Invalid URL</TITLE>
</HEAD><BODY>
<H1>Invalid URL</H1>
The requested URL "http:/whatismyip.akamai.com", is invalid.<p>
Reference #9.64fcff5a.1605358006.0
</BODY></HTML>
options.host = 127.0.0.1
options.port = 6401
options.path = https://en.wikipedia.org/wiki/Main_Page
x-client-ip header: **.**.**.** (Proxy IP)
Status code: 301
location header : https://en.wikipedia.org/wiki/Main_Page
Redirect url: https://en.wikipedia.org/wiki/Main_Page
So there seem to be 2 issues when using the proxy settings - in the first request, the response code is http400 and the body suggests that the path provided is encrypted. What is strange is that sometimes I can get it to work if I rewrite the code and hardcode the settings in, e.g.:
http.get({
host: '127.0.0.1',
port: 6401,
path: 'http://whatismyip.akamai.com/'
}, function (res) {
console.log('Status code: ' + res.statusCode);
res.on('data', function (data) {
console.log('Response body: ' + data.toString());
});
});
will return:
Status code: 200
Response body: **.**.**.** (Proxy IP)
So I am currently at a loss to explain why it fails in the earlier code.
In the second case with the https protocol, the response code of 301 is returned, but the redirect location matches the original URL, so there must be an issue with the way the path is presented when the request is forwarded to the host server.
*** Edit ***
As suggested, tried using the other request libraries. Example with axios below:
axios.get('https://en.wikipedia.org/wiki/Main_Page',{
proxy: {
host: '127.0.0.1',
port: 6401
}
}).then(function (res) {
console.log(res.status);
console.log(res.headers['x-client-ip']);
//console.log(res.data);
}).catch(function (err) {
console.log(err);
});
The Public IP was returned instead of the Proxy IP in this instance.
this is the request i want to perform:
POST /v1/oauth2/token HTTP/1.1
Host: api.sandbox.paypal.com
Accept: application/json
Accept-Language: en_US
Authorization: Basic cGF5cGFsaWQ6c2VjcmV0
Content-Type: application/x-www-form-urlencoded
grant_type=client_credentials
I tried it in nodejs using this code:
paypalSignIn = function(){
var username = process.env.PAYPALID;
var password = process.env.PAYPALSECRET;
var auth = 'Basic ' + Buffer.from(username + ':' + password).toString('base64');
// new Buffer() is deprecated from v6
// auth is: 'Basic VGVzdDoxMjM='
var post_data = querystring.stringify({
'grant_type' : 'client_credentials',
});
var header = {'Accept': 'application/json', 'Authorization': auth, 'Accept-Language': 'en_US'};
const options = {
hostname: 'api.sandbox.paypal.com',
port: 443,
path: '/v1/oauth2/token',
method: 'POST',
headers: header,
}
var post_req = http.request(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('Response: ' + chunk);
});
});
post_req.write(post_data);
post_req.end();
}
Unfortunately i'm getting the following error:
Error: socket hang up
Try using the https module (it's not enough to set port 443, you have to use the HTTPS protocol to connect to an HTTPS endpoint).
I also noticed you didn't set the Content-Type header. It depends on the API, but that may cause problems for you too.
Finally, I'd consider using a library that wraps http/https like node-fetch, bent, or axios for this rather than the standard library directly. It can handle things like writing to the socket, setting the Content-Length header, etc.
I do have the following configuration for my hapi server
const server = new Hapi.Server();
const tls = {
cert: fs.readFileSync(path.join(__dirname, '../certificates/cert.crt')),
key: fs.readFileSync(path.join(__dirname, '../certificates/cert.key')),
};
server.connection({
port: process.env.PORT_HTTP || 80,
host: process.env.HOST || 'localhost',
});
server.connection({
port: process.env.PORT_HTTPS || 443,
host: process.env.HOST || 'localhost',
tls,
});
The server is working ok on both, http and https, but I would like to redirect all the traffic from the http to https.
How should I proceed, tried already to register the hapi-require-https npm module but the traffic still remain the same, nothing happens.
Create an extra server for http requests and bind them to redirect function.
var Hapi = require('hapi');
var http = new Hapi.Server(80);
var server = new Hapi.Server(443, { tls: {} });
var redirect = function () {
this.reply.redirect('https://your.site/' + this.params.path);
});
http.route({ method: '*', path: '/{path*}', handler: redirect });
Update(other option)
server.route({
method: 'GET',
path: '/',
handler: function (request, reply) {
if(request.headers.referer.split(':')[0] == "http"){
this.reply.redirect('https://your.site' + this.params.path);
}
}
});
How about this? Binding them both
var http = new Hapi.Server(80); // our extra server
http.route({
method: '*',
path: '/{path*}',
handler:
function (request, reply) {
// if(request.headers.referer.split(':')[0] == "http"){
this.reply.redirect('https://your.site' + this.params.path);
// }
}
});
Create two server instances to handle http & https traffic seperately.
var Hapi = require('hapi');
var server = new Hapi.Server(80);
var httpsServer = new Hapi.Server(443, { tls: { // your certificates here} });
Now register the hapi-gate plugin to the base server so that it redirects the traffic to https.
server.register({
register: require('hapi-gate'),
options: {https: true} // will force https on all requests
});
You can also use the hapi-require-https plugin instead.
I can use curl for making a GET request ->
`curl -v https://example.com:82/v1/api?a=b` -E client_cert.pem:password
How can I use same in node. I tried request, superagent but not able to pass certificate.
Thanks in advance!
This worked for me -
var https = require('https');
var fs = require('fs');
var options = {
hostname: 'example.com',
port: 83,
path: '/v1/api?a=b',
method: 'GET',
key: fs.readFileSync('/path/to/private-key/key.pem'),
cert: fs.readFileSync('/path/to/certificate/client_cert.pem'),
passphrase: 'password'
};
var req = https.request(options, function(res) {
console.log(res.statusCode);
res.on('data', function(d) {
process.stdout.write(d);
});
});
req.end()
If I send curl -k "https://abc.com?querystring"
-k option to turn off curl's verification of the certificate
How to do this in nodejs If I want to make a GET request?
How to override all http GET request do it in the same way?
Thank you for your support.
Set the rejectUnauthorized option to false.
var https = require('https');
var req = https.request({
hostname: 'example.com',
port: 443,
path: '/',
method: 'GET',
rejectUnauthorized: false
}, function() { ... });
Check the following code:
var http = require('http');
var target = {
host : 'localhost',
port : 3000,
path : 'URL'
//pfx: Certificate, Private key and CA certificates to use for SSL. Default is null.
//cert: Public x509 certificate to use. Default null.
};
var Req_options = {
host: target.host,
port: target.port,
path: target.path,
agent: false,
method: 'GET'
};
callback = function(response) {
var str = ''
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(str);
});
}
var req = http.request(Req_options, callback);
req.end();
Updated as per comments:
In the above code, I have changed the https & target only as follows:
var https = require('https');
var target = {
host : 'www.google.co.in',
port : 443,
path : '/'
};
The output is as follows:
</html>
.........
.........
.........
attachEvent&&window.attachEvent("onload",n);google.timers.load.t.prt=e=(new Date).getTime();})();
</script></body></html>
For more information, check this Node.js API docs