Node.js Https request Error - node.js

I've tried the sample from the documentation and it works great.
But when I change the URL to https://api.mercadolibre.com/sites/, the request hangs. The only thing I get is:
{ [Error: socket hang up] code: 'ECONNRESET' }
Here's my code:
var https = require('https');
this.dispatch = function(req, res) {
var renderHtml = function(content) {
res.writeHead(200, {'Content-Type': 'text/html'});
res.end(content, 'utf-8');
}
var parts = req.url.split('/');
var options = {
host: 'api.mercadolibre.com',
port: 443,
path: '/sites/',
method: 'GET'
};
var request = https.request(options, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
res.on('data', function(d) {
process.stdout.write(d);
});
});
request.on('error', function(e) {
console.error('error');
console.error(e);
});
request.end();
return 'item id:' + parts[2];
};
I've tried with curl, soapui and with a browser. On all cases works great, but with node.js it doesn't.
How can I get more data on what's going on?
added
With curl i do: curl --sslv3 https://api.mercadolibre.com/sites/ works.
I've test same in centos 6 and works too.
I've reinstalled node, this time from source, same problem. My Os is ubuntu 12.04.
Thanks.

I'm not sure about api.mercadolibre.com site, but I can call API if I remove port param, like following code:
var options = {
host: 'api.mercadolibre.com',
path: '/sites/',
method: 'GET'
};
And we also need add param to support SSL version 3:
https.globalAgent.options.secureProtocol = 'SSLv3_method';

Why not use a library like request to deal with the details for you?
var request = require('request');
request('https://api.mercadolibre.com/sites/', {}, function(err, res, body) {
console.log("Got body: ", body);
});
This yields:
Got body: [{"id":"MLA","name":"Argentina"},{"id":"MLB","name":"Brasil"},{"id":"MCO","name":"Colombia"},{"id":"MCR","name":"Costa Rica"},{"id":"MEC","name":"Ecuador"},{"id":"MLC","name":"Chile"},{"id":"MLM","name":"Mexico"},{"id":"MLU","name":"Uruguay"},{"id":"MLV","name":"Venezuela"},{"id":"MPA","name":"Panamá"},{"id":"MPE","name":"Perú"},{"id":"MPT","name":"Portugal"},{"id":"MRD","name":"Dominicana"}]

Since it is working with curl, try using node-curl module. I lost a whole day trying to make it work in node.js with http and/or https modules until I switched to node-curl.
Try this:
var curl = require('node-curl');
curl('https://api.mercadolibre.com/sites/', {SSLVERSION: 3}, function(err, res) {
var body = res.body;
res.close();
console.log(body);
});

Same here, working with curl but not with node.js.
Problem: here on CentOS-5 curl usesthe provides openssl libraries and so uses centos standard /etc/pki/tls/certs/ca-bundle.crt for CA checks.
Where does node.js look for?, via strace there I cannot see any reference to a CA-file for checking.
Node.js request against server with valid SSL-certificate from well known old issuer are accepted, but not against my own webserver with own CA.
I put my own CA.crt in the ca-bundle.crt file, so now curl accepts it, but not node.js.
Only solution for now is to deactivate the verification-check for my dev-box:
var client = require('https');
var download_options = url.parse(sourceUrl);
download_options.method = "GET";
download_options.agent = false;
download_options.rejectUnauthorized = false; / HERE to accept all SSL-certificates */
var download_request = client.request(download_options);

I think you are behind a proxy which you need to specify to request. Proxy settings are detected automatically by libcurl, which node-curl uses. Therefore the request passes in node-curl.
Therefore, find out the proxy IP and port your organization uses, and try this:
var request = require('request');
request({
uri : 'https://mail.google.com/mail',
proxy : 'http://<proxy ip>:<proxy port>'
}, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body) // Print the google web page.
}else{
console.log(error);
console.log(response.statusCode);
}
})

You will get the ECONNRESET error if you do this:
post_options.path = 'history';
...
var req = http.request(post_options, function(res) {
...
That is, you need to make sure your path has a / like this:
post_options.path = '/history';
...

Related

How Nginx handles outgoing requests going from nodejs as a client?

Intro:
I have and angular frontend and node backend (Server A) working fine. Using nginx as well for Server A.
Now from my node server (Server A) I need to call an API endpoint of other server (Server B).
Nginx configurations for Server B wont matter because curl command is working fine.
I have a PUT and POST https outgoing request from my node server to other server. On my system I have nginx service working.
The API is working fine when I use curl from the terminal but for the same thing via node https module the server is giving 400 BAD REQUEST.
What is the process for outgoing requests for nginx?
Why is the curl command working and node env requests not working?
curl -i --insecure --request PUT --url https://example.com:443/update --header 'content-type:application/json' --data '{"example":"Goodmorning"}'
httpsRequest(serverurl, method, header, data, callback){
console.log("httpsRequest function body")
let datastr = "";
if (data !== undefined)
datastr = JSON.stringify(data);
const options = {
host:'example.com',
port : 443,
rejectUnauthorized: false,
path : "/update",
body : datastr,
method : method,
secure:false,
key : readFileSync("example.key"),
cert : readFileSync("example.crt"),
};
if (header !== undefined) {
options['headers'] = header
};
}
console.log("options\n", options);
process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
options.agent = new https.Agent(options);
// options.agent = httpsAgent;
const req = https.request(options, (res) => {
console.log('status code ', res.statusCode);
console.log('headers:', res.headers);
res.on('data', (d) => {
process.stdout.write(d);
console.log(d);
if(res.statusCode === "errorexample"){
callback(null, {data : "success"})
}else{
let errormsg = {
message : res.message,
statusCode : res.statusCode,
failureAlarmCode : res.failureAlarmCode
}
callback(null, { "error": errormsg });
}
});
});
req.on('error', (e) => {
console.error(e);
callback(e, null);
});
req.end();
}
I think the curl from terminal does not route via the client nginx, hence some difference there. AM I correct?
I see a couple problems in the code in your question:
There appears to be some mismatched bracing that closes your function before you want it closed.
You aren't sending the body of the request. https.request() does not have a body option. Instead, you use req.write(datastr) to send the body. You can see in the doc that there is no body property in the options object and it also shows a code example where req.write() goes. Since a POST request is required to have a body, this could be the cause of your error.
You are creating a new https agent, but not passing it appropriate arguments. It's unclear why you are creating a new agent as that would not generally be required and if you are going to do that, you'd have to specify the reason and then use appropriate arguments. In my code solution below, I've removed the code creating a new agent since it isn't correct. In its place, I added agent: false to the option object which will create a new agent specifically for this request with default options. If you want to customize the options, then you need to create an agent using the proper options shown in the doc.
You are including key and cert in a client request. This is not typically required for a client request unless the server specifically requires client certificate authentication. But, your curl request does not supply these so apparently they are not needed. I'd suggest removing those two options.
Here's your code with these issues changed:
httpsRequest(serverurl, method, header, data, callback) {
console.log("httpsRequest function body")
let datastr = "";
if (data !== undefined)
datastr = JSON.stringify(data);
const options = {
host: 'example.com',
port: 443,
rejectUnauthorized: false,
path: "/update",
method: method,
secure: false,
// create new agent with default values just for this request
agent: false,
};
if (header !== undefined) {
options['headers'] = header
}
console.log("options\n", options);
process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
const req = https.request(options, (res) => {
console.log('status code ', res.statusCode);
console.log('headers:', res.headers);
res.on('data', (d) => {
process.stdout.write(d);
console.log(d);
if (res.statusCode === "errorexample") {
callback(null, { data: "success" })
} else {
let errormsg = {
message: res.message,
statusCode: res.statusCode,
failureAlarmCode: res.failureAlarmCode
}
callback(null, { "error": errormsg });
}
});
});
req.on('error', (e) => {
console.error(e);
callback(e, null);
});
req.write(datastr);
req.end();
}
You may also want to note that res.on('data', ...) can receive more than one data event so you could be calling your callback more than once.
One other thing that look suspicious here is your inclusion of these two options:
key: readFileSync("example.key"),
cert: readFileSync("example.crt"),
This is a client request. You would not normally need to supply these unless the target server requires client certificate authentication which it does not appear to because your curl command does not provide these. I would suggest removing these from your options object.

Send new request response with Node HTTP server

Stack Overflow community, greetings. I'm trying to pass the response of a new request on the request object using the Node HTTP Module for a basic autocomplete search app for my website (i.e using Node as a proxy that will transform and redirect the requests within the server).
The flow basically is:
Client Browser - Node - ElasticSearch - Node - Client Browser
I've started with:
Listen to requests with http.createServer (function (req,res)
Get the body from the req object and use it in a new request with http.request(options, function (newReqResponse)
Get the body from that newReqResponse object and send it back to the client on the res object
The problem is that the content of newReqResponse is always outdated (trails behind the last typed character). i.e.:
If I type "te", the content of newReqResponse corresponds to that if I had typed only "t".
If I type "test", it corresponds to that if I had typed "tes".
And so on.
I've tried to solve it using Node.js streams and using the file system module to write and read files sync and async, but the result is the same. Here's a sample of the whole -code- picture:
var http = require('http');
var fs = require('fs');
http.createServer(function (req, res) {
var reqBody = '';
var newReqResponseBody = "";
req.on('data', function (chunk) {
reqBody += chunk;
fs.writeFile('reqbody.json', reqBody, function(err) {
if (err) {
throw err;
}
});
var options = {
hostname: '127.0.0.1',
port: 9500,
method: 'GET',
path: '/_search',
headers: { host: 'es',
'content-length': Buffer.byteLength(reqBody),
'content-type': 'application/json',
accept: 'application/json' },
};
var newReq = http.request(options, function (newReqResponse) {
newReqResponse.setEncoding("UTF-8");
newReqResponse.on('data', function (ch) {
newReqResponseBody += ch;
});
newReqResponse.on("end", function() {
fs.writeFile("newReqResponseBody.json", newReqResponseBody, function(err) {
if (err) {
throw err;
}
});
});
});
newReq.on("error", function(err) {
console.log(`problem with request: ${err.message}`);
});
newReq.write(reqBody);
newReq.end();
});
req.on('end', function() {
var responseBody = fs.readFileSync('newReqResponseBody.json', 'utf8');
console.log(responseBody);
res.end(responseBody);
});
}).listen(3000, '127.0.0.1');
Is there a workaround to work with requests and responses within the http server? If there isn't, I'll be very grateful if you give me any directions on how to solve this.
Since the planned use for Node is rather basic, I prefer to stick with core modules rather than having to get new ones from npm, unless that it's necessary.
Thanks in advance.
EDIT:
All I had to do was to call res.end(responseBody) within the newReqResponse.on("end") callback, which is totally counterintuitive for me, but... it works.
Glad you solved your own problem. However, I see room for improvement (not sure if you're new), especially if you're transferring data. Which you can do with streams.
You can see that I didn't calculate the content length, you're asked not to and should get ignore (for this specific case) according to HTTP specification as streams pass data in chunks with 'Transfer-Encoding': 'chunked' header.
const fs = require('fs');
const http = require('http');
const options = {
hostname: '127.0.0.1',
port: 9500,
method: 'GET',
path: '/_search',
headers: {
'Content-Type': 'application/json'
}
};
http.createServer((req, res) => {
req.pipe(fs.createWriteStream('reqBody.json'));
let request = http.request(options, (newRes) => {
newRes.pipe(res);
});
fs.createReadStream('reqBody.json').pipe(request);
res.setHeader('Content-Type', 'application/json');
}).listen(3000, '127.0.0.1');
You can shorten this snippet more if you don't want your data saved in the future and only want to pipe the req stream to request.

DELETE request to REST API returns 500

function delete(id, response) {
var https = require('https');
var linkpath = "/v1/endpoint/" + id + "/?token=" + AUTH_KEY;
var req = https.request({
hostname: 'api.foo.com',
port: 443,
path: linkpath,
agent: false,
method: 'DELETE',
}, (res) => {
if (res.statusCode !== 200) {
response.send('HTTP ' + res.statusCode + ' ' + res.statusMessage);
}
res.on('error', function (err) {
response.send(err);
});
res.on('end', function (data) {
response.send(data);
});
});
req.on('error', function(e) {
response.send(e.message);
});
req.end();
}
This code, adapted from my (working) code that uses a POST request to do other things with this API, nets me a status code of 500 from the endpoint.
I don't know how to debug this. I can't send the URL manually to the server because it's a DELETE operation instead of a GET or POST.
Has anyone seen this problem? Or do you have ideas on how to debug it?
Postman (https://www.getpostman.com/) is a great tool for manually sending specific HTTP requests, including DELETE!
There are all sorts of tools that will let you manually send any HTTP to the server. For instance, you can get quite a bit of information with curl, which will happily send a DELETE request.
For example:
curl -v -X "DELETE" https://jsonplaceholder.typicode.com/posts/1
will return the request and response headers as well as the body of the return value if any.

HTTP Get Request from NodeJS

I am trying to create http get request from node, to get information from youtube URL. When I click it in browser I get json response but if I try it from node, I get ssl and other types of error. What I have done is,
this.getApiUrl(params.videoInfo, function (generatedUrl) {
// Here is generated URL - // https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus
console.log(generatedUrl);
var req = http.get(generatedUrl, function (response) {
var str = '';
console.log('Response is ' + response.statusCode);
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(str);
});
});
req.end();
req.on('error', function (e) {
console.log(e);
});
});
I get this error
{
"error": {
"message": "Protocol \"https:\" not supported. Expected \"http:\".",
"error": {}
}
}
When I make it without https I get this error,
Response is 403
{"error":{"errors":[{"domain":"global","reason":"sslRequired","message":"SSL is required to perform this operation."}],"code":403,"message":"SSL is required to perform this operation."}}
You need to use the https module as opposed to the http module from node, also I would suggest one of many http libraries that provide a higher level api such as wreck or restler which allow you to control the protocol via options as opposed to a different required module.
Your problem is obviously accessing content served securely with http request hence, the error. As I have commented in your question, you can make use of https rather than http and that should work but, you can also use any of the following approaches.
Using request module as follow:
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
});
Using https module you can do like below:
var https = require('https');
var options = {
hostname: 'www.googleapis.com', //your hostname youtu
port: 443,
path: '//youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus',
method: 'GET'
};
//or https.get() can also be used if not specified in options object
var req = https.request(options, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
res.on('data', function(d) {
process.stdout.write(d);
});
});
req.end();
req.on('error', function(e) {
console.error(e);
});
You can also use requestify module and
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
requestify.get(url).then(function(response) {
// Get the response body
console.log(response.body);
});
superagent module is another option
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
superagent('GET', url).end(function(response){
console.log('Response text:', response.body);
});
Last but not least is the unirest module allow you to make http/https request as simple as follow:
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
unirest.get(url).end(function(res) {
console.log(res.raw_body);
});
There might be more options out there. Obviously you need to load the modules using require before using it
var request = require('request');
var https = require('https');
var requestify = require('requestify');
var superagent = require('superagent');
var unirest = require('unirest');
I provided extra details, not only to answer the question but, also to help others who browse for similiar question on how to make http/https request in nodejs.

Node.js proxy to return exact response from the external website

I'm trying to create the express.js proxy to an external website to obtaining audio data from there. I know about modules like http-proxy, but think they are excessive for the case when only one url-bound request goes through proxy. I'm using the code below:
express.get('/proxy', function (req, res) {
var options ={
host: "website.com",
port: 80,
path: "/audio/test.mp3",
method: 'GET'
};
http.get(options, function (audioRes) {
var data = [], dataLen = 0;
audioRes.on('data', function(chunk) {
data.push(chunk);
dataLen += chunk.length;
})
.on('end', function() {
var buf = new Buffer(dataLen);
res.set(audioRes.headers);
res.send(buf);
});
})
.on('error', function (error) {
console.log(error.message);
});
});
I get response, but it cannot be decoded as a valid audio. While debugging with the Fiddler, I found out that the number of bites sent by a server mismatches the number specified in the Content-Length header (which indicates fewer bytes being retrieved).
I cannot figure out how to properly return the exact response that's been retrieved from the remote server. Would be grateful for any help.
To send request via proxy, you can set the proxy url in Host header. Also you have to specify the full URL of the external resource you are trying to access via proxy.
var http = require("http");
var options = {
host: "proxy",
port: 8080,
path: "http://www.google.com", //full URL
headers: {
Host: "10.1.2.3" //your proxy location
}
};
http.get(options, function(res) {
console.log(res);
});
I am not sure why it is not returning full response. Can you post your options.
Update
Try this inside /proxy after putting the options
http.get(options, function (audioRes) {
audioRes.pipe(res);
})
.on('error', function (error) {
console.log(error.message);
});

Resources