I'm making a simple request to an rss feed:
var request = require('request');
var req = request('http://www.govexec.com/rss/contracting/');
req.on('error', function (error) {
console.log('error:',arguments);
});
req.on('response', function (res) {
var stream = this;
if (res.statusCode != 200) return this.emit('error', new Error('Bad status code'),res.statusCode);
});
output is error: { '0': [Error: Bad status code], '1': 500 }
However, if I hit the url from the browser, or do a simple curl request, I get the correct response
curl 'http://www.govexec.com/rss/contracting/'
It's not a programming problem, per say.
Most websites do expect you to send the header user-agent with your request. It seems to be this way with the website you have provided too.
Fixing this is trivial, since you can use include the user-agent like so:
var req = request({
url:'http://www.govexec.com/rss/contracting/',
headers: {
'User-Agent': 'request'
}
});
Related
TLDR: How do I log http 500 errors in Lambda?
I'm trying to log errors from my internal API method (not accessible to SNS subscription directly), invoked in Lambda, in CloudWatch. Here is my Lambda method in nodejs 6.10:
var http = require('http');
exports.handler = function(event, context, callback) {
var post_data = event.Records[0].Sns.Message;
var post_options = {
host: 'myhost.com',
port: 80,
path: '/path/to/api/',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': post_data.length
}
};
var post_request = http.request(post_options, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
callback(null, body);//This event happens, even when server returns 500
});
res.on('error', function(e) {
callback(e);//This event never gets called
});
});
post_request.on('error', function(e) {
callback(e);//This event never gets called either
});
post_request.end(post_data);
};
My API method is returning a 500. However, my post_request.on('error') or res.on('error') event is never executing. It always runs the res.on('end') instead.
I'd like to report the full error in cloudwatch, but instead it just gives me a generic error message, while saying the lambda method was successful.
It's a quirk with the what node considers an 'error'. As far as the http library is concerned, there was no error. It connected successfully to your api and successfully returned the result the server gave. The error handler will only fire with things like TCP errors or a mangled repsponse. If you want sensible behavior then you'll need to check the status code on the response.
var post_request = http.request(post_options, function(res) {
if (res.statusCode < 200 || res.statusCode > 299)
// handle non-200 https errors;
}
// more code
})
function delete(id, response) {
var https = require('https');
var linkpath = "/v1/endpoint/" + id + "/?token=" + AUTH_KEY;
var req = https.request({
hostname: 'api.foo.com',
port: 443,
path: linkpath,
agent: false,
method: 'DELETE',
}, (res) => {
if (res.statusCode !== 200) {
response.send('HTTP ' + res.statusCode + ' ' + res.statusMessage);
}
res.on('error', function (err) {
response.send(err);
});
res.on('end', function (data) {
response.send(data);
});
});
req.on('error', function(e) {
response.send(e.message);
});
req.end();
}
This code, adapted from my (working) code that uses a POST request to do other things with this API, nets me a status code of 500 from the endpoint.
I don't know how to debug this. I can't send the URL manually to the server because it's a DELETE operation instead of a GET or POST.
Has anyone seen this problem? Or do you have ideas on how to debug it?
Postman (https://www.getpostman.com/) is a great tool for manually sending specific HTTP requests, including DELETE!
There are all sorts of tools that will let you manually send any HTTP to the server. For instance, you can get quite a bit of information with curl, which will happily send a DELETE request.
For example:
curl -v -X "DELETE" https://jsonplaceholder.typicode.com/posts/1
will return the request and response headers as well as the body of the return value if any.
I am trying to create http get request from node, to get information from youtube URL. When I click it in browser I get json response but if I try it from node, I get ssl and other types of error. What I have done is,
this.getApiUrl(params.videoInfo, function (generatedUrl) {
// Here is generated URL - // https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus
console.log(generatedUrl);
var req = http.get(generatedUrl, function (response) {
var str = '';
console.log('Response is ' + response.statusCode);
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(str);
});
});
req.end();
req.on('error', function (e) {
console.log(e);
});
});
I get this error
{
"error": {
"message": "Protocol \"https:\" not supported. Expected \"http:\".",
"error": {}
}
}
When I make it without https I get this error,
Response is 403
{"error":{"errors":[{"domain":"global","reason":"sslRequired","message":"SSL is required to perform this operation."}],"code":403,"message":"SSL is required to perform this operation."}}
You need to use the https module as opposed to the http module from node, also I would suggest one of many http libraries that provide a higher level api such as wreck or restler which allow you to control the protocol via options as opposed to a different required module.
Your problem is obviously accessing content served securely with http request hence, the error. As I have commented in your question, you can make use of https rather than http and that should work but, you can also use any of the following approaches.
Using request module as follow:
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
});
Using https module you can do like below:
var https = require('https');
var options = {
hostname: 'www.googleapis.com', //your hostname youtu
port: 443,
path: '//youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus',
method: 'GET'
};
//or https.get() can also be used if not specified in options object
var req = https.request(options, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
res.on('data', function(d) {
process.stdout.write(d);
});
});
req.end();
req.on('error', function(e) {
console.error(e);
});
You can also use requestify module and
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
requestify.get(url).then(function(response) {
// Get the response body
console.log(response.body);
});
superagent module is another option
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
superagent('GET', url).end(function(response){
console.log('Response text:', response.body);
});
Last but not least is the unirest module allow you to make http/https request as simple as follow:
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
unirest.get(url).end(function(res) {
console.log(res.raw_body);
});
There might be more options out there. Obviously you need to load the modules using require before using it
var request = require('request');
var https = require('https');
var requestify = require('requestify');
var superagent = require('superagent');
var unirest = require('unirest');
I provided extra details, not only to answer the question but, also to help others who browse for similiar question on how to make http/https request in nodejs.
I can happily get the following url below CURL or the browser, but when I try it using node I get an ECONNREFUSED error.
Try this code...
var request = require('request')
var url = "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-ash3/t39.2365-6/851565_602269956474188_918638970_n.png"
var r = request({url:url}, function callback(error, response, body) {
console.log("Error:", error)
console.dir(r.headers)
})
I think your code works. I have change the code to test on some urls and log show the requests is ok.
So, i think you could paste the detail error log for deeper analysis !
This is my test code:
var request = require('request')
var urls = ["https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-ash3/t39.2365-6/851565_602269956474188_918638970_n.png",
"https://www.google.com",
"http://nodejs.org/"]
urls.forEach(function(url){
request({url:url}, function callback(error, response) {
console.log("Response for "+url);
if(error){
console.error(error);
}else{
console.log("statusCode: "+response.statusCode);
console.log("body.length: ", response.body.length + " bytes \n");
}
})
})
Console log:
Response for https://www.google.com
statusCode: 200
body.length: 44898 bytes
Response for http://nodejs.org/
statusCode: 200
body.length: 6318 bytes
Response for https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-ash3/t39.2365-6/851565_602269956474188_918638970_n.png
statusCode: 200
body.length: 722 bytes
I've tried the sample from the documentation and it works great.
But when I change the URL to https://api.mercadolibre.com/sites/, the request hangs. The only thing I get is:
{ [Error: socket hang up] code: 'ECONNRESET' }
Here's my code:
var https = require('https');
this.dispatch = function(req, res) {
var renderHtml = function(content) {
res.writeHead(200, {'Content-Type': 'text/html'});
res.end(content, 'utf-8');
}
var parts = req.url.split('/');
var options = {
host: 'api.mercadolibre.com',
port: 443,
path: '/sites/',
method: 'GET'
};
var request = https.request(options, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
res.on('data', function(d) {
process.stdout.write(d);
});
});
request.on('error', function(e) {
console.error('error');
console.error(e);
});
request.end();
return 'item id:' + parts[2];
};
I've tried with curl, soapui and with a browser. On all cases works great, but with node.js it doesn't.
How can I get more data on what's going on?
added
With curl i do: curl --sslv3 https://api.mercadolibre.com/sites/ works.
I've test same in centos 6 and works too.
I've reinstalled node, this time from source, same problem. My Os is ubuntu 12.04.
Thanks.
I'm not sure about api.mercadolibre.com site, but I can call API if I remove port param, like following code:
var options = {
host: 'api.mercadolibre.com',
path: '/sites/',
method: 'GET'
};
And we also need add param to support SSL version 3:
https.globalAgent.options.secureProtocol = 'SSLv3_method';
Why not use a library like request to deal with the details for you?
var request = require('request');
request('https://api.mercadolibre.com/sites/', {}, function(err, res, body) {
console.log("Got body: ", body);
});
This yields:
Got body: [{"id":"MLA","name":"Argentina"},{"id":"MLB","name":"Brasil"},{"id":"MCO","name":"Colombia"},{"id":"MCR","name":"Costa Rica"},{"id":"MEC","name":"Ecuador"},{"id":"MLC","name":"Chile"},{"id":"MLM","name":"Mexico"},{"id":"MLU","name":"Uruguay"},{"id":"MLV","name":"Venezuela"},{"id":"MPA","name":"Panamá"},{"id":"MPE","name":"Perú"},{"id":"MPT","name":"Portugal"},{"id":"MRD","name":"Dominicana"}]
Since it is working with curl, try using node-curl module. I lost a whole day trying to make it work in node.js with http and/or https modules until I switched to node-curl.
Try this:
var curl = require('node-curl');
curl('https://api.mercadolibre.com/sites/', {SSLVERSION: 3}, function(err, res) {
var body = res.body;
res.close();
console.log(body);
});
Same here, working with curl but not with node.js.
Problem: here on CentOS-5 curl usesthe provides openssl libraries and so uses centos standard /etc/pki/tls/certs/ca-bundle.crt for CA checks.
Where does node.js look for?, via strace there I cannot see any reference to a CA-file for checking.
Node.js request against server with valid SSL-certificate from well known old issuer are accepted, but not against my own webserver with own CA.
I put my own CA.crt in the ca-bundle.crt file, so now curl accepts it, but not node.js.
Only solution for now is to deactivate the verification-check for my dev-box:
var client = require('https');
var download_options = url.parse(sourceUrl);
download_options.method = "GET";
download_options.agent = false;
download_options.rejectUnauthorized = false; / HERE to accept all SSL-certificates */
var download_request = client.request(download_options);
I think you are behind a proxy which you need to specify to request. Proxy settings are detected automatically by libcurl, which node-curl uses. Therefore the request passes in node-curl.
Therefore, find out the proxy IP and port your organization uses, and try this:
var request = require('request');
request({
uri : 'https://mail.google.com/mail',
proxy : 'http://<proxy ip>:<proxy port>'
}, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body) // Print the google web page.
}else{
console.log(error);
console.log(response.statusCode);
}
})
You will get the ECONNRESET error if you do this:
post_options.path = 'history';
...
var req = http.request(post_options, function(res) {
...
That is, you need to make sure your path has a / like this:
post_options.path = '/history';
...