I am trying to create http get request from node, to get information from youtube URL. When I click it in browser I get json response but if I try it from node, I get ssl and other types of error. What I have done is,
this.getApiUrl(params.videoInfo, function (generatedUrl) {
// Here is generated URL - // https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus
console.log(generatedUrl);
var req = http.get(generatedUrl, function (response) {
var str = '';
console.log('Response is ' + response.statusCode);
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(str);
});
});
req.end();
req.on('error', function (e) {
console.log(e);
});
});
I get this error
{
"error": {
"message": "Protocol \"https:\" not supported. Expected \"http:\".",
"error": {}
}
}
When I make it without https I get this error,
Response is 403
{"error":{"errors":[{"domain":"global","reason":"sslRequired","message":"SSL is required to perform this operation."}],"code":403,"message":"SSL is required to perform this operation."}}
You need to use the https module as opposed to the http module from node, also I would suggest one of many http libraries that provide a higher level api such as wreck or restler which allow you to control the protocol via options as opposed to a different required module.
Your problem is obviously accessing content served securely with http request hence, the error. As I have commented in your question, you can make use of https rather than http and that should work but, you can also use any of the following approaches.
Using request module as follow:
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
});
Using https module you can do like below:
var https = require('https');
var options = {
hostname: 'www.googleapis.com', //your hostname youtu
port: 443,
path: '//youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus',
method: 'GET'
};
//or https.get() can also be used if not specified in options object
var req = https.request(options, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
res.on('data', function(d) {
process.stdout.write(d);
});
});
req.end();
req.on('error', function(e) {
console.error(e);
});
You can also use requestify module and
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
requestify.get(url).then(function(response) {
// Get the response body
console.log(response.body);
});
superagent module is another option
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
superagent('GET', url).end(function(response){
console.log('Response text:', response.body);
});
Last but not least is the unirest module allow you to make http/https request as simple as follow:
var url = "https://www.googleapis.com/youtube/v3/videos?key=AIzaSyAm_1TROkfNgY-bBuHmSaletJhVQmkycJc&id=_H_r9qVrf24&part=id%2Csnippet%2CcontentDetails%2Cplayer%2Cstatistics%2Cstatus";
unirest.get(url).end(function(res) {
console.log(res.raw_body);
});
There might be more options out there. Obviously you need to load the modules using require before using it
var request = require('request');
var https = require('https');
var requestify = require('requestify');
var superagent = require('superagent');
var unirest = require('unirest');
I provided extra details, not only to answer the question but, also to help others who browse for similiar question on how to make http/https request in nodejs.
Related
I am trying to revoke a salesforce token from nodejs using an https request (both GET and POST methods tried).
This is my code for GET method
var token = user.token;
var uri = token.instanceUrl+'/services/oauth2/revoke?token='+token.accessToken;
console.log("data: "+postData+", options: "+JSON.stringify(postOptions)+ ", \r\n" + uri);
https.get(uri, function(response){
var buffer = '';
console.log(response.statusCode);
response.on('data', function (chunk) {
buffer += chunk;
});
response.on('end', function(){
console.log(buffer);
});
});
But i keep on getting this
error=unsupported_token_type&error_description=this%20token%20type%20is%20not%20supported, code: 400
I have also tried the address in browser and get a 400 Bad Request status.
All required options have been set according to salesforce online documentation https://help.salesforce.com/apex/HTViewHelpDoc?id=remoteaccess_revoke_token.htm&language=en
What am I missing which makes it a bad request?
I'm showing you how to do it using the request module:
var request = require('request')
request.post('https://login.salesforce.com/services/oauth2/revoke', {
form: {
token: '[ACCESS_TOKEN]'
},
followAllRedirects: true
}, function (err, res, body) {})
No idea what URL you are trying with, but the login URLs are identical for all users, you don't have to use your instance subdomain.
Also note this from their docs:
For a sandbox, use test.salesforce.com instead of login.salesforce.com.
I have been trying this out for a long time now.
I want to scrap contents from a subreddit that has adult contents.
But, the problem is that, you have to answer a simple question before you are given access to that page i.e. if you are 18+ or not.
I did some research on the source code and found that the solution is a simple POST request. where you need to send the parameters "over18=yes".
But my problem is that, I am not able to access the response body after the post.
Here's the code using http request in node. I have even tried it out with the node "request" module, but no help from that either.
Hoping to find someone who can help me out here.
var http = require("http");
var options = {
host: 'www.reddit.com',
port: 80,
path: '/over18?dest=http%3A%2F%2Fwww.reddit.com%2Fr%2Fnsfw&over18=yes',
method: 'POST'
};
var req = http.request(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
// write data to request body
req.write('data\n');
req.write('data\n');
req.end();
And here is the code using the Node Request module
var request = require("request");
request.post({url:'http://www.reddit.com/over18?dest=http%3A%2F%2Fwww.reddit.com%2Fr%2Fnsfw', form: {over18:'yes'}}, function(err,httpResponse,body){
console.log(body);
});
the URL i am trying to access is http://www.reddit.com/r/nsfw
In short, when you click YES button, the form sends over18=yes parameter to url http://www.reddit.com/over18?dest=http%3A%2F%2Fwww.reddit.com%2Fr%2Fnsfw using POST method. Then, server responds with an 302 Redirection header, cookie with value over18=1 and finally redirects to url http://www.reddit.com/r/nsfw using GET request. THen, server just checks if youa have a cookie with needed valuse.
All what you need is to do request directly to the final url with cookies using GET method.
var request = require("request");
var target = "http://www.reddit.com/r/nsfw";
var jar = request.jar();
var cookie = request.cookie("over18=1");
cookie.domain = "reddit.com";
cookie.path = "/";
jar.setCookie(cookie, target, function(error, cookie) {
console.log(error);
console.log(cookie);
});
request({
uri: target,
method: "GET",
jar: jar
}, function(error, response, body) {
console.log(response.statusCode);
console.log(body);
});
I too ran into this while ahem doing some research.. Here's my version:
var url = 'http://www.reddit.com/r/nsfw/';
var request = require('request');
request = request.defaults({jar: true });
request.post({
followAllRedirects: true,
url: 'http://www.reddit.com/over18?dest=' + encodeURIComponent(url),
form: {uh: '', over18: 'yes', }
}, function(err, httpResponse, html) {
…
});
Also worth a try are Reddit's Node.js APIs, of which I personally liked Snoocore.
I am trying to access opencorporates.com and using their REST API.
I got this code from How to make remote REST call inside Node.js? any CURL?. But it is not fetching any data. I tried wget on the url and it worked perfectly fine.
app.js
var https = require('http');
var optionsget = {
host : 'opencorporates.com',
port : 8080,
path : '/v0.2/companies/search?q=barclays+bank&jurisdiction_code=gb',
method : 'GET'
};
console.info('Options prepared:');
console.info(optionsget);
console.info('Do the GET call');
var reqGET = https.get(optionsget, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
res.on('data', function(d) {
console.info('GET result:\n');
process.stdout.write(d);
console.info('\n\nCall completed');
});
});
reqGet.end();
reqGet.on('error', function(e) {
console.error(e);
});
Because Node runs asynchronously, the returned data is broken into chunks.
The .on('data') event returns a portion of the data, which you then need to stitch/append back to a variable. You can then capture the complete output with .on('end').
See this example for more info: Why is node.js breaking incoming data into chunks? (#vossad01's answer)
That said, #SilviuBurcea's suggestion to use request is a much simpler way of handling http requests so you don't have to write and handle all of this yourself.
Try using request module. https://github.com/mikeal/request It's the http module on steroids.
Tried running the code locally, and first there is a capitalization error
var reqGET = https.get(optionsget, function(res) {
reqGet.end();
Second, the web address was not working at the address, nor with secure
var optionsget = {
host : 'api.opencorporates.com',
port : 80,
path : '/v0.2/companies/search?q=barclays+bank&jurisdiction_code=gb',
method : 'GET'
};
Its worth noting that if you wanted to actually use https, you would need to change the require line
var https = require('https');
This is fully functional version for your reference:
var http = require('http');
var optionsget = {
host : 'api.opencorporates.com',
port : 80,
path : '/v0.2/companies/search?q=barclays+bank&jurisdiction_code=gb',
method : 'GET'
};
console.info('Options prepared:');
console.info(optionsget);
console.info('Do the GET call');
var reqGet = http.get(optionsget, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
buffer='';
res.on('data', function(d) {
//console.info('GET result:\n');
//process.stdout.write(d);
buffer += d.toString();
//console.info('\n\nCall completed');
});
res.on('end', function() {
console.info('GET result:\n');
console.log(buffer);
console.info('\n\nCall completed');
});
});
reqGet.on('error', function(e) {
console.error(e);
});
reqGet.end();
I've tried the sample from the documentation and it works great.
But when I change the URL to https://api.mercadolibre.com/sites/, the request hangs. The only thing I get is:
{ [Error: socket hang up] code: 'ECONNRESET' }
Here's my code:
var https = require('https');
this.dispatch = function(req, res) {
var renderHtml = function(content) {
res.writeHead(200, {'Content-Type': 'text/html'});
res.end(content, 'utf-8');
}
var parts = req.url.split('/');
var options = {
host: 'api.mercadolibre.com',
port: 443,
path: '/sites/',
method: 'GET'
};
var request = https.request(options, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
res.on('data', function(d) {
process.stdout.write(d);
});
});
request.on('error', function(e) {
console.error('error');
console.error(e);
});
request.end();
return 'item id:' + parts[2];
};
I've tried with curl, soapui and with a browser. On all cases works great, but with node.js it doesn't.
How can I get more data on what's going on?
added
With curl i do: curl --sslv3 https://api.mercadolibre.com/sites/ works.
I've test same in centos 6 and works too.
I've reinstalled node, this time from source, same problem. My Os is ubuntu 12.04.
Thanks.
I'm not sure about api.mercadolibre.com site, but I can call API if I remove port param, like following code:
var options = {
host: 'api.mercadolibre.com',
path: '/sites/',
method: 'GET'
};
And we also need add param to support SSL version 3:
https.globalAgent.options.secureProtocol = 'SSLv3_method';
Why not use a library like request to deal with the details for you?
var request = require('request');
request('https://api.mercadolibre.com/sites/', {}, function(err, res, body) {
console.log("Got body: ", body);
});
This yields:
Got body: [{"id":"MLA","name":"Argentina"},{"id":"MLB","name":"Brasil"},{"id":"MCO","name":"Colombia"},{"id":"MCR","name":"Costa Rica"},{"id":"MEC","name":"Ecuador"},{"id":"MLC","name":"Chile"},{"id":"MLM","name":"Mexico"},{"id":"MLU","name":"Uruguay"},{"id":"MLV","name":"Venezuela"},{"id":"MPA","name":"Panamá"},{"id":"MPE","name":"Perú"},{"id":"MPT","name":"Portugal"},{"id":"MRD","name":"Dominicana"}]
Since it is working with curl, try using node-curl module. I lost a whole day trying to make it work in node.js with http and/or https modules until I switched to node-curl.
Try this:
var curl = require('node-curl');
curl('https://api.mercadolibre.com/sites/', {SSLVERSION: 3}, function(err, res) {
var body = res.body;
res.close();
console.log(body);
});
Same here, working with curl but not with node.js.
Problem: here on CentOS-5 curl usesthe provides openssl libraries and so uses centos standard /etc/pki/tls/certs/ca-bundle.crt for CA checks.
Where does node.js look for?, via strace there I cannot see any reference to a CA-file for checking.
Node.js request against server with valid SSL-certificate from well known old issuer are accepted, but not against my own webserver with own CA.
I put my own CA.crt in the ca-bundle.crt file, so now curl accepts it, but not node.js.
Only solution for now is to deactivate the verification-check for my dev-box:
var client = require('https');
var download_options = url.parse(sourceUrl);
download_options.method = "GET";
download_options.agent = false;
download_options.rejectUnauthorized = false; / HERE to accept all SSL-certificates */
var download_request = client.request(download_options);
I think you are behind a proxy which you need to specify to request. Proxy settings are detected automatically by libcurl, which node-curl uses. Therefore the request passes in node-curl.
Therefore, find out the proxy IP and port your organization uses, and try this:
var request = require('request');
request({
uri : 'https://mail.google.com/mail',
proxy : 'http://<proxy ip>:<proxy port>'
}, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body) // Print the google web page.
}else{
console.log(error);
console.log(response.statusCode);
}
})
You will get the ECONNRESET error if you do this:
post_options.path = 'history';
...
var req = http.request(post_options, function(res) {
...
That is, you need to make sure your path has a / like this:
post_options.path = '/history';
...
How would I go about downloading the contents of a URL in Node when using the Express framework? Basically, I need to complete the Facebook authentication flow, but I can't do this without GETing their OAuth Token URL.
Normally, in PHP, I'd use Curl, but what is the Node equivalent?
var options = {
host: 'www.google.com',
port: 80,
path: '/index.html'
};
http.get(options, function(res) {
console.log("Got response: " + res.statusCode);
}).on('error', function(e) {
console.log("Got error: " + e.message);
});
http://nodejs.org/docs/v0.4.11/api/http.html#http.get
The problem that you will front is: some webpage loads its contents using JavaScript. Thus, you needs a package, like After-Load which simulates browser's behavior, then gives you the HTML content of that URL .
var afterLoad = require('after-load');
afterLoad('https://google.com', function(html){
console.log(html);
});
Using http way requires way more lines of code for just a simple html page .
Here's an efficient way : Use request
var request = require("request");
request({uri: "http://www.sitepoint.com"},
function(error, response, body) {
console.log(body);
});
});
Here is the doc for request : https://github.com/request/request
2nd Method using fetch with promises :
fetch('https://sitepoint.com')
.then(resp=> resp.text()).then(body => console.log(body)) ;
Using http module:
const http = require('http');
http.get('http://localhost/', (res) => {
let rawHtml = '';
res.on('data', (chunk) => { rawHtml += chunk; });
res.on('end', () => {
try {
console.log(rawHtml);
} catch (e) {
console.error(e.message);
}
});
});
rawHtml - complete html of the page.
I just simplified example from official docs.
using Axios is much simpler
const axios = require("axios").default
const response = axios.get("https://google.com")
console.log(response.data)
or
const axios = require("axios").default
const response = axios.get("https://google.com").then((response)=>{
console.log(response.data)
})
for full docs, you can head over Axios Github