Encoding issue with requesting JSON from StackOverflow API - node.js

I can't figure this out for the life of me. Below is an implementation with the request module, but I've also tried with the node-XMLHttpRequest module to no avail.
var request = require('request');
var url = 'http://api.stackexchange.com/2.1/questions?pagesize=100&fromdate=1356998400&todate=1359676800&order=desc&min=0&sort=votes&tagged=javascript&site=stackoverflow';
request.get({ url: url }, function(error, response, body) {
if (error || response.statusCode !== 200) {
console.log('There was a problem with the request');
return;
}
console.log(body); // outputs gibberish characters like �
console.log(body.toString()); // also outputs gibberish
});
Seems to be an encoding issue, but I've used the exact same code (with native XHR objects) in the browser and it works without problems. What am I doing wrong?

The content is gzipped. You can use request and zlib to decompress a streamed response from the API:
var request = require('request')
,zlib = require('zlib');
var url = 'http://api.stackexchange.com/2.1/questions?pagesize=100&fromdate=1356998400&todate=1359676800&order=desc&min=0&sort=votes&tagged=javascript&site=stackoverflow';
request({ url: url, headers: {'accept-encoding': 'gzip'}})
.pipe(zlib.createGunzip())
.pipe(process.stdout); // not gibberish
Further Reading: Easy HTTP requests with gzip/deflate compression

While pero's answer is correct, there's a simpler way to do this.
Since you're using request, you can also just add the gzip: true flag:
var request = require('request');
var url = 'http://api.stackexchange.com/2.1/questions?pagesize=100&fromdate=1356998400&todate=1359676800&order=desc&min=0&sort=votes&tagged=javascript&site=stackoverflow';
request.get({ url: url, headers: {'accept-encoding': 'gzip'}, gzip: true }, function(error, response, body) {
console.log(body); // not gibberish
});

Related

Request module return unicode characters

I'm trying to get normal response using request module in node.js
and i've got problem to get response from amazon.com as normal string
i dont know why, but ive got problem only with amazon.com (eg. amazon.it, amazon.co.uk does return normal string).
const request = require('request');
request.get(
{
uri: 'https://www.amazon.com',
encoding: 'utf-8'
},
function (error, response, body) {
console.log(body)
});
The code above return something like:
b��╝��W>�S�Uk��z�=8~r����9|r|P^?}p o╗��l���ߋ�t`ޜ^]��n!��
���U�>>�#�w-z�.��O�����Oo��������y�����g�N�/��{����_>���鳟�=s���w?�z��_W)i؃�
��;���2��9<�0ٷ8����<=�ϱ��ղ��3�=(�"�ԯ�; �3��=�8�2;=���28����#+,3��0"�+DZ �)�2�<�
���7�(W?�8�9\?�)#'���";�ķ���ܣ�ѽ����|�8 ��╚ ��'
The response returned by Amazon is gzipped. You have to provide the gzip option to your request.
const request = require('request');
request.get(
{
uri: 'https://www.amazon.com',
gzip: true,
},
function (error, response, body) {
console.log(body)
});

Send unparsed data in a Node JS request

I want to send a POST request (for example, with the 'request' module), but I don't find a way of sending unparsed data*.
*unparsed data => copied directly from the Chrome dev tool. Something like: tipo_accion=3&filtro=descarga&fecha=actual
It would also do the trick some way of translating that string to JSON.
I've tried so far with...
var request = require('request');
request.post({ url: 'https://target.com/form/', form: 'tipo_accion=3&filtro=descarga&fecha=actual' },
function (error, response, body) {
console.log(body)
}
);
... but it didn't work.
Firstly you should understand the difference between the request methods post and get.
The structure that you want to send:
tipo_accion=3&filtro=descarga&fecha=actual is telling me that you want to use a get request. So the proper code for that will be something like:
request(
'https://target.com/form/&tipo_accion=3&filtro=descarga&fecha=actual',
function (error, response, body) {
console.log(body)
},
);
But if it is a post request then you should use the json format
request.post({
url: 'https://target.com/form/', form: {
tipo_accion: 3,
filtro: 'descarga',
fecha: 'actual'
},
function(error, response, body) {
console.log(body)
}
);
You can convert form_data to string format and it works for me you can try it :
const request = require('request-promise');
var j = request.jar()
var data_fom = `a=value_a&b=value_b&c=value_c`
request({
url:"https://target.com/form/",
jar:j,
method:"POST",
resolveWithFullResponse: true,
form:data_form
})

How do I convert result from opening gzip file to string

var request = require('request')
request(
{ method: 'GET'
, uri: 'http://www.examplewebsite.com'
, gzip: true
}
, function (error, response, body) {
console.log(body); //I am trying print body here
console.log(response.body); // //I am trying print body here too
}
)
All I received is just �V*.I,)-V�*)*M�QJI,IT��V��,.���%�E��)JV�����d��$1
If I try to use JSON.stringify(body), the result is:
\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\u0003�V*.I,)-V�*)*M�QJI,IT��V��,.\u0001��\u0019�\u0005%�E��)JV�����\u0000d��$1\u0000\u0000\u0000
All I want is to see the plain string. How do I do that?
I would expect that setting gzip : true would decompress the response body automatically, but perhaps the server doesn't set the correct Content-Encoding header.
In that case, you can try this:
const zlib = require('zlib');
const request = require('request');
request({
method : 'GET',
uri : 'http://www.examplewebsite.com',
gzip : true,
encoding : null,
}, function (error, response, body) {
let decompressed = zlib.gunzipSync(body).toString();
...
})

Nodejs request module return incorrect statusCode

I'm trying to use request module to get HTTP status code. But sometimes it works wrong. For example:
var request = require('request');
var requestSettings = {
method: 'HEAD',
url: 'http://stackoverflow.com/help'
};
request(requestSettings, function(error, response, body) {
console.log(response.statusCode);
});
This code returns 404 (not 200 as expected).
Why might this happen?
Thanks.
You're doing everything fine. stackoverflow didn't implement HEAD

invalid oauth2 token request

I'm developing a node application which needs to authenticate with google. When I request a token, https://accounts.google.com/o/oauth2/token responds with:
error: 400
{
"error" : "invalid_request"
}
I've tried making the same request in curl, and have received the same error, so I suspect there is something wrong with my request but I can't figure out what. I've pasted my code below:
var request = require('request');
var token_request='code='+req['query']['code']+
'&client_id={client id}'+
'&client_secret={client secret}'+
'&redirect_uri=http%3A%2F%2Fmassiveboom.com:3000'+
'&grant_type=authorization_code';
request(
{ method: 'POST',
uri:'https://accounts.google.com/o/oauth2/token',
body: token_request
},
function (error, response, body) {
if(response.statusCode == 201){
console.log('document fetched');
console.log(body);
} else {
console.log('error: '+ response.statusCode);
console.log(body);
}
});
I've triple checked to make sure all the data I'm submitting is correct and i'm still getting the same error. What can I do to debug this further?
It turns out that request.js (https://github.com/mikeal/request) doesn't automatically include the content-length to the headers. I added it manually and it worked on the first try. I've pasted the code below:
exports.get_token = function(req,success,fail){
var token;
var request = require('request');
var credentials = require('../config/credentials');
var google_credentials=credentials.fetch('google');
var token_request='code='+req['query']['code']+
'&client_id='+google_credentials['client_id']+
'&client_secret='+google_credentials['client_secret']+
'&redirect_uri=http%3A%2F%2Fmyurl.com:3000%2Fauth'+
'&grant_type=authorization_code';
var request_length = token_request.length;
console.log("requesting: "+token_request);
request(
{ method: 'POST',
headers: {'Content-length': request_length, 'Content-type':'application/x-www-form-urlencoded'},
uri:'https://accounts.google.com/o/oauth2/token',
body: token_request
},
function (error, response, body) {
if(response.statusCode == 200){
console.log('document fetched');
token=body['access_token'];
store_token(body);
if(success){
success(token);
}
}
else {
console.log('error: '+ response.statusCode);
console.log(body)
if(fail){
fail();
}
}
}
);
}
from here How to make an HTTP POST request in node.js? you could use querystring.stringify to escape query string of request parameters. Plus you'd better add 'Content-Type': 'application/x-www-form-urlencoded' for POST request.
post here the final string generated from token_request var.that may have something wrong. or may be authentication code is expired or not added correctly to the URL. Usually code has '/' in it that needs to escaped.

Resources