While working with Facebook graph api, I have used https.get to make a request for facebook user data.
var optionsP = {
host: 'graph.facebook.com',
path: '/me?access_token=XXXX'
};
https.get(optionsP, function(resp) {
resp.on('data', function(d) {
console.log('ondata')
console.log(d.length)
process.stdout.write(d)
});
}).on('error', function(e) {
console.error(e);
});
But the response data comes as 2 parts! First time it prints upto 1034 characters, then again same call back will work and prints remaining 1347 characters. What is the reason for these partial responses?
That's normal. resp is a stream. It's a ClientResponse object, that implements the readable stream interface. Here are the docs: http://nodejs.org/api/http.html#http_http_clientresponse
You can either pipe the output somewhere that accepts streams, or store it in a buffer until you receive the 'end' event.
Here is an example that stores the data in a String in memory, until it has all arrived:
https.get(optionsP, function(resp) {
resp.setEncoding(); //Now the data is a string!
var store = "";
resp.on('data', function(d) {
store += d;
});
resp.on('end', function() {
console.log("this is all: " + store);
});
}).on('error', function(e) {
console.error(e);
});
Related
I'm aware that there are several questions related to mine, but I didn't find any of them useful:
this one doesn't apply to my case, I'm actually getting the answer, it's the contents that I can't get.
on this one, on the other hand, the problem is a wrong handling of an asynchronous call, which is not my case
there, well, I really didn't fully understand this question
And so on...
Then, I think this is a legitimate question. I'm actually performing some encryption in my server (express routing in node) through a post request:
app.post('/encrypt', encrypt);
Encrypt is doing:
function encrypt(req,res) {
if(req.body.key && req.body.message) {
var encryptedMessage = Encrypter.encrypt(req.body.key,req.body.message);
return res.status(200).json({ message: encryptedMessage });
}
res.status(409).json({ message: 'the message could not be encrypted, no key found' });
}
}
So, I tested this via console.log, and it's working. When the server receives the request, the encrypted message is being generated.
At the same time, I'm testing my thing with mocha and I'm doing it like so:
describe('# Here is where the fun starts ', function () {
/**
* Start and stop the server
*/
before(function () {
server.listen(port);
});
after(function () {
server.close();
});
it('Requesting an encrypted message', function(done) {
var postData = querystring.stringify({
key : key,
message : message
});
var options = {
hostname: hostname,
port: port,
path: '/encrypt',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': postData.length
}
};
var req = http.request(options, function(res) {
res.statusCode.should.equal(200);
var encryptedMessage = res.message;
encryptedMessage.should.not.equal(message);
done();
});
req.on('error', function(e) {
//I'm aware should.fail doesn't work like this
should.fail('problem with request: ' + e.message);
});
req.write(postData);
req.end();
});
});
So, whenever I execute the tests, it fails with Uncaught TypeError: Cannot read property 'should' of undefined because res.message does not exist.
None of the res.on (data, end, events is working, so I suppose the data should be available from there. First I had this:
var req = http.request(options, function(res) {
res.statusCode.should.equal(200);
var encryptedMessage;
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
encryptedMessage = chunk.message;
});
encryptedMessage.should.not.equal(message);
done();
});
But res.on was never accessed (the console.log didn't show anything). I'm therefore a bit stuck here. I'm surely doing some basic stuff wrong, but I don't have a clue, and the many questions I found doesn't seem to apply to my case.
Weird enough, if I launch a test server and then I curl it
curl --data "key=secret&message=veryimportantstuffiabsolutellyneedtoprotect" localhost:2409/encrypt
Curl justs waits ad aeternam.
Actually I was doing it properly at the beginning, and the problem was indeed the same than in the second question I mentionned I was actually "clearing" my context with done() before the post data arrived. The solution is:
var req = http.request(options, function(res) {
res.statusCode.should.equal(200);
res.on('data', function(data) {
encryptedMessage = JSON.parse(data).message;
encryptedMessage.should.not.equal(message);
done();
});
});
In such a way that done() is only called when the data has been threated. Otherwise, mocha will not wait for the answer.
I'm using superagent to receive a notifications stream from a server
require('superagent')
.post('www.streaming.example.com')
.type('application/json')
.send({ foo: 'bar' })
.on('data', function(chunk) {
console.log('chunk:' + chunk); // nothing shows up
})
.on('readable', function() {
console.log('new data in!'); // nothing shows up
})
.pipe(process.stdout); // data is on the screen
For some reason data and readable events aren't registered, hovewer I can pipe data to the sceeen. How can I process data on the fly?
Looking at the source of pipe method, you can get access to the original req object and add listeners on it:
require('superagent')
.post('www.streaming.example.com')
.type('application/json')
.send({ foo: 'bar' })
.end().req.on('response',function(res){
res.on('data',function(chunk){
console.log(chunk)
})
res.pipe(process.stdout)
})
But this won't handle the redirection if any.
It looks like superagent doesn't return a real stream, but you can use something like through to process the data:
var through = require('through');
require('superagent')
.post('www.streaming.example.com')
.type('application/json')
.send({ foo: 'bar' })
.pipe(through(function onData(chunk) {
console.log('chunk:' + chunk);
}, function onEnd() {
console.log('response ended');
}));
(although you have to check if superagent won't first download the entire response before it sends data through the pipe)
I am trying to process Wikipedia articles, and want to receive a list of all Wikipedia articles. In order to do this I am frequently sending http requests to the Wikipedia API, which allows you to receive 500 titles at time and also returns an apcontinue string, which, when used in the following request, returns title starting from that string.
In order to do this, I am using the agentkeepalive module:
var http = require('http');
var Agent = require('agentkeepalive');
var keepaliveAgent = new Agent({
keepAlive: true,
maxSockets: 5,
timeout: 5000,
keepAliveTimeout: 3000
});
To send an http request to Wikipedia, I use the following code:
function wikipediaApiCall(params, callback) {
var options = {
host: 'en.wikipedia.org',
path: '/w/api.php?' + createParamString(params),
method: 'GET',
agent: keepaliveAgent
};
var callbackFunc = function(response) {
var err;
var str = '';
if (('' + response.statusCode).match(/^5\d\d$/)) {
err = new Error('Server error');
}
//another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
response.on('error', function (e) {
err = new Error('Request error');
});
response.on('timeout', function () {
err = new Error('Timeout');
response.abort();
callback(err);
});
response.on('end', function () {
var obj = JSON.parse(str);
if (obj.warnings) {
err = new Error('Request error');
}
callback(err, obj);
});
}
var req = http.request(options, callbackFunc);
req.setTimeout(5000);
req.on('error', function(err) {
callback(err, null);
return;
});
req.on('timeout', function () {
err = new Error('Timeout');
response.abort();
callback(err);
});
req.on('finish', function(){
console.log('ended');
});
req.end();
}
However, after sending between 16 and 20 request, I am not getting any response, but my request also does not time out.
Any ideas why this is happening?
Update
The request I send to Wikipedia contains the following parameters:
var params = {
list: 'allpages',
aplimit: limit,
apfrom: from,
continue: cont,
// apfilterredir: 'nonredirects'
};
Interestingly, after leaving out the nonredirects setting, I was able to send and receive up to 330 requests, but no more than that.
Update 2
I was able to register a finished event. It appears to be fired for the request that is failing as well. I modified the code accordingly.
Perhaps you need a bot flag to have higher API limits. Maybe there are too many requests in parallel; WMF recommendation is to make requests serially in case of such big tasks. Also, you should use the maxlag parameter with low values, per WMF API Etiquette.
I am trying to learn node.js.
I am trying to understand streams and piping.
Is it possible to pipe the response of http request to console.log?
I know how to do this by binding a handler to the data event but I am more interested in streaming it to the console.
http.get(url, function(response) {
response.pipe(console.log);
response.on('end', function() {
console.log('finished');
});
});
console.log is just a function that pipes the process stream to an output.
Note that the following is example code
console.log = function(d) {
process.stdout.write(d + '\n');
};
Piping to process.stdout does exactly the same thing.
http.get(url, function(response) {
response.pipe(process.stdout);
response.on('end', function() {
console.log('finished');
});
});
Note you can also do
process.stdout.write(response);
While playing with https in node.js, I have stucked in reading response data. Following is the code for https request;
https.get(options, function(resp) {
console.log(resp.headers) //working fine
resp.on('data', function(d) {
console.log(d) // buffered data; like <Buffer 7b 22 69...
process.stdout.write(d); // working fine(prints decoded data in console)
var decoded_data=??? });
}).on('error', function(e) {
console.error(e);
});
But, how can I decode response data & write it into a variable?
var decoded_data = d.toString('utf8');
or, earlier on:
resp.setEncoding('utf8');
and then all your on events will give you a string instead of a buffer.