I'm currently messing around with NodeJS and try to use the Twitter-Search API. With curl it works fine - so there's nothing wrong with my firewall or whatever. Yet, I never get a response within NodeJS.
var sys = require("sys"),
http = require("http"),
events = require("events");
sys.puts("Hi there… ");
var client = http.createClient(80, "search.twitter.com"),
body = "",
query = "foobar";
function getResults() {
sys.puts("fetching for "+query);
var request = client.request("GET", "/search.json?q="+query);
request.on("response", function(data){
/* this somehow never gets fired :( */
sys.puts("BODY:"+ data);
});
}
var interval = setInterval(getResults, 5000);
And the URL is also working.
Any hints or solutions are welcome!
Thanks in advance.
You never send the request.
You need to use request.end()
NOTE: the request is not complete. This method only sends the header of the request. One needs to call request.end() to finalize the request and retrieve the response. (This sounds convoluted but it provides a chance for the user to stream a body to the server with request.write().)
Also the response event'S parameter is the response object NOT the body. You need to set the data event on the response object and then listen for the end event to make sure you got all the data.
request.on('response', function (response) {
var body = '';
response.on('data', function (chunk) {
body += chunk;
});
response.on('end', function () {
console.log('BODY: ' + body);
});
});
request.end(); // start the request
See: http://nodejs.org/api/http.html#http_class_http_clientrequest
A few more tips
You might want to use querystring.escape to urlencode your search parameter
You also need to set the Host header, otherwise Twitter will return a 404
Fixed code:
var querystring = require('querystring');
...
var request = client.request("GET", "/search.json?q=" + querystring.escape(query), {'Host': 'search.twitter.com'});
Related
i was in the process of answering this question
Retrieving the http.ServerResponse body
im trying to capture the raw string of an https request and response, for http it works fine
var http = require('http');
var util = require('util');
var req,res;
var server = http.createServer(onrequest);
server.listen(81);
server.on('connection',socket=>{
console.log('*** connection ***');
req = '';
res = '';
socket.on('data',data=>{
req += data.toString();
});
var write = socket.write
socket.write=function(data,encoding,callback){
res += data.toString();
write.apply(socket,arguments);
}//write
});
function onrequest(request,response){
console.log('*** request : '+request.url+' ***');
response.on('finish',()=>{
console.log('--- request ---');
console.log('['+req+']');
console.log();
console.log('--- response ---');
console.log('['+res+']');
});
request.on('data',data=>{});
request.on('end',()=>response.end('Hi. This is my http response.'));
}//onresquest
simplifying things using a single curl request
curl http://localhost:81/
gives the required data, congrats i thought onto my project and .. nope
when using https with the following code
var key; // = .....
var cert; // = ....
var server = https.createServer({key,cert},onrequest);
and again using curl
curl -k https://localhost:81/
no data is logged, the problem is this
on the connection event the socket passed is of type "socket" presumably a net.socket
the socket that is passed to the onrequest function is of type "tlssocket"
so i thought maybe i could use the underlying "duplex" which they do both share, however when i try to add the on data event to the duplex, with either
//on connection
var duplex = socket.__proto__;
duplex.on('data',data=>{
or
duplex.on.call(duplex,'data',data=>
it fails at Node.js JavaScript runtime
does anybody know how to get further than this?
related questions :
Get raw HTTP response in NodeJS
How can I get the raw HTTP message body using the request library in Node.js?
okey dokey
the solution is to use the "secureConnection" event, like so :
var https = require('https');
var req,res;
var key; // = ...
var cert; // = ...
var server = https.createServer({key,cert},onrequest);
server.listen(81);
server.on('secureConnection',socket=>{
console.log('*** connection ***');
req = '';
res = '';
socket.on('data',data=>{
req += data.toString();
});
var write = socket.write
socket.write=function(data,encoding,callback){
res += data.toString();
write.apply(socket,arguments);
}//write
});
function onrequest(request,response){
console.log('*** request : '+request.url+' ***');
response.on('finish',()=>{
console.log('--- request ---');
console.log('['+req+']');
console.log();
console.log('--- response ---');
console.log('['+res+']');
});
request.on('data',data=>{});
request.on('end',()=>response.end('Hi. This is my http response.'));
}//onresquest
this can be seen implemented
github: simple-server - no dependencies
I have a NodeJS API. The logic in the API needs to make an http get request to google.com, capture the response from google.com, and then return the html response to the original API call. My problem is capturing the http response from google asynchronously and returning it to the original API call.
// Entry point to /api/specialday
module.exports = function(apiReq, apiRes, apiNext) {
var options = {
host: 'www.google.com'
};
callback = function(googleRes) {
var str = '';
// another chunk of data has been recieved, so append it to `str`
googleRes.on('data', function (chunk) {
str += chunk;
});
// capture the google response and relay it to the original api call.
googleRes.on('end', function () {
apiRes.send(str);
});
}
http.request(options, callback).end();
}
The error I get here is Uncaught TypeError: Cannot read property 'send' of undefined. I understand why I'm getting the error (because apiRes is out of scope), I just can't figure out how to do it right. Any help much appreciated!
The reason you are seeing the above error is because the original response object apiRes is gone by the time you have received the response from the google API.
As far as I can tell you will have to bind() the apiRes twice (untested):
callback = function(googleRes) {
var str = '';
// another chunk of data has been recieved, so append it to `str`
googleRes.on('data', function (chunk) {
str += chunk;
});
// capture the google response and relay it to the original api call.
googleRes.on('end', function () {
apiRes.send(str);
}.bind(apiRes));
}.bind(apiRes)
A more modern solution would be to use promises for this task https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Promise
Promises, that's it! Thanks Michal. Below is a simplified version of my implementation.
// Entry point to /api/specialday
module.exports = function(apiReq, apiRes, apiNext) {
var p1 = new Promise(
// The resolver function is called with the ability to resolve or
// reject the promise
function(resolve, reject) {
var options = {
host: 'www.google.com'
};
callback = function(googleRes) {
var str = '';
// another chunk of data has been recieved, so append it to `str`
googleRes.on('data', function (chunk) {
str += chunk;
});
// capture the google response and relay it to the original api call.
googleRes.on('end', function () {
resolve(str);
});
}
http.request(options, callback).end();
)};
p1.then(function(googleHtml) {
apiRes.status(200).send(googleHtml);
}
}
Then I can run my app and call the api using Postman at http://localhost:8080/api/gains:
Directly pipe output with apiRes, sample using request :
var request = require("request");
// Entry point to /api/specialday
module.exports = function(apiReq, apiRes, apiNext) {
request.get('http://www.google.fr').pipe(apiRes);
});
This is a different question and I am unable to get a solution for this, please do not mark it duplicate.
I cannot access variable op outside the function. Should I be using a async module of nodjes?
I have two console.logs. But only inside function log works.
I have tried other questions answers. Still it is not working
var http = require('http');
console.log("hi")
var options = {
host: 'api.usergrid.com',
path: '/siddharth1/sandbox/restaurants'
};
var op = []; //declaring outside function
var req = http.get(options, function(res) {
// Buffer the body entirely for processing as a whole.
var bodyChunks = [];
res.on('data', function(chunk) {
// You can process streamed parts here...
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
// ...and/or process the entire body here.
var body2 = JSON.parse(body);
op = body2.entities.map(function(item) {
return item.name;
});
console.log(op); // only this works
})
});
req.on('error', function(e) {
console.log('ERROR: ' + e.message);
});
console.log("outside function " + op); //this doesnt work
console.log('Server listening on port 80');
Node.js instantiates the variable op as an empty array:
var op = []; //declaring outside function
It then calls the .get() function of the http module, and passes it options and a callback function.
var req = http.get(options, function(res) {
...
});
The code inside the callback function is not executed until an HTTP GET request is received by your application.
Node then continues, and executes the remainder of your code:
console.log("outside function " + op); //this doesnt work
The above line is executed and indeed, op is an empty array as you defined it to be an empty array - and nothing has yet modified 'op'.
The server then idles, waiting for any incoming HTTP requests.
Much later, you of course issue a HTTP GET request to your server. The callback function you registered gets called, and the code inside that function executes.
If I were you I would look into some fundamental tutorials on Node.js, specifically looking into it's non-blocking model. Good luck.
Note: Ryan Dahl's original node.js presentation is a rather long video and a little old, but perfectly explains the way Node.js works, and I highly recommend you give it a watch.
I'm using Node.js and connect to create a simple web server. I have something similar to the following code and I can't figure out how to access the actual request message body from the request object. I'm new to this so bear with me. I'm also taking out some of the stuff that's not necessary for the example.
function startServer(dir) {
var port = 8888,
svr = connect().use(connect.static(dir, {"maxAge" : 86400000}))
.use(connect.directory(dir))
/*
* Here, I call a custom function for when
* connect.static can't find the file.
*/
.use(custom);
http.createServer(svr).listen(port);
}
function custom(req, res) {
var message = /* the message body in the req object */;
// Do some stuff with message...
}
startServer('dirName');
Make sense? I've tried logging that object to the console and it is full of TONS of stuff. I can easily see headers in there plus the request URL and method. I just can't seem to isolate the actual message body.
You should include the connect.bodyParser middleware as well:
svr = connect().use(connect.static(dir, {"maxAge" : 86400000}))
.use(connect.directory(dir))
.use(connect.bodyParser())
.use(custom);
That will provide the parsed message body as req.body to your handler.
If you want the raw message body, you shouldn't use it but instead read the req stream yourself:
function custom(req, res) {
var chunks = [];
req.on('data', function(chunk) {
chunks.push(chunk);
});
req.on('end', function() {
var rawbody = Buffer.concat(chunks);
...do stuff...
// end the request properly
res.end();
});
}
if(req.method == "POST"){
var body = '';
req.on('data', function(data){
body += data;
});
}
Then body should contain your message if you posted correctly.
A better idea would be to use Express, then use the bodyparser middleware - which will give you this functionality out of the box without worrying about somebody hammering your server. The code above has NO functionality to worry about attacks - but it will get you started.
app.get('/', function(req, res){
var options = {
host: 'www.google.com'
};
http.get(options, function(http_res) {
http_res.on('data', function (chunk) {
res.send('BODY: ' + chunk);
});
res.end("");
});
});
I am trying to download google.com homepage, and reprint it, but I get an "Can't use mutable header APIs after sent." error
Anyone know why? or how to make http call?
Check out the example here on the node.js doc.
The method http.get is a convenience method, it handles a lot of basic stuff for a GET request, which usually has no body to it. Below is a sample of how to make a simple HTTP GET request.
var http = require("http");
var options = {
host: 'www.google.com'
};
http.get(options, function (http_res) {
// initialize the container for our data
var data = "";
// this event fires many times, each time collecting another piece of the response
http_res.on("data", function (chunk) {
// append this chunk to our growing `data` var
data += chunk;
});
// this event fires *one* time, after all the `data` events/chunks have been gathered
http_res.on("end", function () {
// you can use res.send instead of console.log to output via express
console.log(data);
});
});