I wish to connect to the list of news feed urls via node and get the real-time data via socket.io. For that I tried with single url in server.js as below:
var http = require("http");
var options = {
host: 'http://economictimes.feedsportal.com/c/33041/f/534037/'
};
http.get(options, function (http_res) {
// initialize the container for our data
var data = "";
// this event fires many times, each time collecting another piece of the response
http_res.on("data", function (chunk) {
// append this chunk to our growing `data` var
data += chunk;
});
// this event fires *one* time, after all the `data` events/chunks have been gathered
http_res.on("end", function () {
// you can use res.send instead of console.log to output via express
console.log(data);
});
});
When I execute node server.js, it throws me an error
"Error: getaddrinfo ENOTFOUND
at errnoException (dns.js:37:11)
at Object.onanswer [as oncomplete] (dns.js:124:16)"
Is there any way to pass each news feed url from an array to connect it via node and get latest news via socket.io ???
From the node doc for the http module, this is what a typical options object looks like:
var options = {
hostname: 'www.google.com',
port: 80,
path: '/upload',
method: 'POST'
};
Per the doc, the host option you are using should be A domain name or IP address of the server to issue the request to. Defaults to 'localhost'. So, it looks like you just aren't calling .get() correctly.
If you just want to pass the whole URL, then don't use the options object, just pass the URL like this and the method will parse the URL for you into the relevant parts:
http.get('http://economictimes.feedsportal.com/c/33041/f/534037/', function (http_res) {...});
Related
I'm trying to use streams to send data to the browser with Hapi, but can't figure our how. Specifically I am using the request module. According to the docs the reply object accepts a stream so I have tried:
reply(request.get('https://google.com'));
The throws an error. In the docs it says the stream object must be compatible with streams2, so then I tried:
reply(streams2(request.get('https://google.com')));
Now that does not throw a server side error, but in the browser the request never loads (using chrome).
I then tried this:
var stream = request.get('https://google.com');
stream.on('data', data => console.log(data));
reply(streams2(stream));
And in the console data was outputted, so I know the stream is not the issue, but rather Hapi. How can I get streaming in Hapi to work?
Try using Readable.wrap:
var Readable = require('stream').Readable;
...
function (request, reply) {
var s = Request('http://www.google.com');
reply(new Readable().wrap(s));
}
Tested using Node 0.10.x and hapi 8.x.x. In my code example Request is the node-request module and request is the incoming hapi request object.
UPDATE
Another possible solution would be to listen for the 'response' event from Request and then reply with the http.IncomingMessage which is a proper read stream.
function (request, reply) {
Request('http://www.google.com')
.on('response', function (response) {
reply(response);
});
}
This requires fewer steps and also allows the developer to attach user defined properties to the stream before transmission. This can be useful in setting status codes other than 200.
2020
I found it !! the problem was the gzip compression
to disable it just for event-stream you need provide the next config to Happi server
const server = Hapi.server({
port: 3000,
...
mime:{
override:{
'text/event-stream':{
compressible: false
}
}
}
});
in the handler I use axios because it support the new stream 2 protocol
async function handler(req, h) {
const response = await axios({
url: `http://some/url`,
headers: req.headers,
responseType: 'stream'
});
return response.data.on('data',function (chunk) {
console.log(chunk.toString());
})
/* Another option with h2o2, not fully checked */
// return h.proxy({
// passThrough:true,
// localStatePassThrough:true,
// uri:`http://some/url`
// });
};
I am new to node.js and I am trying to make an http request. I followed a tutorial trying to call random.org. this is my app.js file:
var http = require('http');
//The url we want is: 'www.random.org/integers/?num=1&min=1&max=10&col=1&base=10&format=plain&rnd=new'
var options = {
host: 'www.random.org',
path: '/integers/?num=1&min=1&max=10&col=1&base=10&format=plain&rnd=new'
};
callback = function(response) {
var str = '';
//another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
//the whole response has been recieved, so we just print it out here
response.on('end', function () {
console.log(str);
});
response.on('error', function () {
console.log("err");
});
}
http.request(options, callback).end();
So, anyway, when running this file using
node app.js
I get the following error to the console:
C:\Program Files\nodejs>node app.js
events.js:72
throw er; // Unhandled 'error' event
^
Error: connect ETIMEDOUT
at errnoException (net.js:901:11)
at Object.afterConnect [as oncomplete] (net.js:892:19)
2 problems:
1. Why am I getting time out error (the site works - I checked)
2. Anyway - why am I not catching this error althought I have an error listener in the response.
Any possible help will be much appreciated!
Thanks
You are attaching the error listener to the response, but error happens in request itself. Do someting like this :
http.request(options, callback)
.on('error', function () {
console.log("err in request");
})
.end();
The error means the site is available, if not the error would show EHOSTUNREACH. ETIMEDOUT means that request was not responded fast enough. It needs some investigation, lke is the network down, try getting www.google.com. What is the timeout value it is considering ? etc.
OK. I tried to run this as standalone node js process without browser as background.
This is not recommended, and in the cases specified above the problem was that this headerless request lacked cookies.
If I would wrap everything in a createServer method, and upon request I would do the http.request with the same parameters + the cookies from the server request - this would work.
I am new to Node.js, learning with examples.
Here is what I am trying to do, I have a geoserver running to serve GeoJson, I want to call geoserver WFS url and get json data using node.js. Here is code, when I run it, I get :
getaddrinfo ENOENT
var http = require('http');
var options = {
host: "local:8080/geoserver/wfs?service=WFS&version=1.0.0&request=GetFeature&typeName=layername&outputFormat=JSON&cql_filter=id=1";
path: '/'
}
var request = http.request(options, function (res) {
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
console.log(data);
});
});
request.on('error', function (e) {
console.log(e.message);
});
request.end();
Please guide me in right direction. Thank you.
You need to pass in the correct options:
host - should only be the host name
path - should the path to the resource on the host (all the stuff you have after the host name
method - should be GET or POST (GET in your case).
var options = {
host: "local:8080";
path: '/geoserver/wfs?service=WFS&version=1.0.0&request=GetFeature&typeName=layername&outputFormat=JSON&cql_filter=id=1',
method: 'GET'
}
I was also trying to get a GeoJSON via Node.js and took a different approach; I used Express and sequelizejs. This allowed me to get objects directly from Postgres / PostGIS. I needed to do a little formatting client-side to form a valid GeoJSON from the express response.
I'm trying to catch ECONNREFUSED errors when using a HTTP client in node.js. I'm making requests like this:
var http = require('http');
var options = { host: 'localhost', port: '3301', path: '/', method: 'GET' };
http.request(options).on('response', function (res) {
// do some stuff
});
I can't figure out how to catch this error:
Error: connect ECONNREFUSED
at errnoException (net.js:614:11)
at Object.afterConnect [as oncomplete] (net.js:605:18)
If I do request.on('error', function () {});, it doesn't catch it. If I do it like this:
var req = request.on(etc)
req.on('error', function blah () {});
Then I get TypeError: Object false has no method 'on'.
Do I really have to do a top-level uncaught error thing to deal with this? At the moment whatever I do my whole process quits out.
Edit: I found some blog posts on how to do it by creating a connection object, calling request on that, and then binding to errors on the connection object, but doesn't that make the entire http.request() shortcut useless?
Any reason you're not using http://nodejs.org/docs/v0.6.5/api/http.html#http.request as your base? Try this:
var req = http.request(options, function(res) {
// Bind 'data', 'end' events here
});
req.on('error', function(error) {
// Error handling here
});
req.end();
Each call to http.request() returns its self.
So try it like this...
http.request(options.function(){}).on('error',function(){}).end();
I've got a solution for this, having tried all the suggestions on this (and many other) pages.
My client needs to detect a turnkey product that runs embedded windows. The client is served from a different machine to the turnkey.
The turnkey can be in 3 states:
turned off
booted into windows, but not running the turnkey app
running the turnkey app
My client sends a 'find the turnkey product' GET message to my nodejs/express service, which then tries to find the turnkey product via http.request. The behavior for each of the 3 use cases are;
timeout
ECONNREFUSED - because the windows embedded phase of the turnkey is
refusing connections.
normal response to request (happy day scenario)
The code below handles all 3 scenarios. The trick to catching the ECONNREFUSED event was learning that its handler binds to the socket event.
var http = require('http');
var express = require('express');
var url = require('url');
function find (req, res) {
var queryObj = url.parse(req.url, true).query;
var options = {
host: queryObj.ip, // client attaches ip address of turnkey to url.
port: 1234,
path: '/some/path',
}; // http get options
var badNews = function (e) {
console.log (e.name + ' error: ', e.message);
res.send({'ok': false, 'msg': e.message});
}; // sends failure messages to log and client
// instantiate http request object and fire it
var msg = http.request(options, function (response) {
var body = '';
response.on ('data', function(d) {
body += d;
}); // accumulate response chunks
response.on ('end', function () {
res.send({'ok': true, 'msg': body});
console.log('sent ok');
}); // done receiving, send reply to client
response.on('error', function (e) {
badNews(e);
}); // uh oh, send bad news to client
});
msg.on('socket', function(socket) {
socket.setTimeout(2000, function () { // set short timeout so discovery fails fast
var e = new Error ('Timeout connecting to ' + queryObj.ip));
e.name = 'Timeout';
badNews(e);
msg.abort(); // kill socket
});
socket.on('error', function (err) { // this catches ECONNREFUSED events
badNews(err);
msg.abort(); // kill socket
});
}); // handle connection events and errors
msg.on('error', function (e) { // happens when we abort
console.log(e);
});
msg.end();
}
For those not using DNS (you can also use request instead of get by simply replacing get with request like so: http.request({ ... })):
http.get({
host: '127.0.0.1',
port: 443,
path: '/books?author=spongebob',
auth: 'user:p#ssword#'
}, resp => {
let data;
resp.on('data', chunk => {
data += chunk;
});
resp.on('end', () => console.log(data));
}).on('error', err => console.log(err));
app.get('/', function(req, res){
var options = {
host: 'www.google.com'
};
http.get(options, function(http_res) {
http_res.on('data', function (chunk) {
res.send('BODY: ' + chunk);
});
res.end("");
});
});
I am trying to download google.com homepage, and reprint it, but I get an "Can't use mutable header APIs after sent." error
Anyone know why? or how to make http call?
Check out the example here on the node.js doc.
The method http.get is a convenience method, it handles a lot of basic stuff for a GET request, which usually has no body to it. Below is a sample of how to make a simple HTTP GET request.
var http = require("http");
var options = {
host: 'www.google.com'
};
http.get(options, function (http_res) {
// initialize the container for our data
var data = "";
// this event fires many times, each time collecting another piece of the response
http_res.on("data", function (chunk) {
// append this chunk to our growing `data` var
data += chunk;
});
// this event fires *one* time, after all the `data` events/chunks have been gathered
http_res.on("end", function () {
// you can use res.send instead of console.log to output via express
console.log(data);
});
});