I'm piping to a file an HTTPS request, it works ok 99.9% of calls, but occasionally (maybe when server or network are not available) hangs indefinitely...
This obviously cause my application to stop working and requiring a manual restart...
I have other https connections that used to occasionally hang that always complete now using the following error code on the request object, as suggested on node documentation:
request.on('socket', function(socket) {
socket.setTimeout(10000);
socket.on('timeout', function() { request.abort(); });
});
request.on('error', function(e) {
// Handle the error...
console.error("FAILED!");
});
... but it seems that timeouts on the request are ignored if the destination is piped to a file stream, maybe I should handle an error with a timeout on the filesystem object, but the documentation is not clear if there is an event I have to wait for except for 'finish'...
Here is the sample code, I hope someone can help me:
var https = require('https'),
fs = require('fs');
var opts = {
host: 'www.google.com',
path: '/',
method: 'GET',
port: 443
};
var file = fs.createWriteStream('test.html');
var request = https.request(opts, function(response) {
response.pipe(file);
file.on('finish', function() {
file.close(function(){
console.log("OK!");
});
});
});
request.on('socket', function(socket) {
socket.setTimeout(10000);
socket.on('timeout', function() { request.abort(); });
});
request.on('error', function(e) {
console.error("FAILED!");
});
request.end();
If you wanna try the hang, change host and path with a huge file and disconnect the network cable during the transfer, it should time out after 10 seconds, but it doesn't...
I set up a demo node.js http server that sends a very slow answer and a client similar to your sample code.
When I start the client and then stop the server while sending the response then I also don't get a timeout event on the socket but I get a end event on the response within the client:
var request = https.request(opts, function(response) {
response.pipe(file);
file.on('finish', function() {
file.close(function(){
console.log("OK!");
});
});
response.on('end', function() {
// this is printed when I stop the server
console.log("response ended");
});
});
```
Maybe you could listen to that event?
Related
I have used dicer for parsing pipe by request but I don't know which command for stop pipe.
dicer.on('part', function(part) {
var frameEncoded = '';
part.setEncoding('base64');
part.on('header', function(header) { });
part.on('data', function(data) { frameEncoded += data; });
part.on('end', function() { console.log(frameEncoded); });
});
var options = {
method: 'GET',
uri: 'http://192.168.1.2/video.mjpeg/'
}
request(options).pipe(dicer)
//only for test
setTimeout(function() {
console.log('stop request.pipe');
var options = {
method: 'GET',
uri: 'http://192.168.1.2/video.mjpeg/'
}
request(options).end() //<-- ?? which command?? This not work...
}, 5000);
The uri pointing to a stream of ipcamera and impossible grab event connection close of request because stream it's infinite.
One way to get access to the underlying socket for the request is to listen for the socket event on the request object. Then it's just a matter of closing that socket. For example:
var req = request(options);
req.on('socket', function(sock) {
setTimeout(function() {
console.log('stop request.pipe');
sock.end(); // or sock.destroy();
}, 5000);
});
req.pipe(dicer);
Note though, the request may not have started yet when the socket event is emitted. The socket event basically lets you know that a socket has been assigned to be used for the request (this is especially useful when you are using an http Agent that has maxSockets set to some finite value so you may not necessarily get assigned a socket right away).
You could also try:
var req = request(options);
req.on('response', function(res) {
setTimeout(function() {
console.log('stop request.pipe');
res.socket.end(); // or res.socket.destroy();
}, 5000);
});
req.pipe(dicer);
#mscdex's method works for me. Here is the code :-
request("https://www.linkedin.com/afhdjalsdjhdks")
.on("error", function (error) { process.stderr.write(error.code) })
.on("response", function (resp) {
if ((resp.statusCode !== 200) && (resp.statusCode !== 201)) {
process.stderr.write(format("%s - %s", resp.statusCode, resp.statusMessage))
resp.socket.destroy()
}
})
// .pipe(StreamAll())
.pipe(process.stdout)
I have a very simple configuration in a node server with socket.io installed (a little bit more complex but essentially like this one):
var main = require('express')();
server = require('http').createServer(main);
io = require('socket.io')(server);
io.use(function(socket, next) {
console.log("middleware!");
next();
});
io.on('connection', function (socket) {
console.log('connected...');
socket.on('pong', function (data) {
console.log(data.message);
});
setTimeout(function() {
console.log("Saying hello");
socket.emit('ping', { message: 'Hello from server ' + Date.now() });
}, 1000);
});
server.listen(2080, function onCreateServerMain() {
console.log('Server main is listening on port 2080';
console.log('************************************************************');
});
In the client:
var socketIoScript,
loadSocketTimeout,
trialsToLoadSocketIo = 0,
APP_CFG = {baseUrl : "http://192.168.1.13:2080"};
function loadSocketIo(socketIoIp) {
socketIoScript = document.createElement('script');
socketIoScript.setAttribute('src', socketIoIp);
socketIoScript.setAttribute('onload', 'onSocketLoaded();');
document.head.appendChild(socketIoScript);
}
window.onSocketLoaded = function onSocketLoaded() {
if (typeof(io.connect) === 'function') {
var mSocket,
mIoSocket;
$timeout.cancel(loadSocketTimeout);
mIoSocket = new io.Manager(APP_CFG.baseUrl);
mIoSocket.connect(function(socket) {
console.log('Connected!!');
});
mIoSocket.on('error', function onSocketError(e) {
console.log('WebSocket Error ' + error);
});
mIoSocket.on('ping', function onPingReceived(e) {
console.log('Server emitted ping: ' + e.data);
mSocket.emit('pong', 'hi server!');
});
}
}
~(function onLoadSocketTimeout() {
var nextTimeout;
if (trialsToLoadSocketIo < 10) {
nextTimeout = 5000;
} else if (trialsToLoadSocketIo > 60) {
nextTimeout = 60000;
} else {
nextTimeout = 1000 * trialsToLoadSocketIo;
}
if (socketIoScript) {
document.head.removeChild(socketIoScript);
}
loadSocketIo(APP_CFG.baseUrl + '/socket.io/socket.io.js#' + trialsToLoadSocketIo);
loadSocketTimeout = $timeout(onLoadSocketTimeout, nextTimeout);
trialsToLoadSocketIo += 1;
})();
(I'm doing like this because it's mobile app so it may have not connection). I'm testing it with Brackets and Chrome. Server and client are in the same machine. In the app the script is loaded fine and it connects to the server as I can it see in node log (edit: and this is all what I get in the node console):
Server main is listening on port 2080
************************************************************
middleware!
connected...
Saying hello
Edit: in Chrome console I don't get any message, and any breakpoint stops at on listeners. If I stop node, the console for the Chrome immediately starts logging that it has been disconnected:
GET http://192.168.1.13:2080/socket.io/?EIO=3&transport=polling&t=1413066902601-6 net::ERR_CONNECTION_REFUSED
GET http://192.168.1.13:2080/socket.io/?EIO=3&transport=polling&t=1413066906606-7 net::ERR_CONNECTION_REFUSED
But I can't see any incoming message. In the app I don't receive any incoming message. Is there any reason why I could not communicate in this environment even if socket is successfully connected?
EDIT
No app is receiving events sent from the other side. Logs from node show this, logs from Chrome are empty.
EDIT
In Chrome app I don't receive console.log("Connected!");. But neither I receive ERR_CONNECTION_REFUSED errors: I don't receive anything.
EDIT
I managed to get console.log("Connected!"); in the app by changing Manager options:
mIoSocket = new io.Manager(APP_CFG.baseUrl, { autoConnect: false });
As it was auto connecting and the events were attached after connection was made, "Connected" was never reached. But I'm still not receiving any event in any app.
I had a similar issue were event callbacks on the server were not firing when emitting. My event names were ping and pong. As soon as I renamed these events everything worked.
I suspect the event names ping and pong are reserved by socket.io and so cannot be used.
Ok, so a few things :
First, var mSocket doesn't seem to be initialized, so it may be difficult for it to emit() anything (am I missing something?)
Second, when you do :
socket.on('pong', function (data) {
console.log(data.message);
});
the server expects to receive an object containing a message property, eg : data = {message:'hi server'} In your case, you send a string, so data is 'Hi server !' and your log will say 'undefined'. You should change this bit to :
socket.on('pong', function (data) {
console.log(data);
});
and you have a similar problem the other way around, you send an object : { message: 'Hello from server ' + Date.now() }, and are trying to log a data property which does not exist. Change this bit to :
console.log('Server emitted ping: ' + e.message);
And third , you have to listen for events on the socket, not the 'manager'
Client :
mIoSocket.connect(function(socket) {
console.log('Connected!!');
socket.emit('pong');
socket.on('error', function onSocketError(e) {
console.log('WebSocket Error ' + error);
});
socket.on('ping', function onPingReceived(e) {
console.log('Server emitted ping: ' + e.data);
socket.emit('pong', 'hi server!');
});
});
Server :
io.on('connection', function (socket) {
console.log('connected...');
socket.on('pong', function (data) {
console.log(data);
});
setTimeout(function() {
console.log("Saying hello");
socket.emit('ping', { message: 'Hello from server ' + Date.now() });
}, 1000);
});
I need to connect to a web page and return the status code of the page, which I've been able to achieve using http.request however the pages I need to request can take a long time, sometimes several minutes, so I'm always getting a socket hang up error.
I'm using the following code so far:
var reqPage = function(urlString, cb) {
// Resolve the URL
var path = url.parse(urlString);
var req = http.request({
host: path.hostname,
path: path.pathname,
port: 80,
method: 'GET'
});
req.on('end', function() {
cb.call(this, res);
});
req.on('error', function(e) {
winston.error(e.message);
});
};
What do I need to do to ensure that my application still attempts to connect to the page even if it's going to take a few minutes?
Use the request module and set the timeout option to an appropriate value (in milliseconds)
var request = require('request')
var url = 'http://www.google.com' // input your url here
// use a timeout value of 10 seconds
var timeoutInMilliseconds = 10*1000
var opts = {
url: url,
timeout: timeoutInMilliseconds
}
request(opts, function (err, res, body) {
if (err) {
console.dir(err)
return
}
var statusCode = res.statusCode
console.log('status code: ' + statusCode)
})
Add this if you don't want to use a higher level http client like request or superagent , then add this...
req.on("connection", function(socket){
socket.setTimeout((1000*60*5)); //5 mins
});
Fairly new to Node.js
Made an app that runs a server process and serve files (does not use express or any frameworks), Now I'm trying to unit test it.
I'm trying to use a mocha test for that... I intended to start my server process and then run requests against it to test the expected results (stats code, body content and the likes)
However it's not working properly, all the request fail to connect to the server... I'm pretty sure that the issue is because node is juts running one process loop, the server is not running "in the background" while the queries run or possibly the server is not running yet (started ASYNC) while the request are being made ?
Anyway I was wondering what was the proper way to test this, I assume that either I need to have the server run in the background (like a forked process) and/or maybe I need to find a way to wait for the server process to be "up" first but not sure how.
Or at least recommendations on testing such server process (with Mocha or other).
Thanks.
Here is example test code (Updated since original question)
var server = new Server302('./fixture/');
var instance;
describe('Tests', function() {
before(function(done) {
instance = http.createServer(function(request, response) {
console.log(request.url);
server.serve(request, response);
}).listen(8000);
instance.on("listening", function() {
console.log("started");
done();
});
});
after(function(done){
instance.close();
console.log("stopped");
done();
});
it("Should fetch test.html", function(done) {
console.log("test1");
http.get("http://localhost:8000/", function(res) {
res.on('data', function(body) {
console.log(body)
expect(body).toEqual("test");
done();
});
})
});
It seem to Execute in order but still fails with a connection error, whereas it works when testing manually with the browser:
started
test1
․․․stopped
✖ 1 of 1 tests failed:
1) Tests Should fetch test.html:
Error: connect ECONNREFUSED
at errnoException (net.js:670:11)
at Object.afterConnect [as oncomplete] (net.js:661:19)
In your before don't call done until you get the "listening" event fired by the server.
before(function(done) {
instance = http.createServer(function(request, response) {
console.log(request.url);
server.serve(request, response);
}).listen(8000);
instance.on("listening", function() {
console.log("started");
done();
});
});
That should ensure your test connections don't start before the server is ready.
See also the documentation for server.listen
Also had to deal with the body coming in chunks, here is the final thing that works, in case that helps somebody else:
var Server302 = require('../lib/server302.js'),
http = require('http'),
assert = require("assert");
var server = new Server302('./fixture/');
var instance;
describe('Tests', function() {
before(function(done) {
instance = http.createServer(function(request, response) {
server.serve(request, response);
}).listen(8100);
instance.on("listening", function() {
done();
});
});
after(function(done) {
instance.close();
done();
});
it("Should fetch test.html", function(done) {
console.log("test1");
var body = "";
http.get({host: "localhost", port:8100, path: "/"}, function(res) {
res.on('data', function(chunk) {
// Note: it might be chunked, so need to read the whole thing.
body += chunk;
});
res.on('end', function() {
assert.ok(body.toString().indexOf("<a href='/dummy.txt'>") !== -1);
assert.equal(res.statusCode, 200);
done();
});
})
});
it("Should fetch dummy.txt", function(done) {
http.get({host: "localhost", port:8100, path: "/dummy.txt"}, function(res) {
res.on('data', function(body) {
assert.equal(res.statusCode, 200);
assert.ok(body.toString().indexOf("test") === 0);
done();
});
});
});
it("Should get 404", function(done) {
http.get({host: "localhost", port:8100, path: "/qwerty"}, function(res) {
assert.equal(res.statusCode, 404);
done();
});
});
});
Using SuperTest
Here is a full and straightforward example using SuperTest and Mocha:
var server = new Server302('./fixture/');
var request = require('supertest');
describe('Tests', function() {
it('Should fetch test.html', function(done) {
request(server)
.get('/')
.expect('test', done);
});
});
SuperTest allows you to:
Request your server using SuperAgent (much easier to use than the low level http agent).
Bound your server to an ephemeral port so there is no need to keep track of ports (you can still do it manually if needed).
Use sugary expect methods that works with Mocha (or any other test framework).
I'm trying to catch ECONNREFUSED errors when using a HTTP client in node.js. I'm making requests like this:
var http = require('http');
var options = { host: 'localhost', port: '3301', path: '/', method: 'GET' };
http.request(options).on('response', function (res) {
// do some stuff
});
I can't figure out how to catch this error:
Error: connect ECONNREFUSED
at errnoException (net.js:614:11)
at Object.afterConnect [as oncomplete] (net.js:605:18)
If I do request.on('error', function () {});, it doesn't catch it. If I do it like this:
var req = request.on(etc)
req.on('error', function blah () {});
Then I get TypeError: Object false has no method 'on'.
Do I really have to do a top-level uncaught error thing to deal with this? At the moment whatever I do my whole process quits out.
Edit: I found some blog posts on how to do it by creating a connection object, calling request on that, and then binding to errors on the connection object, but doesn't that make the entire http.request() shortcut useless?
Any reason you're not using http://nodejs.org/docs/v0.6.5/api/http.html#http.request as your base? Try this:
var req = http.request(options, function(res) {
// Bind 'data', 'end' events here
});
req.on('error', function(error) {
// Error handling here
});
req.end();
Each call to http.request() returns its self.
So try it like this...
http.request(options.function(){}).on('error',function(){}).end();
I've got a solution for this, having tried all the suggestions on this (and many other) pages.
My client needs to detect a turnkey product that runs embedded windows. The client is served from a different machine to the turnkey.
The turnkey can be in 3 states:
turned off
booted into windows, but not running the turnkey app
running the turnkey app
My client sends a 'find the turnkey product' GET message to my nodejs/express service, which then tries to find the turnkey product via http.request. The behavior for each of the 3 use cases are;
timeout
ECONNREFUSED - because the windows embedded phase of the turnkey is
refusing connections.
normal response to request (happy day scenario)
The code below handles all 3 scenarios. The trick to catching the ECONNREFUSED event was learning that its handler binds to the socket event.
var http = require('http');
var express = require('express');
var url = require('url');
function find (req, res) {
var queryObj = url.parse(req.url, true).query;
var options = {
host: queryObj.ip, // client attaches ip address of turnkey to url.
port: 1234,
path: '/some/path',
}; // http get options
var badNews = function (e) {
console.log (e.name + ' error: ', e.message);
res.send({'ok': false, 'msg': e.message});
}; // sends failure messages to log and client
// instantiate http request object and fire it
var msg = http.request(options, function (response) {
var body = '';
response.on ('data', function(d) {
body += d;
}); // accumulate response chunks
response.on ('end', function () {
res.send({'ok': true, 'msg': body});
console.log('sent ok');
}); // done receiving, send reply to client
response.on('error', function (e) {
badNews(e);
}); // uh oh, send bad news to client
});
msg.on('socket', function(socket) {
socket.setTimeout(2000, function () { // set short timeout so discovery fails fast
var e = new Error ('Timeout connecting to ' + queryObj.ip));
e.name = 'Timeout';
badNews(e);
msg.abort(); // kill socket
});
socket.on('error', function (err) { // this catches ECONNREFUSED events
badNews(err);
msg.abort(); // kill socket
});
}); // handle connection events and errors
msg.on('error', function (e) { // happens when we abort
console.log(e);
});
msg.end();
}
For those not using DNS (you can also use request instead of get by simply replacing get with request like so: http.request({ ... })):
http.get({
host: '127.0.0.1',
port: 443,
path: '/books?author=spongebob',
auth: 'user:p#ssword#'
}, resp => {
let data;
resp.on('data', chunk => {
data += chunk;
});
resp.on('end', () => console.log(data));
}).on('error', err => console.log(err));