I spawned the following child: var spw = spawn('ping', ['-n','10', '127.0.0.1']) and I would like to receive the ping results on the client side (browser) one by one, not as a whole.
So far I tried this:
app.get('/path', function(req, res) {
...
spw.stdout.on('data', function (data) {
var str = data.toString();
res.write(str + "\n");
});
...
}
and that:
...
spw.stdout.pipe(res);
...
In both cases browser waits 10 of the pings to complete, and then prints the result as a whole. I would like to have them one by one, how to accomplish that?
(Client is just making a call to .../path and console.logs the result)
EDIT: Although I do believe that websockets are necessary to implement this, I just want to know whether there are any other ways. I saw several confusing SO answers, and blog posts (in this post, at step one OP streams the logs to the browser) which didn't help, therefore I decided to go for a bounty for some attention.
Here's a complete example using SSE (Server sent events). This works in Firefox and probably Chrome too:
var cp = require("child_process"),
express = require("express"),
app = express();
app.configure(function(){
app.use(express.static(__dirname));
});
app.get('/msg', function(req, res){
res.writeHead(200, { "Content-Type": "text/event-stream",
"Cache-control": "no-cache" });
var spw = cp.spawn('ping', ['-c', '100', '127.0.0.1']),
str = "";
spw.stdout.on('data', function (data) {
str += data.toString();
// just so we can see the server is doing something
console.log("data");
// Flush out line by line.
var lines = str.split("\n");
for(var i in lines) {
if(i == lines.length - 1) {
str = lines[i];
} else{
// Note: The double-newline is *required*
res.write('data: ' + lines[i] + "\n\n");
}
}
});
spw.on('close', function (code) {
res.end(str);
});
spw.stderr.on('data', function (data) {
res.end('stderr: ' + data);
});
});
app.listen(4000);
And the client HTML:
<!DOCTYPE Html>
<html>
<body>
<ul id="eventlist"> </ul>
<script>
var eventList = document.getElementById("eventlist");
var evtSource = new EventSource("http://localhost:4000/msg");
var newElement = document.createElement("li");
newElement.innerHTML = "Messages:";
eventList.appendChild(newElement);
evtSource.onmessage = function(e) {
console.log("received event");
console.log(e);
var newElement = document.createElement("li");
newElement.innerHTML = "message: " + e.data;
eventList.appendChild(newElement);
};
evtSource.onerror = function(e) {
console.log("EventSource failed.");
};
console.log(evtSource);
</script>
</body>
</html>
Run node index.js and point your browser at http://localhost:4000/client.html.
Note that I had to use the "-c" option rather than "-n" since I'm running OS X.
If you are using Google Chrome, changing the content-type to "text/event-stream" does what your looking for.
res.writeHead(200, { "Content-Type": "text/event-stream" });
See my gist for complete example: https://gist.github.com/sfarthin/9139500
This cannot be achieved with the standard HTTP request/response cycle. Basically what you are trying to do is make a "push" or "realtime" server. This can only be achieved with xhr-polling or websockets.
Code Example 1:
app.get('/path', function(req, res) {
...
spw.stdout.on('data', function (data) {
var str = data.toString();
res.write(str + "\n");
});
...
}
This code never sends an end signal and therefore will never respond. If you were to add a call to res.end() within that event handler, you will only get the first ping – which is the expected behavior because you are ending the response stream after the first chunk of data from stdout.
Code Sample 2:
spw.stdout.pipe(res);
Here stdout is flushing the packets to the browser, but the browser will not render the data chunks until all packets are received. Thus the reason why it waits 10 seconds and then renders the entirety of stdout. The major benefit to this method is not buffering the response in memory before sending — keeping your memory footprint lightweight.
Related
I am trying to test some some examples from the book I'm reading, "Learning Node 2012". And my application for testing the server by doing 2000 requests is pausing. The tester pauses after 5 requests and sends another 5 after a certain interval. Why is it pausing? How can I fix this?
The server code:
var http = require('http');
var fs = require('fs');
// write out numbers
var counter = 0;
function writeNumbers(res)
{
for (var i = 0; i < 100; i++)
{
counter++;
res.write(counter.toString() + '\n');
}
}
// create the http server
http.createServer( function(req, res) {
var query = require('url').parse(req.url).query;
var app = require('querystring').parse(query).file + ".txt";
// content header
res.writeHead(200, { 'Content-Type': 'text/plain' } );
// write out numbers
writeNumbers(res);
// timer to open file and read contents
setTimeout(function() {
console.log('opening ' + app);
// open and read in file contents
fs.readFile(app, 'utf8', function(err, data) {
if (err)
res.write('Could not find or open file for reading\n');
else
res.write(data);
res.end();
});
}, 2000);
}).listen(3000);
console.log('Server is running on port 3000');
The spam test code:
var http = require('http');
// the url we want, plus the path and options we need
var options = {
host: 'localhost',
port: 3000,
path: '/?file=secondary',
method: 'GET'
};
var processPublicTimeline = function(response) {
// finished? ok, write the data to a file
console.log('finished request');
};
for (var i = 0; i < 2000; i++)
{
// make the request, and then end it, to close the connection
http.request(options, processPublicTimeline).end();
}
While this definitely does have some relation to Why is node.js only processing six requests at a time?
It is also because you are using a timeOut to call res.end() to close the connection/respond, and thus move onto the next connection in queue.
You should instead think about these types of things aynchronously, without the use of timeOuts, but instead with callBacks.
So your code for your two main blocks could be more like:
var counter = 0;
function writeNumbers(res, callBack){
// notice callBack argument
for (var i = 0; i < 100; i++){
counter++;
res.write(counter.toString() + '\n');
}
// execute callBack (if it exists)
if(callBack && typeof callBack === "function") callBack();
}
http.createServer( function (req, res){
var query = require('url').parse(req.url).query;
var app = require('querystring').parse(query).file + ".txt";
res.writeHead(200, { 'Content-Type': 'text/plain' } );
writeNumbers(res, function(){
// Notice this function which is passed as a callBack argument for writeNumbers to evaluate.
// This executes when the main writeNumbers portion finishes.
console.log('opening ' + app);
fs.readFile(app, 'utf8', function(err, data) {
if (err)
res.write('Could not find or open file for reading\n');
else
res.write(data);
res.end();
});
});
}).listen(3000);
Notice that your writeNumbers function now takes a callBack argument to execute when it is done, and that when you call it in your server's object, you pass a function as that callBack argument. This is one of the core patterns used very frequently in node.js/javascript applications.
This means that you aren't waiting for a timeOut to execute to end your request response, but rather it is ended as it processes your response, and moves onto the next connection immediately. This is likely to happen wayyyy quicker than 2 seconds (the amount of your timeOut). So you should see your connections being processed much quicker.
Because (as someone pointed out in your comments) your system is only able to handle a few open TCP connections at a time, you want to move through your connections as quickly as possible. Leveraging a callBack chain can help you do that when you want to do things in a certain order, or if you need to wait for certain processes to finish before executing others, without guessing with a timeOut.
Hope this helps!
I'm working on an RSS aggregator via node-webkit, and I've stumbled upon an issue where a seemingly random volume of my requests (code below) will fail with any of the following errors: HPE_INVALID_CONSTANT, ETIMEDOUT, EAGAIN, and HPE_INVALID_HEADER_TOKEN. I've sorted those in regularity of occurrence.
I made a test.html file with as much relevance as possible. Just plug in a large volume of web page urls into httpList and it will work. though, be careful what URLs you use, you wouldn't want to crash someones server! A list of around 56 seems to work for causing the errors.
As an aside, this is really hard to reproduce, meaning its gotta be something I can change on my end. Sometimes the same URL will fail, while on the next pass it will work. Also I have had URLs that have failed, that were indeed accessible (I opened them up in chrome no problem).
<!DOCTYPE html>
<html>
<head>
<title>HTTP Get test</title>
</head>
<body>
<script>
var httpList = [];
var fromUrl = function(stringUrl) {
var http = require('http');
var url = require("url");
var urlData = url.parse(stringUrl);
var parser = window.document.createElement('a');
parser.href = stringUrl;
var options = {
host: urlData.host,
path: urlData.path + urlData.search + urlData.hash
};
var httpResponse = function(response) {
response.setEncoding('utf8');
request.setSocketKeepAlive(true);
var str = '';
response.on('data', function(chunk) {
str += chunk;
});
response.on('end', function() {
request.setSocketKeepAlive(false);
console.log("url",stringUrl,"was successfully parsed");
});
}
var request = http.request(options, httpResponse)
request.on('error', function(e) {
if (e.code === "ENOTFOUND") {
console.error("The feed", stringUrl, "could not be fetched.")
} else {
console.log(e, e.message, stringUrl);
}
});
request.end();
};
for(var i=0;i<httpList.length;i++) {
fromUrl(httpList[i]);
}
</script>
</body>
</html>
See this node issue. BTW is not "solved".
I have two node servers, one on port 5000 (call it "Face") and another on port 5001 (call it "Hands")
Both are started via a foreman procfile at the same time. Ports are fixed and the url I'm targeting works in the browser.
When the Hands starts up, it needs to talk to the Face (Facepalm?) and register itself. However, the below code doesn't seem to be working. (This is coffeescript generated JS)
Register gets called during server initialization, after the http server has been started. In case it was a timing issue, I kick off the register function with a setTimeout() of 2 seconds. I know the page that its hitting (/home/register) is available and working.
Right now I can see it get to the "Posting to" console log line. On the Face I have put a console.log in the register code and its never logging anything - meaning I don't think its actually getting hit. (It DOES log if hit from browser) And nothing errors out - it just calls the request and then wanders off to get a sandwich.
Both servers are "roll your own" - not using any frameworks. Let me know if you see a weird typo or need more info. Thanks!
register = function() {
var _this = this;
console.log('Registering with Face Server');
return post_face('/home/register', GLOBAL.data, function(rs) {
console.log(rs);
if (rs.registered) {
GLOBAL.data.registered = true;
return console.log("Registered with face at " + GLOBAL.config.face_host + ":" + GLOBAL.config.face_port);
} else {
throw "ERROR: Could not register with face server! " + GLOBAL.config.face_host + ":" + GLOBAL.config.face_port;
return false;
}
});
};
post_face = function(path, data, cb) {
var post_data, post_options, post_req;
post_data = querystring.stringify({
'registration': data
});
post_options = {
host: GLOBAL.config.face_host,
port: GLOBAL.config.face_port,
path: path,
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': post_data.length
}
};
console.log("Posting to " + post_options.host + ":" + post_options.port);
post_req = http.request(post_options, function(res) {
var response,
_this = this;
console.log(res);
response = "";
res.setEncoding('utf8');
res.on('data', function(chunk) {
return response += chunk;
});
return res.on('end', function() {
return cb.call(_this, response);
});
});
return true;
};
Thanks to Bill above, the answer was that I wasn't actually posting the data and ending the request! Bad copy / paste / edit from some samples I was referring to. Here's the last two lines of code I should have had:
post_req.write(post_data);
post_req.end();
I have a program written with node that connects to Flash. In Flash when a key is pressed the keypress is sent through a socket to node, which then echoes the keypress back to Flash with some extra text. The Flash part works fine. The node portion basically works, except for one part where I connect to a web page. What I want to do is get the contents of a web page and echo that back to Flash though a socket. Here's my code:
var net = require('net');
var http = require('http'),
sys = require('sys'),
fs = require('fs');
var options = {
host: 'theurl.com',
port: 80,
path: '/',
method: 'POST'
};
var msg;
var mySocket;
var server = net.createServer(function(socket) {
mySocket = socket;
mySocket.on("connect", onConnect);
mySocket.on("data", onData);
});
function onConnect() {
console.log("Connected to Flash");
}
function onData(d) {
if(d == "exit\0") {
console.log("exit");
mySocket.end();
server.close();
} else if(d == "32\0") {
var req = http.request(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
msg = chunk;
console.log("in function: msg = " + msg);
mySocket.write(msg, 'utf8');
});
});
console.log("right after res: msg = " + msg);
mySocket.write("right after res: "+msg, 'utf8');
// write data to request body
req.write('data\n');
req.write('data\n');
req.end();
mySocket.write("32 part: " + d, 'utf8');
} else {
console.log("From Flash = " + d );
mySocket.write(d, 'utf8');
}
}
server.listen(8080, "192.168.0.1");
When I press spacebar (no 32) the data seems to be written but doesn't show up on the Flash side until I press 2 keys. Note that the part outside of the "http.request" area DOES show up on the Flash side correctly. So the mysocket.write within the res.on area doesn't show up until 2 keypresses, but the mysocket.write that says "right after res" shows up just like I want it to. SO I figure that I need one of these things to happen:
1) The chunk returned from the http.request area be visible outside of that area
2) the mysocket.write within the http.request area to work the same way as the mysocket.write outside of the http.request area
3) store the chunk to the hard drive and then reread the contents stored later and send that
Those are the options I've thought of. I think #3 makes no sense since I seem to have the var in the program, but just can't access it where I want to. Not sure about why the socket doesn't write the same within that area as it does outside of that area, but if there were a way to do #1 that seems to me to make the most sense. I am open to suggestions though. Thanks for your time.
Darryl
app.get('/', function(req, res){
var options = {
host: 'www.google.com'
};
http.get(options, function(http_res) {
http_res.on('data', function (chunk) {
res.send('BODY: ' + chunk);
});
res.end("");
});
});
I am trying to download google.com homepage, and reprint it, but I get an "Can't use mutable header APIs after sent." error
Anyone know why? or how to make http call?
Check out the example here on the node.js doc.
The method http.get is a convenience method, it handles a lot of basic stuff for a GET request, which usually has no body to it. Below is a sample of how to make a simple HTTP GET request.
var http = require("http");
var options = {
host: 'www.google.com'
};
http.get(options, function (http_res) {
// initialize the container for our data
var data = "";
// this event fires many times, each time collecting another piece of the response
http_res.on("data", function (chunk) {
// append this chunk to our growing `data` var
data += chunk;
});
// this event fires *one* time, after all the `data` events/chunks have been gathered
http_res.on("end", function () {
// you can use res.send instead of console.log to output via express
console.log(data);
});
});