I have this basic express app:
var express = require('express');
var app = express();
var PORT = 3000;
var through = require('through');
function write(buf) {
console.log('writing...');
this.queue('okkkk');
}
function end() {
this.queue(null);
}
var str = through(write, end);
/* routes */
app.get('/', function(req, res){
res.send("Hello!");
})
app.post('/stream', function(req, res){
var s = req.pipe(str).pipe(res);
s.on('finish', function() {
console.log('all writes are now complete.'); // printed the first time
});
});
/* listen */
app.listen(PORT, function () {
console.log('listening on port ' + PORT + '...');
});
When I post some data to /stream endpoint for the first time after starting the server I get okkk as the response which is what I expect. However, after that, any requests to /stream endpoint just timeout and not return any response.
Why is it so? What's exactly happening here?
I had this same problem and looks like res was not being finished properly. So I added a callback to my stream and ended que res myself. That fixed my problem:
stream.on('end', () => res.end());
stream.pipe(res);
It worked when I replaced req.pipe(str).pipe(res) with req.pipe(through(write, end)).pipe(res) which essentially makes sure that a new instance of through stream is created for every request.
Related
I'm trying to build an endpoint that will receive a request, emit the request data to a WebSocket client, wait for an event, then send back the response using express + socketio. This question is similar to it: Wait for socketio event inside express route
1) Receive request at http://localhost:3000/endpoint
2) Emit the event to web sockets as 'req'
3) Wait for 'res' event from ws
4) Send the received events details as the response of express.
Here is how I'm implemented:
server.js
var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);
app.get('/', (req, res) => {
res.sendFile(__dirname + '/index.html');
});
var socket;
io.on('connection', function (s) {
socket = s;
});
http.listen(3000);
app.get('/endpoint', function (req, res) {
console.log('new request')
io.emit('req', { data: 'hello' });
socket.on('res', function (data) {
res.status(200).json(data);
});
});
index.html
<script src="/socket.io/socket.io.js"></script>
<script>
var socket = io();
socket.on('req', (data) => {
console.log(data)
socket.emit('res', data);
});
</script>
The script works fine for the first request on /endpoint. But if i hit the url again, it says
Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent
to the client
Please note that:
socket.on('res', function (data) {
res.status(200).json(data);
});
Is being called each time a socket is sending a response, thus showing the above error. You should unbind the listener inside the callback function.
Keep an array of express responses and set an id to each request. So it can be used later and delete if needed.
var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);
var timeout = require('connect-timeout');
var uuid = require('uuidv4');
var _ = require('lodash');
app.use(timeout('10s'));
app.get('/', (req, res) => {
res.sendFile(__dirname + '/index.html');
});
let responses = []
io.on('connection', (socket) => {
socket.on('res', (e) => {
var obj = _.find(responses, r => r.id === e.id);
obj.res.send(e)
_.remove(responses, r => r.id === e.id);
})
})
app.get('/endpoint', (req, res) => {
const id = uuid()
io.emit('req', { id, ip: req.ip, header: req.headers, method: req.method });
responses.push({ id, res })
});
http.listen(3000);
You're trying to do two different async tasks for the same data.
First, take your socket.on('res'...) out of the app.get().
Send back res.status(200) immediately with express to say you received the request and it is processing. Then send the socket message to the client using socket.io when it's complete. You'll want to save the connected users socket client ID and use io.to(socketId).emit(...data...) to do this
the other option is what I always do (assuming it's not a crazy large payload of data you're sending) Just use socket.io for the whole process.
client
function makeRequest () {
socket.on('data-complete--error', function ( error ) {
// ... message to user :(
// also remove these handlers when finished
socket.off('data-complete--error');
socket.off('data-complete--success');
});
socket.on('data-complete--success', function ( data ) {
// ... message to user :)
// ... handle data
// also remove these handlers when finished
socket.off('data-complete--error');
socket.off('data-complete--success');
});
socket.emit('request-data');
}
makeRequest();
server
move your stuff out and handle without using express at all
I have a REST api hosted on Amazon EC2, which is written with Nodejs (Express).
In a particular REST call, a reply of about 5MB is sent to the client. Before the client completely receives the reply, client prints following error message.
Premature end of Content-Length delimited message body
I added a connection listener in nodejs server like below to check what is going on the server.
var app = express();
var server = http.createServer(app);
var port = app.get('port');
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
server.on('connection', function (socket) {
log.debug('SOCKET OPENED' + JSON.stringify(socket.address()));
socket.setTimeout(300000); //5 minute timeout
socket.on('end', function () {
log.debug('SOCKET END: other end of the socket sends a FIN packet');
});
socket.on('timeout', function () {
log.warn('SOCKET TIMEOUT');
});
socket.on('error', function (error) {
log.warn('SOCKET ERROR: ' + JSON.stringify(error));
});
socket.on('close', function (had_error) {
log.debug('SOCKET CLOSED. IT WAS ERROR: ' + had_error);
});
});
I observed that SOCKET TIMEOUT gets logged in backend. In above code, I have increased the socket timeout to 5 minutes, but it doesn't seem to have any effect.
Earlier I had the REST API hosted in Google compute engine, and I didn't have this problem back then.
What could be the problem here?
Edit: Here is the code of REST API call.
I have following code in my app.js
require('./routes/index')(app);
Following is the index.js of routes directory.
var changeCase = require('change-case');
var express = require('express');
var routes = require('require-dir')();
module.exports = function (app) {
Object.keys(routes).forEach(function (routeName) {
var router = express.Router();
require('./' + routeName)(router);
app.use('/api/' + changeCase.paramCase(routeName), router);
});
};
As it can be seen, it loops through all the js files in the routes directory and registers the file name as the URL path in app.
Here is the code of this particular route for which I face this problem.
module.exports = function (router) {
router.get("/fetch", function (req, res, next) {
itemModel.fetch(req.user.clientId, function (error, items) {
if (error) {
res.status(500).json({error: error});
} else {
res.json(items); //items is a JSON array
}
});
});
}
Setting timeout for the HTTP server resolved the issue.
var server = http.createServer(app);
var port = app.get('port');
server.listen(port);
server.setTimeout(300000, function (socket) {
});
I'm new to node and got stuck with handling multiple async tasks.
Except from node, I've got another server (S1) which doesn't return data immediately to requests, it can returns multiple types of data and also can send notifications without requesting them specifically, so node have to listen to data from it , parse it and act accordingly.
The connection to this server (S1) is done by using:
S1 = net.createConnection({'host':S1Host, 'port': S1Port});
And node listens to data with:
S1.on('data', function(data){
S1DataParse(data);
});
I have to route the correct data (after parsing it) to a specific POST request.
app.post('/GetFooFromS1', function(req, res){
// Send request to S1
S1.write({'type':'foo'});
// If got the correct data sometime in the future, send response to the browser
res.setHeader('Content-Type', 'application/json');
res.json({'status':'success', 'value':S1FooData});
});
I tried to use the async module for that, but with no success.
What I was trying to do:
var asyncTasks = [];
app.post('/GetFooFromS1', function(req, res){
asyncTasks.push(function(callback){
// Send request to S1
S1.write({'type':'foo'});
});
async.parallel(asyncTasks, function(response){
res.setHeader('Content-Type', 'application/json');
res.json({'status':'success', 'value':response});
});
});
and another task in S1DataParse:
function S1DataParse(){
if(data.type='foo'){
asyncTasks.push(function(callback){
callback(data);
});
}
}
But, of course, the second task never added to the asyncTasks array. I really got stuck with that.
Can you please help me with that?
Thanks
-=-=-=- Edit -=-=-=-
Eventually, I came accross with events and EventEmitter().
From the POST request I call the function that sends requests to the data server (DataServerClientGet).
In this function I register a listener which will get the future data.
eventEmitter.on('getData', returnDataServerData);
It all works great except for one thing. Whenever I refresh the page or add other POST requests, I get an error:
Error: Can't set headers after they are sent.
It would be great if I solve this problem. Help me, please.
Thanks ;)
The whole code looks like this:
var express = require('express');
var app = express();
var http = require('http').Server(app);
var bodyParser = require('body-parser')
var net = require('net');
var events = require('events');
var dataServerHost = '127.0.0.1';
var dataServerPort = 12345;
var dataServerClient;
var logMsg;
var eventEmitter = new events.EventEmitter();
/*******************************************/
// Init
/*******************************************/
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));
app.use(express.static(__dirname + '/public'));
/*******************************************/
// Connect to the data server
/*******************************************/
DataServerConnect();
/*******************************************/
// Open listener on port 3000 (to browser)
/*******************************************/
http.listen(3000, function(){
logMsg = 'listening on *:3000';
console.log(logMsg);
});
/*******************************************/
// Routing
/*******************************************/
app.get('/', function(req, res){
res.sendFile(__dirname + '/index.html');
});
app.post('/GetDataFoo', function(req, res){
var msg;
var size;
msg ='\n{"Type":"Query", "SubType":"GetDataFoo","SearchFilter":""}';
size = msg.length;
logMsg = 'Client to DataServer: GetDataFoo';
console.log(logMsg);
DataServerClientGet('GetDataFoo', size, msg, res);
});
/*******************************************/
// Functions
/*******************************************/
function DataServerConnect(){
dataServerClient = net.createConnection({'host':dataServerHost, 'port': dataServerPort}, function(){
logMsg = 'Connected to DataServer ['+dataServerHost+':'+dataServerPort+']';
console.log(logMsg);
});
dataServerClient.on('data', function(data){
logMsg = 'DataServerData>>>\n'+data.toString()+'DataServerData<<<';
console.log(logMsg);
DataServerDataParse(data.toString());
});
dataServerClient.on('end', function(){
logMsg = 'Disconnected from DataServer';
console.log(logMsg);
});
}
function DataServerClientGet(type, size, msg, res){
dataServerClient.write('Type: Json\nSize: '+size+'\n\n'+msg, function(err){
var returnDataServerData = function returnDataServerData(results){
res.setHeader('Content-Type', 'application/json');
res.json({'status':'success', 'value':results});
}
eventEmitter.on('getData', returnDataServerData);
}
function DataServerDataParse(json){
if(json.Type=='GetDataFoo')
{
var MessageList = json.MessageList;
eventEmitter.emit('getData', MessageList);
}
}
-=-=-=- Edit -=-=-=-
The Error: Can't set headers after they are sent. caused by adding the same listener of the same type each time the DataServerClientGet was called and the res was sending multiple times.
I solved this one by adding: removeListener(event, listener)
right after the res, inside the function. Anyway, I think it's wrong and can cause problems if there will be multiple calling to DataServerClientGet with the same type etc.
There is a optional callback parameter that you can pass to write function(docs), something like :
S1.write({'type':'foo'},function(err){
if(err){
//Handle error
}else{
res.setHeader('Content-Type', 'application/json');
res.json({'status':'success', 'value':response});
}
})
This can work with post route , but in your 'data' listener ,you cant send data from server to client when there is not connection initialized by client (it is not bidireccional ) if you want bidireccional behavior you can check socket.io
Tried different methods, but the data is sent to a maximum of one or two clients. How to send data to all the clients connected to the server ? What am I doing wrong?
Server.js:
var PORT = 3000;
var options = {
// 'log level': 0
};
var express = require('express');
var app = express();
var http = require('http');
var server = http.createServer(app);
var io = require('socket.io').listen(server, options);
server.listen(PORT);
app.get('/', function (req, res) {
res.sendfile(__dirname + '/attantions/templates/.default/template.php');
});
io.sockets.on('connection', function (client) {
client.on('attantion', function (data) {
try {
// Tried so
io.sockets.volatile.emit('attantion', data);
// And tried so
io.sockets.emit('attantion', data);
client.emit('attantion', data);
client.broadcast.emit('attantion', data );
} catch (e) {
console.log(e);
client.disconnect();
}
});
});
Client.js:
socket.emit("attantion", data);
socket.on('attantion', function (data) {
pushData(data);
});
See this post for different options for socket.io messages
Send response to all clients except sender (Socket.io)
io.sockets.on('connection', function (client) {
client.on('attantion', function (data) {
//client.emit('attantion', data ); // This will send it to only the client
//client.broadcast.emit('attantion', data); // This will send it to everyone but this client
io.emit('attantion', data); // This will send it to all attached sockets.
});
});
Edit
I wonder if this post can help you?
Socket.io - Cannot load file
I was curious how sending the php file to the client through node.js works? are you using another framework?
Could you show more of what your client code looks like? loading the lib and the instantiation of the socket.
The thing I want to do is modify the response body.
For this I am using a middleware that gets called at every request.
To achieve it, I took a demo application from github https://github.com/ccoenraets/nodecellar . I added a middleware in the server.js similar to the example given on express logging response body.
Still I am unable to modify the response body, as res.send = function (string) does not get called.
Below mentioned is the code. Please let me know what wrong am I doing here.
var express = require('express'),
path = require('path'),
http = require('http'),
wine = require('./routes/wines');
var app = express();
app.configure(function () {
app.set('port', process.env.PORT || 4000);
app.use(express.logger('dev')); /* 'default', 'short', 'tiny', 'dev' */
app.use(express.bodyParser()),
app.use(express.static(path.join(__dirname, 'public')));
app.use(logResponseBody);
});
app.get('/wines', wine.findAll);
app.get('/wines/:id', wine.findById);
app.post('/wines', wine.addWine);
app.put('/wines/:id', wine.updateWine);
app.delete('/wines/:id', wine.deleteWine);
http.createServer(app).listen(app.get('port'), function () {
console.log("Express server listening on port " + app.get('port'));
});
function logResponseBody(req,res,next){
var send = res.send;
console.log("send resp is: "+send);
res.send = function (string) {
var body = string instanceof Buffer ? string.toString() : string;
console.log("Body found is: "+body);
body = body.replace(/<\/head>/, function (w) {
return 'Modified head' + w;
});
send.call(this, body);
};
res.on('finish', function(){
console.log("Finished " + res.headersSent); // for example
console.log("Finished " + res.statusCode); // for example
})
next();
}
PS: I am starting a new thread for a similar question as I have less than 50 reputation.Therefore cant add comments there.
The best way to add middleware to an express app is to use the app.use method, so you can remove the whole http.createServer block and replace it for something like
app.use(function(req, res, next) {
//do everything you want to happen on every request
next();
});
app.listen(app.get('port'), function() {
console.log('listening on port ' + app.get('port'));
});
For more info on app.use, check the express app use documentation.