I had an expectation that middleware that is bound to a Domain would be handled by the error handler for that domain.
In Express this did not turn out to be true.
I created a repository to illustrate this issue
https://github.com/rook2pawn/express-domains-issue
var app = express();
app.get('/',d.bind(function(req,res,next) {
throw new Error("error")
}));
var server = http.createServer(app);
Will not route the error to the domain error handler registered at d
whereas
var app = d.bind(function(req,res,next) {
throw new Error("error")
});
var server = http.createServer(app);
Will properly route the error to the domain without express.
Requesting any comments or thoughts about this?
It's just a bad example, because express wraps middleware in try-catch. This works:
app.get('/',d.bind(function(req,res,next) {
process.nextTick(function () {
throw new Error("error")
})
}));
By document of node js:
"This method is almost identical to domain.bind(callback). However, in addition to catching thrown errors, it will also intercept Error objects sent as the first argument to the function."
and I write a demo code:
var domain = require('domain');
var fs = require('fs');
var d = domain.create();
require('http').createServer(function(req, res, next) {
d.on('error', function (err) {
console.log(err);
res.writeHead(500, "content-type: plain/text");
res.end("Something missing!");
});
// This is for async
function readFile(filename, cb) {
fs.readFile(filename, 'utf8', d.bind(function (er, data) {
return cb(er, data ? JSON.parse(data) : null);
}));
}
readFile("unknow file");
// This is for sync
(d.bind(function() {
throw new Error();
}))();
}).listen(1337);
=> d.bind total can resolve all error sync and async to domain. That is the definition
Related
When I generate a webserver with express-generator, I get this folder structure :
bin/www
views/...
app.js
package.json
...
bin/www call app.js like that :
var app = require('../app');
// ...
var server = http.createServer(app);
server.listen(port);
app.js create the app like that :
var express = require('express')
var mongoose = require('mongoose')
mongoose.connect(process.env.DATABASE_URL).then(
() => {
debug('Database is connected')
},
err => {
debug('An error has occured with the database connection')
process.exit(1)
}
)
var app = express()
// Midllewares
app.use(/* some middleware 1 */)
app.use(/* some middleware 2 */)
app.use(/* some middleware 3 */)
app.use(/* some middleware ... */)
// Routes
app.get('/', function(req, res, next) {
res.json({'message': 'Welcome to my website'})
})
app.get('/users', function(req, res, next) {
Users.find({}).exec(function(err, users) {
if (err) {
res.json({'message': 'An error occured'})
return
}
res.json('users': users)
})
})
// ... others routes ...
module.exports = app
ok, this is the webserver boilerplate from express-generator. But if I want to start my app by the good way, I must call process.send('ready') when my app is ready. ("ready" mean that all services are ready to use: database, redis, scheduler...) (call process.send('ready') when your app is ready is a best practice to know that your webserver app si ready. This signal can be used by process management or other system)
The probleme is that in bin/www, the app is started (server.listen() is called) without insurance that the database connection is established. In other word, without the insurance that the webserver app is ready to listen to the traffic.
I read that start the server in bin/www is a best practice
The above example is not complete, we can considere that we have an app with multiple services that we must start before accept requests (services examples: redis, job scheduler, database connection, ftp connection to another server...)
I already check some popular and advanced boilerplate of Node.js app :
https://github.com/sahat/hackathon-starter
https://github.com/kriasoft/nodejs-api-starter
https://github.com/madhums/node-express-mongoose
https://github.com/icebob/vue-express-mongo-boilerplate
https://github.com/talyssonoc/node-api-boilerplate
But none of them take care of the ready state of the app before calling server.listen(port) which make the webserver starting to listen to the incoming requests. That surprises me a lot and I don't understand why
Code example of a webserver app with multiple services that we must wait for before accept incomings requests:
bin/www:
var app = require('../app');
// ...
var server = http.createServer(app);
server.listen(port);
app.js:
var express = require('express')
var mongoose = require('mongoose')
// **************
// Service 1 : database
mongoose.connect(process.env.DATABASE_URL).then(
() => {
debug('Database is connected')
},
err => {
debug('An error has occured with the database connection')
process.exit(1)
}
)
// **************
// **************
// Service 2
// Simulate a service that take 10 seconds to initialized
var myWeatherService = null
setTimeout(function() {
myWeatherService.getWeatherForTown = function(town, callback) {
weather = 'sun'
callback(null, weather)
}
}, 10*1000)
// **************
// **************
// Other services...
// **************
var app = express()
// Midllewares
app.use(/* some middleware 1 */)
app.use(/* some middleware 2 */)
app.use(/* some middleware 3 */)
app.use(/* some middleware ... */)
// Routes
app.get('/', function(req, res, next) {
res.json({'message': 'Welcome to my website'})
})
app.get('/users', function(req, res, next) {
Users.find({}).exec(function(err, users) {
if (err) {
res.json({'message': 'An error occured'})
return
}
res.json({'users': users})
})
})
app.get('/getParisWeather', function(req, res, next) {
Users.getWeatherForTown('Paris', function(err, weather) {
if (err) {
res.json({'message': 'An error occured'})
return
}
res.json({'town': 'Paris', weatcher: weather})
})
})
// ... others routes ...
module.exports = app
If I start my app, and then I call localhost:port/getParisWeather before the myWeatherService is initialized, I will get an error
I already think about a solution: move each service declaration in bin/www and let in app.js only code that concern the declaration of the express app:
bin/www:
var app = require('../app');
var mongoose = require('mongoose')
var server = null;
Promise.resolve()
.then(function () {
return new Promise(function (resolve, reject) {
// start service 1
console.log('Service 1 is ready')
resolve()
})
})
.then(function () {
return new Promise(function (resolve, reject) {
// start service 2
console.log('Service 2 is ready')
resolve()
})
})
.then(function () {
return new Promise(function (resolve, reject) {
// start other services...
console.log('Others services is ready')
resolve()
})
})
.then(function () {
return new Promise(function (resolve, reject) {
server = http.createServer(app);
server.listen(port);
console.log('Server start listenning')
})
})
.then(function () {
next()
})
.catch(next)
.finally(function () {
})
.done()
app.js:
var express = require('express')
var app = express()
// Midllewares
app.use(/* some middleware 1 */)
app.use(/* some middleware 2 */)
app.use(/* some middleware 3 */)
app.use(/* some middleware ... */)
// Routes
app.get('/', function(req, res, next) {
res.json({'message': 'Welcome to my website'})
})
app.get('/users', function(req, res, next) {
Users.find({}).exec(function(err, users) {
if (err) {
res.json({'message': 'An error occured'})
return
}
res.json({'users': users})
})
})
app.get('/getParisWeather', function(req, res, next) {
Users.getWeatherForTown('Paris', function(err, weather) {
if (err) {
res.json({'message': 'An error occured'})
return
}
res.json({'town': 'Paris', weatcher: weather})
})
})
// ... others routes ...
module.exports = app
But I know that put logic in bin/www is not a good practice, it must only contains the server start lines...
So, my question is, how we must start a webserver app to respect the bests practices // what is the bests practices to ?
I know that I can put everything in only one file and start the webserver at the end of this file, this is not my question. What I ask is how to do it in the good way and in the best practices
The answer really depends on what your ecosystem is like. If you know all of the services that your app will use, then you might try checking them in an expressjs middleware function that is called before the routing code. You can use a set of promises to keep track of service readiness and a boolean to tell whether all services are ready. If all services are ready, then the middleware function can call next(), but if not, then you might return an HTML page that tells the user the site is undergoing maintenance or isn't ready and they should try back later. I can see you encapsulating all those promises in a middleware function that manages whether or not they are ready as to not clutter your app.js or bin/www files.
Update:
If you want to prevent the server from listening until the services are ready, then you'll need to setup your own infrastructure in the same process or use something like supervisord to manage the processes. For example, you can setup a "startup" process that checks for the services. Once the services are ready, your startup process can fork and start the node server or create a child process that runs the server. You don't need to have any of the service checking logic in your node app; the assumption is that if it is started by the other process, then the services are already up and running. You can introduce a high-level process management system like supervisord, or keep it all in nodejs and use the child_process module. This approach will help to keep the "startup" code separate from the "run/app" code.
Consider a simple express api which returns 'OK' on port 3000.
const express = require('express');
const app = express();
app.get('/', (req, res) => {
res.send('OK');
});
app.listen(3000, () => {
console.log("App listening on port 3000");
});
This becomes ready to accept connections as soon as the app is fired up. Now let's sleep for 5 seconds to fake the database getting ready, and fire a 'ready' event manually afterwards. We start listening for connections when the event is caught.
const express = require('express');
const sleep = time => new Promise(resolve => setTimeout(resolve, time));
const app = express();
app.get('/', (req, res) => {
res.send('OK');
});
sleep(5000)
.then(() => {
process.emit("ready");
});
process.on("ready", () => {
app.listen(3000, () => {
console.log("App listening on port 3000");
});
});
Test if by going to http://localhost:3000. You'll see 'OK' only after 5 seconds.
Here, you don't have to emit the 'ready' event on process object itself. A custom EventEmitter object will do the job as well. The process object inherits from EventEmitter and is available globally. So it's a convenient way of listening to any global events.
I'm trying to build an endpoint that will receive a request, emit the request data to a WebSocket client, wait for an event, then send back the response using express + socketio. This question is similar to it: Wait for socketio event inside express route
1) Receive request at http://localhost:3000/endpoint
2) Emit the event to web sockets as 'req'
3) Wait for 'res' event from ws
4) Send the received events details as the response of express.
Here is how I'm implemented:
server.js
var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);
app.get('/', (req, res) => {
res.sendFile(__dirname + '/index.html');
});
var socket;
io.on('connection', function (s) {
socket = s;
});
http.listen(3000);
app.get('/endpoint', function (req, res) {
console.log('new request')
io.emit('req', { data: 'hello' });
socket.on('res', function (data) {
res.status(200).json(data);
});
});
index.html
<script src="/socket.io/socket.io.js"></script>
<script>
var socket = io();
socket.on('req', (data) => {
console.log(data)
socket.emit('res', data);
});
</script>
The script works fine for the first request on /endpoint. But if i hit the url again, it says
Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent
to the client
Please note that:
socket.on('res', function (data) {
res.status(200).json(data);
});
Is being called each time a socket is sending a response, thus showing the above error. You should unbind the listener inside the callback function.
Keep an array of express responses and set an id to each request. So it can be used later and delete if needed.
var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);
var timeout = require('connect-timeout');
var uuid = require('uuidv4');
var _ = require('lodash');
app.use(timeout('10s'));
app.get('/', (req, res) => {
res.sendFile(__dirname + '/index.html');
});
let responses = []
io.on('connection', (socket) => {
socket.on('res', (e) => {
var obj = _.find(responses, r => r.id === e.id);
obj.res.send(e)
_.remove(responses, r => r.id === e.id);
})
})
app.get('/endpoint', (req, res) => {
const id = uuid()
io.emit('req', { id, ip: req.ip, header: req.headers, method: req.method });
responses.push({ id, res })
});
http.listen(3000);
You're trying to do two different async tasks for the same data.
First, take your socket.on('res'...) out of the app.get().
Send back res.status(200) immediately with express to say you received the request and it is processing. Then send the socket message to the client using socket.io when it's complete. You'll want to save the connected users socket client ID and use io.to(socketId).emit(...data...) to do this
the other option is what I always do (assuming it's not a crazy large payload of data you're sending) Just use socket.io for the whole process.
client
function makeRequest () {
socket.on('data-complete--error', function ( error ) {
// ... message to user :(
// also remove these handlers when finished
socket.off('data-complete--error');
socket.off('data-complete--success');
});
socket.on('data-complete--success', function ( data ) {
// ... message to user :)
// ... handle data
// also remove these handlers when finished
socket.off('data-complete--error');
socket.off('data-complete--success');
});
socket.emit('request-data');
}
makeRequest();
server
move your stuff out and handle without using express at all
I know Javascript but very new to NodeJS. I want to reuse http request anywhere in my code, outside of server.js. Please take a look:
server.js
var http = require('http');
var mongodb = require('./db');
var settings = require('./settings');
var oauth = require('./oauth');
//start mongodb connection
mongodb.connect(function(error){
if (error) console.error(error);
else {
//set db as mongodb client
var db = mongodb.use();
//start http server
http.createServer(function(request,response){
//set POST body
request.body = '';
request.on('error',function(error){
console.error(error);
response.statusCode = 404;
response.end();
}).on('data',function(chunk){
request.body += chunk;
}).on('end',function(){
//database and http server ready, start logic
//Now I want to "require" a file here, such as test.js and call "request.headers" to work
});
}).listen(settings.httpPort,function(){
console.log('Server listening on '+settings.httpServer+':'+settings.httpPort);
});
}
});
And now I want to require test.js which do the following thing
test.js
console.log(request.headers);
Note: I don't want to use any framework like Express. I want to write my own things from scratch. Please help me, thank you so much.
If I understand correctly, the problem seems to be this: You want to access the
result of an asynchronous function (the HTTP request) in other modules. However,
Node's require() is synchronous; there is no asynchronous require(). There
are a few solutions to this, none of which will be unfamiliar if you know
JavaScript.
The simplest solution is to wrap your server.js module in a function that takes
a callback. Then, call the callback once the request is available, like so:
// server.js
'use strict';
// ...
module.exports = function(callback) {
// ...
http.createServer((req, res) => {
req.body = '';
// Call end with error on error
req.on('error', err => res.end(err));
// Append chunks to body
req.on('data', chunk => req.body += chunk);
// Call callback here
req.on('end', err => {
// Call with error as first argument on error
if (err) callback(err);
// Call with request as second argument on success
else callback(null, req);
});
}).listen(/*...*/);
// ...
};
And in your test.js file:
// test.js
'use strict';
const server = require('./server');
// Do something with the request here.
server((err, req) => {
if (err) console.error(err);
else console.log(req.headers);
});
There is a problem with this approach. Every time you want to access the
request, your server function will run all over again. What if you want to run
the server once and then have access to the request as many times as you want?
Consider using Node's events module for this. In your server.js file, you can
export an EventEmitter instance instead of a function. Emit appropriate events
in that module, and then you can add listeners in any other module that needs
access to the request. Your server.js file will look something like this:
// server.js
'use strict';
const EventEmitter = require('events');
const emitter = new EventEmitter();
// ...
http.createServer((req, res) => {
req.body = '';
req.on('error', err => res.end(err));
req.on('data', chunk => req.body += chunk);
// Emit events here:
req.on('end', err => {
// Emit 'error' event on error.
if (err) emitter.emit('error', err);
// Emit 'data' event on success.
else emitter.emit('data', req);
});
}).listen(/*...*/);
// ...
module.exports = emitter;
And in your test.js file:
// test.js
'use strict';
const server = require('./server');
// Do something on error event.
server.on('error', console.error);
// Do something on data event.
server.on('data', req => {
console.log(req.headers);
});
I need help in trying to solve this scenario
I have a file web.js. Over there I have
var express = require("express");
var app = express();
var web2 = require("./web2");
/* Code the start the server on the required port*/
app.get('/param1', function(req, res){
console.log("INSIDE GET METHOD OF WEB.JS");
});
module.exports.app = app
I have another file web2.js. over there I have
var web = require("./web");
app = web.app;
app.get('/param2', function(req, res){
console.log("INSIDE GET METHOD OF WEB2.JS");
});
While starting I get an error
TypeError: Cannot call method 'post' of undefined
If I remove the line 3 from web.js -- I am able to start the server, but a request for http:///param2 gives a 404
Updated scenario:
I am using pg database and I try to create a client that keeps an instance of the client(in web.js). I then pass this to other file(web2.js). In web.js I always get this client as null
in web.js I have the following code
var pg = require("pg");
var pgclient;
app.get('*', function(req,res,next){
pg.connect(process.env.DATABASE_URL, function(err, client, done) {
if(client != null){
pgclient = client;
console.log("Client connection with Postgres DB is established");
next();
}
}
}
require("./web2.js")(app, pgclient);
in web2.js, I have the following code
module.exports = function(app, pgclient){
app.get('/param1', function(req,res){
if(pgclient != null){
}
else{
res.send(500, "pgclient is NULL");
}
});
}
The code never reaches the if block(if(pgclient != null)) in web2.js
The problem is the cyclic dependency between web.js and web2.js. When web2.js requires web.js, web.js's module.exports hasn't been set yet. I would rather do something like this:
web.js
var express = require("express");
var app = express();
app.get("/param1", function (req, res) {
// ...
});
require("./web2")(app);
app.listen(/* port number */);
web2.js
module.exports = function (app) {
app.get("/param2", function (req, res) {
// ...
});
};
I tried to use connect-domain to handling error. In most cases it ok, but it fail with redis callback. How to fix this?
Here's my app
var http = require('http');
var express = require('express');
var connectDomain = require('connect-domain');
var redis = require("redis").createClient();
var app = express();
app.use(connectDomain());
app.get('/', function (req, res) {
throw new Error("Handler OK");
});
app.get('/error', function (req, res) {
redis.get("akey", function(err, reply) {
throw new Error("Handler error");
res.end("ok");
});
});
app.use(function (err, req, res, next) {
res.end(err.message);
});
http.createServer(app).listen(8989, function() {
console.log("Express server started ");
});
I use nodejs 0.8.16, all modules are latest
Not sure if the domain should be catching that or not - but you can capture redis errors by setting up an error handler, like this:
// handle redis connection temporarily going down without app crashing
redisClient.on("error", function(err) {
console.error("Error connecting to redis", err);
});
While the connection is broken your handler will keep getting called as redis tries to reconnect. If it's eventually successful everything will come back online on it's own.
You can also try https://www.npmjs.org/package/wait-for-redis. It ensures clients can wait for server to be up in case when clients start early.