I tried to use connect-domain to handling error. In most cases it ok, but it fail with redis callback. How to fix this?
Here's my app
var http = require('http');
var express = require('express');
var connectDomain = require('connect-domain');
var redis = require("redis").createClient();
var app = express();
app.use(connectDomain());
app.get('/', function (req, res) {
throw new Error("Handler OK");
});
app.get('/error', function (req, res) {
redis.get("akey", function(err, reply) {
throw new Error("Handler error");
res.end("ok");
});
});
app.use(function (err, req, res, next) {
res.end(err.message);
});
http.createServer(app).listen(8989, function() {
console.log("Express server started ");
});
I use nodejs 0.8.16, all modules are latest
Not sure if the domain should be catching that or not - but you can capture redis errors by setting up an error handler, like this:
// handle redis connection temporarily going down without app crashing
redisClient.on("error", function(err) {
console.error("Error connecting to redis", err);
});
While the connection is broken your handler will keep getting called as redis tries to reconnect. If it's eventually successful everything will come back online on it's own.
You can also try https://www.npmjs.org/package/wait-for-redis. It ensures clients can wait for server to be up in case when clients start early.
Related
When I generate a webserver with express-generator, I get this folder structure :
bin/www
views/...
app.js
package.json
...
bin/www call app.js like that :
var app = require('../app');
// ...
var server = http.createServer(app);
server.listen(port);
app.js create the app like that :
var express = require('express')
var mongoose = require('mongoose')
mongoose.connect(process.env.DATABASE_URL).then(
() => {
debug('Database is connected')
},
err => {
debug('An error has occured with the database connection')
process.exit(1)
}
)
var app = express()
// Midllewares
app.use(/* some middleware 1 */)
app.use(/* some middleware 2 */)
app.use(/* some middleware 3 */)
app.use(/* some middleware ... */)
// Routes
app.get('/', function(req, res, next) {
res.json({'message': 'Welcome to my website'})
})
app.get('/users', function(req, res, next) {
Users.find({}).exec(function(err, users) {
if (err) {
res.json({'message': 'An error occured'})
return
}
res.json('users': users)
})
})
// ... others routes ...
module.exports = app
ok, this is the webserver boilerplate from express-generator. But if I want to start my app by the good way, I must call process.send('ready') when my app is ready. ("ready" mean that all services are ready to use: database, redis, scheduler...) (call process.send('ready') when your app is ready is a best practice to know that your webserver app si ready. This signal can be used by process management or other system)
The probleme is that in bin/www, the app is started (server.listen() is called) without insurance that the database connection is established. In other word, without the insurance that the webserver app is ready to listen to the traffic.
I read that start the server in bin/www is a best practice
The above example is not complete, we can considere that we have an app with multiple services that we must start before accept requests (services examples: redis, job scheduler, database connection, ftp connection to another server...)
I already check some popular and advanced boilerplate of Node.js app :
https://github.com/sahat/hackathon-starter
https://github.com/kriasoft/nodejs-api-starter
https://github.com/madhums/node-express-mongoose
https://github.com/icebob/vue-express-mongo-boilerplate
https://github.com/talyssonoc/node-api-boilerplate
But none of them take care of the ready state of the app before calling server.listen(port) which make the webserver starting to listen to the incoming requests. That surprises me a lot and I don't understand why
Code example of a webserver app with multiple services that we must wait for before accept incomings requests:
bin/www:
var app = require('../app');
// ...
var server = http.createServer(app);
server.listen(port);
app.js:
var express = require('express')
var mongoose = require('mongoose')
// **************
// Service 1 : database
mongoose.connect(process.env.DATABASE_URL).then(
() => {
debug('Database is connected')
},
err => {
debug('An error has occured with the database connection')
process.exit(1)
}
)
// **************
// **************
// Service 2
// Simulate a service that take 10 seconds to initialized
var myWeatherService = null
setTimeout(function() {
myWeatherService.getWeatherForTown = function(town, callback) {
weather = 'sun'
callback(null, weather)
}
}, 10*1000)
// **************
// **************
// Other services...
// **************
var app = express()
// Midllewares
app.use(/* some middleware 1 */)
app.use(/* some middleware 2 */)
app.use(/* some middleware 3 */)
app.use(/* some middleware ... */)
// Routes
app.get('/', function(req, res, next) {
res.json({'message': 'Welcome to my website'})
})
app.get('/users', function(req, res, next) {
Users.find({}).exec(function(err, users) {
if (err) {
res.json({'message': 'An error occured'})
return
}
res.json({'users': users})
})
})
app.get('/getParisWeather', function(req, res, next) {
Users.getWeatherForTown('Paris', function(err, weather) {
if (err) {
res.json({'message': 'An error occured'})
return
}
res.json({'town': 'Paris', weatcher: weather})
})
})
// ... others routes ...
module.exports = app
If I start my app, and then I call localhost:port/getParisWeather before the myWeatherService is initialized, I will get an error
I already think about a solution: move each service declaration in bin/www and let in app.js only code that concern the declaration of the express app:
bin/www:
var app = require('../app');
var mongoose = require('mongoose')
var server = null;
Promise.resolve()
.then(function () {
return new Promise(function (resolve, reject) {
// start service 1
console.log('Service 1 is ready')
resolve()
})
})
.then(function () {
return new Promise(function (resolve, reject) {
// start service 2
console.log('Service 2 is ready')
resolve()
})
})
.then(function () {
return new Promise(function (resolve, reject) {
// start other services...
console.log('Others services is ready')
resolve()
})
})
.then(function () {
return new Promise(function (resolve, reject) {
server = http.createServer(app);
server.listen(port);
console.log('Server start listenning')
})
})
.then(function () {
next()
})
.catch(next)
.finally(function () {
})
.done()
app.js:
var express = require('express')
var app = express()
// Midllewares
app.use(/* some middleware 1 */)
app.use(/* some middleware 2 */)
app.use(/* some middleware 3 */)
app.use(/* some middleware ... */)
// Routes
app.get('/', function(req, res, next) {
res.json({'message': 'Welcome to my website'})
})
app.get('/users', function(req, res, next) {
Users.find({}).exec(function(err, users) {
if (err) {
res.json({'message': 'An error occured'})
return
}
res.json({'users': users})
})
})
app.get('/getParisWeather', function(req, res, next) {
Users.getWeatherForTown('Paris', function(err, weather) {
if (err) {
res.json({'message': 'An error occured'})
return
}
res.json({'town': 'Paris', weatcher: weather})
})
})
// ... others routes ...
module.exports = app
But I know that put logic in bin/www is not a good practice, it must only contains the server start lines...
So, my question is, how we must start a webserver app to respect the bests practices // what is the bests practices to ?
I know that I can put everything in only one file and start the webserver at the end of this file, this is not my question. What I ask is how to do it in the good way and in the best practices
The answer really depends on what your ecosystem is like. If you know all of the services that your app will use, then you might try checking them in an expressjs middleware function that is called before the routing code. You can use a set of promises to keep track of service readiness and a boolean to tell whether all services are ready. If all services are ready, then the middleware function can call next(), but if not, then you might return an HTML page that tells the user the site is undergoing maintenance or isn't ready and they should try back later. I can see you encapsulating all those promises in a middleware function that manages whether or not they are ready as to not clutter your app.js or bin/www files.
Update:
If you want to prevent the server from listening until the services are ready, then you'll need to setup your own infrastructure in the same process or use something like supervisord to manage the processes. For example, you can setup a "startup" process that checks for the services. Once the services are ready, your startup process can fork and start the node server or create a child process that runs the server. You don't need to have any of the service checking logic in your node app; the assumption is that if it is started by the other process, then the services are already up and running. You can introduce a high-level process management system like supervisord, or keep it all in nodejs and use the child_process module. This approach will help to keep the "startup" code separate from the "run/app" code.
Consider a simple express api which returns 'OK' on port 3000.
const express = require('express');
const app = express();
app.get('/', (req, res) => {
res.send('OK');
});
app.listen(3000, () => {
console.log("App listening on port 3000");
});
This becomes ready to accept connections as soon as the app is fired up. Now let's sleep for 5 seconds to fake the database getting ready, and fire a 'ready' event manually afterwards. We start listening for connections when the event is caught.
const express = require('express');
const sleep = time => new Promise(resolve => setTimeout(resolve, time));
const app = express();
app.get('/', (req, res) => {
res.send('OK');
});
sleep(5000)
.then(() => {
process.emit("ready");
});
process.on("ready", () => {
app.listen(3000, () => {
console.log("App listening on port 3000");
});
});
Test if by going to http://localhost:3000. You'll see 'OK' only after 5 seconds.
Here, you don't have to emit the 'ready' event on process object itself. A custom EventEmitter object will do the job as well. The process object inherits from EventEmitter and is available globally. So it's a convenient way of listening to any global events.
i have a nodejs server which is getting list of a collection from mongodb . Here is its code . since am new to sockets so ..
const express = require("express");
const app = express();
const http = require("http").Server(app);
const socketio = require('socket.io');
after that iam simply getting data in a route . and one thing more all code is in one file and i do need express route as there are other routes in app. here is the mongodb code for getting list
app.post("/getAllOfferManagement",
async (req, res) => {
try {
MongoClient.connect(url,
function(err, db) {
if (err) throw err;
var dbo = db.db("realtime");
dbo
.collection("offer")
.find({})
.toArray(function(err,
result) {
if (err) throw err;
// console.log('getting it ');
res.send(result);
db.close();
});
});
} catch (err) {
res.send("error");
}
}); // its all working fine when i hit the route
http.listen(5000, function() {
console.log("Server Started!");
});
//serversidecode ends here
Now am getting the data through angular and here is the code for it
$scope.getAllOffer = function() {
$scope.mongoloader = true;
//nodejs api endpoint
$http.post("http://localhost:5000/getAllOffer").then(function(res) {
$scope.offersArray = res.data;
console.log('data here', res.data);
});
};
the above works fine . but i need to get data in realtime e.g when somone insert new doc in mongodb the the view get updates . am new to sockets so any help is appreciated. Thanks
For this u have to add an event to backend and as well as in frontend
Backend
io.on('connection', (socket) => {
console.log(socket.id);
socket.on('SEND_TITLE', function(data){
io.emit('RECEIVE_TITLE', data);
console.log('data',data)
})
});
For frontend u have to use socket io client module
import io from "socket.io-client";
socket = io('your backend host url')
socket.on('RECEIVE_TITLE', function(data)
Console. Log(data);
});
Frontend syntax could differ in angular.
As I am not familiar with angular
For more information visit.
Forclient side.
https://socket.io/docs/client-api/
For server.
https://socket.io/docs/server-api/
I have a REST api hosted on Amazon EC2, which is written with Nodejs (Express).
In a particular REST call, a reply of about 5MB is sent to the client. Before the client completely receives the reply, client prints following error message.
Premature end of Content-Length delimited message body
I added a connection listener in nodejs server like below to check what is going on the server.
var app = express();
var server = http.createServer(app);
var port = app.get('port');
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
server.on('connection', function (socket) {
log.debug('SOCKET OPENED' + JSON.stringify(socket.address()));
socket.setTimeout(300000); //5 minute timeout
socket.on('end', function () {
log.debug('SOCKET END: other end of the socket sends a FIN packet');
});
socket.on('timeout', function () {
log.warn('SOCKET TIMEOUT');
});
socket.on('error', function (error) {
log.warn('SOCKET ERROR: ' + JSON.stringify(error));
});
socket.on('close', function (had_error) {
log.debug('SOCKET CLOSED. IT WAS ERROR: ' + had_error);
});
});
I observed that SOCKET TIMEOUT gets logged in backend. In above code, I have increased the socket timeout to 5 minutes, but it doesn't seem to have any effect.
Earlier I had the REST API hosted in Google compute engine, and I didn't have this problem back then.
What could be the problem here?
Edit: Here is the code of REST API call.
I have following code in my app.js
require('./routes/index')(app);
Following is the index.js of routes directory.
var changeCase = require('change-case');
var express = require('express');
var routes = require('require-dir')();
module.exports = function (app) {
Object.keys(routes).forEach(function (routeName) {
var router = express.Router();
require('./' + routeName)(router);
app.use('/api/' + changeCase.paramCase(routeName), router);
});
};
As it can be seen, it loops through all the js files in the routes directory and registers the file name as the URL path in app.
Here is the code of this particular route for which I face this problem.
module.exports = function (router) {
router.get("/fetch", function (req, res, next) {
itemModel.fetch(req.user.clientId, function (error, items) {
if (error) {
res.status(500).json({error: error});
} else {
res.json(items); //items is a JSON array
}
});
});
}
Setting timeout for the HTTP server resolved the issue.
var server = http.createServer(app);
var port = app.get('port');
server.listen(port);
server.setTimeout(300000, function (socket) {
});
I am using the express framework and would like to connect to a mongodb without using mongoose, but with the native nodejs Mongodb driver. How can I do this without creating a new connection every time?
To handle get or post requests I currently open a new connection to the db for every request and close it on completion of the request. Is there a better way to do this? Thanks in advance.
Following the example from my comment, modifying it so that the app handles errors rather than failing to start the server.
var express = require('express');
var mongodb = require('mongodb');
var app = express();
var MongoClient = require('mongodb').MongoClient;
var dbURL = "mongodb://localhost:27017/integration_test";
var db;
// Initialize connection once
MongoClient.connect(dbURL, function(err, database) {
if(err) return console.error(err);
db = database;
// the Mongo driver recommends starting the server here
// because most apps *should* fail to start if they have no DB.
// If yours is the exception, move the server startup elsewhere.
});
// Reuse database object in request handlers
app.get("/", function(req, res, next) {
var collection = "replicaset_mongo_client_collection";
db.collection(collection).find({}, function(err, docs) {
if(err) return next(err);
docs.each(function(err, doc) {
if(doc) {
console.log(doc);
}
else {
res.end();
}
});
});
});
app.use(function(err, req, res){
// handle error here. For example, logging and
// returning a friendly error page
});
// Starting the app here will work, but some users
// will get errors if the db connection process is slow.
app.listen(3000);
console.log("Listening on port 3000");
var mongodb = require('mongodb');
var uri = 'mongodb://localhost:27017/dbname';
module.exports = function(callback) {
mongodb.MongoClient.connect(uri, callback);
};
Ad this snippet in a file say connect.js and then require this file(connect.js) in your file where you are declaring your functions for http requests.
I had an expectation that middleware that is bound to a Domain would be handled by the error handler for that domain.
In Express this did not turn out to be true.
I created a repository to illustrate this issue
https://github.com/rook2pawn/express-domains-issue
var app = express();
app.get('/',d.bind(function(req,res,next) {
throw new Error("error")
}));
var server = http.createServer(app);
Will not route the error to the domain error handler registered at d
whereas
var app = d.bind(function(req,res,next) {
throw new Error("error")
});
var server = http.createServer(app);
Will properly route the error to the domain without express.
Requesting any comments or thoughts about this?
It's just a bad example, because express wraps middleware in try-catch. This works:
app.get('/',d.bind(function(req,res,next) {
process.nextTick(function () {
throw new Error("error")
})
}));
By document of node js:
"This method is almost identical to domain.bind(callback). However, in addition to catching thrown errors, it will also intercept Error objects sent as the first argument to the function."
and I write a demo code:
var domain = require('domain');
var fs = require('fs');
var d = domain.create();
require('http').createServer(function(req, res, next) {
d.on('error', function (err) {
console.log(err);
res.writeHead(500, "content-type: plain/text");
res.end("Something missing!");
});
// This is for async
function readFile(filename, cb) {
fs.readFile(filename, 'utf8', d.bind(function (er, data) {
return cb(er, data ? JSON.parse(data) : null);
}));
}
readFile("unknow file");
// This is for sync
(d.bind(function() {
throw new Error();
}))();
}).listen(1337);
=> d.bind total can resolve all error sync and async to domain. That is the definition