How can I implement clustering with sequelize ORM i.e. multithreading - node.js

Below is a piece of code I wrote but isn't working. With this code - how can I refactor it to implement multithreading or clustering so that I can utilize the number of cpus and thus improve the performance of my application?
if (cluster.isMaster) {
console.log('Master ${process.pid} is running');
for (let i = 0; i < numOfCpus; i++) {
cluster.fork();
}
cluster.on('exit', (worker, code, signal) => {
console.log('worker ${worker.process.pid} died');
});
} else {
db.sequelize.sync({
force: false
}).then(function () {
app.listen(PORT, function (req, res) {
res.writeHead(200);
res.end('hey');
console.log("==> 🌎 Listening on port %s. Visit http://localhost:%s/ in your browser.", PORT, PORT);
});
console.log(' Worker ${process.pid} started');
});
}

I've been working on something similar. I saw in your comment that you moved on to create a Spring Boot application anyways, but just in case, I think your issue came from the callback you passed to the 'listen' method. You wrote the status 200 to the response header then ended the connection with res.end('hey');
Perhaps if you just call the 'listen' method and handle any other logic afterwards with the server.on('listening') method, it will solve your issue? It seems to work when I test it on my application. This is my bin/www so far, where app is an express application, and db is the Sequelize models:
#!/usr/bin/env node
/**
* Module dependencies.
*/
const http = require('http');
const cluster = require('cluster');
const os = require('os');
const config = require('../server/config')[process.env.NODE_ENV || 'development'];
const app = require('../server/app')(config);
const db = require('../server/models');
const log = config.log();
const numCPUs = os.cpus().length;
// Helper functions
/**
* Normalize a port into a number, string, or false.
*/
function normalizePort(val) {
const port = parseInt(val, 10);
if (Number.isNaN(port)) {
// named pipe
return val;
}
if (port >= 0) {
// port number
return port;
}
return false;
}
/**
* Get port from environment and store in Express.
*/
const port = normalizePort(process.env.PORT || '3000');
app.set('port', port);
/**
* Create HTTP server and listen on the provided port
*/
const server = http.createServer(app);
if (cluster.isMaster) {
log.info(`Master ${process.pid} is running`);
for (let i = 0; i < numCPUs; i += 1) {
cluster.fork();
}
cluster.on('exit', (worker) => {
log.fatal(`Worker ${worker.process.pid} just died`);
cluster.fork();
});
} else {
db.sequelize.sync({ force: false })
.then(() => {
log.info('Connected to database');
server.listen(port);
})
.catch((err) => {
log.fatal(err);
});
}
server.on('listening', () => {
const addr = server.address();
const bind = typeof addr === 'string'
? `pipe ${addr}`
: `port ${addr.port}`;
log.info(`Listening on ${bind}`);
});
// Handle server errors
server.on('error', (error) => {
if (error.syscall !== 'listen') {
throw error;
}
const bind = typeof port === 'string'
? `Pipe ${port}`
: `Port ${port}`;
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
log.fatal(`${bind} requires elevated privileges`);
process.exit(1);
break;
case 'EADDRINUSE':
log.fatal(`${bind} is already in use`);
process.exit(1);
break;
default:
log.info(error);
// throw error;
}
});

Related

How can I export socket.io connection to another controller

I am stuck in a problem.
I am making a Socket.IO connection in the bin file which is working, but can anyone tell me how I can export this connection to different controller. This is my bin file code.
var app = require('../app');
var debug = require('debug')('userservice:server');
var http = require('http');
/**
* Get port from environment and store in Express.
*/
var port = normalizePort(process.env.PORT || '3015');
app.set('port', port);
/**
* Create HTTP server.
*/
var server = http.createServer(app);
var io = require('socket.io')(server);
io.on('connection', (socket) => {
console.log('Connection made #######################################################.', socket.id);
socket.on('disconnect', () => {
console.log('Connection disconnected #######################################################.', socket.id);
});
});
/**
* Listen on provided port, on all network interfaces.
*/
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
There is plenty of techniques can be used to re-use the socket instance, an easy and simple one is to create a singular class, to be able to:
Initiate socket instance
Export the instance to other modules
socket.js:
let io;
module.exports = {
init: (server) => {
io = require('socket.io').listen(server); io.origins('*:*');
return io;
},
get: () => {
if (!io) {
throw new Error("socket is not initialized");
}
return io;
}
};
server.js:
const app = require('../app');
const http = require('http');
/**
* Get port from environment and store in Express.
*/
const port = '3015';
app.set('port', port);
/**
* Create HTTP server.
*/
const server = http.createServer(app);
const io = require('./socket.js').init(server);
io.on('connection', (socket) => {
console.log('Connection success', socket.id);
socket.on('disconnect', () => {
console.log('Connection disconnected', socket.id);
});
}
Now you can use it in other modules.
const io = require('./socket.js').get();
You could just create a socket.io controller module that exports a function that you call for every new connection.
So, in your current server file, you add this:
const {socketConnected} = require('socketController.');
And, you modify this portion to call it for each new socket:
var server = http.createServer(app);
var io = require('socket.io')(server);
io.on('connection', (socket) => {
console.log('Connection made #######################################################.', socket.id);
// tell controller about new socket (and pass the io instance)
socketConnected(socket, io);
socket.on('disconnect', () => {
console.log('Connection disconnected #######################################################.', socket.id);
});
});
Then, your socket controller can be like this:
module.exports.socketConnected = function(socket, io) {
// new socket.io socket connected
console.log(`controller: got socket.io connection ${socket.id}`);
// register appropriate event handlers on the socket here
}
We can export the socket module in every file by creating a global object in such a way.
let io;
const connectedUsers = [];
const setupSocketIO = function (server) {
io = require('socket.io')(server, { cors: { origin: '*' } });
io.on('connection', function (socket) {
connectedUsers[connectedUsers.length] = socket.id;
socket.on('getConnectedUsers', () => {
io.sockets.emit('returnConnectedUsers', connectedUsers.length);
});
socket.on('disconnect', () => {
let socketIdToRemoveIndex = -1;
for (let i = 0; i < connectedUsers.length; i++) {
if (connectedUsers[i] === socket.id) {
socketIdToRemoveIndex = i;
}
}
if (socketIdToRemoveIndex !== -1) {
connectedUsers.splice(socketIdToRemoveIndex, 1);
}
io.sockets.emit('connectedUsers', connectedUsers.length);
});
});
};
const Socket = function () {
return {
emit: function (event, data) {
io.sockets.emit(event, data);
},
to: function (roomId, event, data) {
io.sockets.to(roomId).emit(event, data);
},
};
};
exports.setupSocketIO = setupSocketIO;
exports.Socket = Socket;
And in file or componenet, we want to use.
const getAllProjects = async (req, res) => {
let Socket = require('../sockets').Socket();
Socket.emit('SOCKET_PUSH_NOTIFICATION', { data: 'This is random data' });
Socket.to('SOCKET_PUSH_NOTIFICATION', 'event', { data: 'Muhammad' });
}

ExpressJS not call ajax get method

I created a sample ExpressJS App by using Express Application Generator
I want to call localhost:3000/data to get some data. I added these code to app.js:
app.get('/data', function(req, res){
debug("get data");
});
Also I make ajax request in client side js file :
$.ajax({
url: 'http://localhost:3000/data/',
//url: 'http://localhost:3000/data', -> also not work
type:'GET',
dataType: 'json',
success: (data) => {
console.log("get data from client js", data);
}
});
but my app.get method never call. GET /data/ 404 written in console screen. Also when I typed http://localhost:3000/data on browser I got Not Found error. how can I fix this ?
my www file :
#!/usr/bin/env node
/**
* Module dependencies.
*/
var app = require('../app');
var debug = require('debug')('myapp:server');
var http = require('http');
/**
* Get port from environment and store in Express.
*/
var port = normalizePort(process.env.PORT || '3000');
app.set('port', port);
/**
* Create HTTP server.
*/
var server = http.createServer(app);
/**
* Listen on provided port, on all network interfaces.
*/
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
/**
* Normalize a port into a number, string, or false.
*/
function normalizePort(val) {
var port = parseInt(val, 10);
if (isNaN(port)) {
// named pipe
return val;
}
if (port >= 0) {
// port number
return port;
}
return false;
}
/**
* Event listener for HTTP server "error" event.
*/
function onError(error) {
if (error.syscall !== 'listen') {
throw error;
}
var bind = typeof port === 'string'
? 'Pipe ' + port
: 'Port ' + port;
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
console.error(bind + ' requires elevated privileges');
process.exit(1);
break;
case 'EADDRINUSE':
console.error(bind + ' is already in use');
process.exit(1);
break;
default:
throw error;
}
}
/**
* Event listener for HTTP server "listening" event.
*/
function onListening() {
var addr = server.address();
var bind = typeof addr === 'string'
? 'pipe ' + addr
: 'port ' + addr.port;
debug('Listening on ' + bind);
}
I'm not exactly sure where your problem lies because there isn't enough code displayed to determine that. instead I have written a simple express server for you to use:
const express = require('express');
const bodyParser = require('body-parser');
const CORS = require('cors');
const app = express();
app.use(bodyParser.json());
app.use(CORS());
app.get('/data', (req, res) => {
res.json({message: "working"});
});
app.listen(5000, () => {
console.log('Server listening on port 5000');
});

babel-node, cannot get server to listen on port

I am currently trying to run node es6 with babel on a docker container and am running into some issues getting the app to start listening on port 3000. I can see where the app.js file is being processed as I am seeing the database connection code running. The problem seems to be even though app.js is getting called I am not seeing anything from /bin/www getting called which would result in the server listening on port 3000.
This is the command that is getting called to start the container:
nodemon ./bin/www -L --exec babel-node --inspect=0.0.0.0:56745
app.js:
……
(async () => {
try {
console.log('about to start the database connection... - 1');
mongoose.set('useCreateIndex', true);
mongoose.Promise = global.Promise;
console.log('about to start the database connection... - 2');
setTimeout(async () => {
await mongoose.connect(process.env.DB_HOST, {useNewUrlParser: true});
}, 60000);
//await mongoose.connect(process.env.DB_HOST, {useNewUrlParser: true});
console.log('about to start the database connection... - 3');
let db = mongoose.connection;
console.log('about to start the database connection... - 4');
db.on('error', console.error.bind(console, 'MongoDB connection error:'));
console.log('about to start the database connection... - 5');
} catch (e) {
console.log('We have an error.....');
console.log(e);
}
})()
let app = express();
app.use(logger('dev'));
app.use(express.json());
app.use(express.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use(helmet());
app.use(methodOverride());
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'pug');
app.use('/', indexRouter);
app.use('/users', usersRouter);
app.use('/group', groupsRouter);
// catch 404 and forward to error handler
app.use(function(req, res, next) {
next(createError(404));
});
// error handler
app.use((err, req, res, next) => {
if (process.env.NODE_ENV === "development") {
app.use((err, req, res, next) => {
if (err instanceof NotFoundError) {
//res.status(404).send(err.message);
res.statusCode = 404;
return res.json({
errors: [err.stack]
});
} else {
//res.status(500).send('An error occurred while processing your request.');
res.statusCode = 500;
return res.json({
errors: [err.stack]
//errors: ['An error occurred while processing your request.']
});
}
});
}
// production error handler
// no stacktraces leaked to user
console.log('about to begin configurations..... 7');
if (process.env.NODE_ENV !== "development") {
app.use((err, req, res, next) => {
if (err instanceof NotFoundError) {
//res.status(404).send(err.message);
res.statusCode = 404;
return res.json({
errors: [err.stack]
});
} else {
//res.status(500).send('An error occurred while processing your request.');
res.statusCode = 500;
return res.json({
errors: [err.stack]
//errors: ['An error occurred while processing your request.']
});
}
});
}
});
module.exports = app;
/bin/www:
#!/usr/bin/env node
/**
* Module dependencies.
*/
let app = require('../app');
let debug = require('debug’)(‘myapp:server');
let http = require('http');
/**
* Get port from environment and store in Express.
*/
let port = normalizePort(process.env.PORT || '3000');
app.set('port', port);
/**
* Create HTTP server.
*/
let server = http.createServer(app);
/**
* Listen on provided port, on all network interfaces.
*/
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
/**
* Normalize a port into a number, string, or false.
*/
const normalizePort = (val) => {
debug('port = ' + val);
let port = parseInt(val, 10);
if (isNaN(port)) {
// named pipe
return val;
}
if (port >= 0) {
// port number
return port;
}
return false;
}
/**
* Event listener for HTTP server "error" event.
*/
const onError = (error) => {
debug('Houston we have a problem');
if (error.syscall !== 'listen') {
throw error;
}
let bind = typeof port === 'string'
? 'Pipe ' + port
: 'Port ' + port;
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
console.error(bind + ' requires elevated privileges');
process.exit(1);
break;
case 'EADDRINUSE':
console.error(bind + ' is already in use');
process.exit(1);
break;
default:
throw error;
}
}
/**
* Event listener for HTTP server "listening" event.
*/
const onListening = () => {
debug('Listening');
let addr = server.address();
let bind = typeof addr === 'string'
? 'pipe ' + addr
: 'port ' + addr.port;
debug('Listening on ' + bind);
}
.babelrc:
{
"presets": ["env"],
"plugins": ["transform-object-rest-spread", "transform-async-to-generator"]
}
Update:
The issue seems to be with node and the arrow operator. When I changed to the function keyword, it started working. I added the following to my .bablerc file:
{
"presets": ["env"],
"plugins": ["transform-object-rest-spread", "transform-async-to-generator", "transform-es2015-arrow-functions"]
}
but it is still an issue. How can I use the arrow operator with nodejs?
If you aren't able to progress past the normalizePort call, are you sure it exists at that time?
You need to move the definition of the function above the place you use it.
(If you are used to old, pre-ES6 "function hoisting" using vars and functions, you should note that that does not work with const and let statements.)

A standard for error handling and graceful shutdown of Node/Express server

Below is a basic script I usually use for my express servers. Specifically, I am looking for answers to the following questions: Is it needed to include SIGINT and SIGTERM process signal handlers in production? Do I need to actually include an uncaught exception handler? What about tracking and cleaning up sockets? Do I use process.exit(), process.exit(0), or process.exit(1), and if so, where exactly and why?
var express = require('express');
var sockets = [];
process.on('SIGINT', cleanup);
process.on('SIGTERM', cleanup);
var port = 7000;
var app = express();
var server = app.listen(port, function () {
console.log(`app listening on port ${port}`);
})
.on("connection", onConnection)
.on("listening", onListening)
.on("error", onError)
.on("close", onClose);
var cleanup = function () {
server.close(function () {
console.log("Closed out remaining connections.");
process.exit();
});
sockets.forEach(function (socket) {
socket.destroy();
});
setTimeout(function () {
console.log("Could not close connections in time, forcing shut down");
process.exit(1);
}, 30 * 1000);
}
function onConnection(socket) {
sockets.push(socket);
}
function onError(error) {
if (error.syscall !== 'listen') throw error;
var bind = typeof port === 'string'? 'Pipe ' + port : 'Port ' + port;
switch (error.code) {
case 'EACCES':
console.error(bind + ' requires elevated privileges');
process.exit(1);
break;
case 'EADDRINUSE':
console.error(bind + ' is already in use');
process.exit(1);
break;
default:
throw error;
}
}
function onListening() {
var addr = server.address();
var bind = typeof addr === 'string'? 'pipe ' + addr : 'port ' + addr.port;
console.log('Listening on ' + bind);
}

Express and Cluster node modules in the same app

I am creating an app that allows users to upload files in their browsers to a server.
I started it by typing express in the folder of my project. With that express created all the folders and such. It created a bin folder containing a www file that starts a lot of stuff for us like the ports.
At first i only had one core so i was doing it like so:
var express = require('express');
var multer = require('multer');
var app = express();
var done = false;
var fileSize = 0;
app.use(multer({ dest: './uploads/',
rename: function (fieldname, filename){
return filename;
},
onFileUploadStart: function(file){
console.log(file.originalname + ' is starting...');
},
onFileUploadComplete: function(file){
console.log(file.fieldname + ' uploaded to ' + file.path);
done = true;
},
onFileUploadData: function(file, data){
fileSize += data.length;
console.log("FileUpload " + fileSize);
}
}));
app.get('/', function(request, response){
response.sendFile(__dirname + '/index.html');
});
app.post('/upload/', function(request, response){
console.log(request.body);
if (done == true)
response.end('file uploaded');
});
module.exports = app;
Eventually i needed to user a machine with several cores in order to be able to respond to more client requests. So I am trying to use the cluster module.
I changed the code accordingly:
var cluster = require('cluster');
if(cluster.isMaster){
var numCPUS = require('os').cpus().length;
for(var i=0; i<numCPUS; i++)
cluster.fork();
cluster.on('exit', function(worker, code, signal) {
console.log('worker ' + worker.process.pid + ' died');
});
cluster.on('online', function(worker) {
console.log('worker ' + worker.process.pid + ' started');
});
}else{
var express = require('express');
var multer = require('multer');
var app = express();
var done = false;
var fileSize = 0;
app.use(multer({ dest: './uploads/',
rename: function (fieldname, filename){
return filename;
},
onFileUploadStart: function(file){
console.log(file.originalname + ' is starting...');
},
onFileUploadComplete: function(file){
console.log(file.fieldname + ' uploaded to ' + file.path);
done = true;
},
onFileUploadData: function(file, data){
fileSize += data.length;
console.log("FileUpload " + fileSize);
}
}));
app.get('/', function(request, response){
response.sendFile(__dirname + '/index.html');
});
app.post('/upload/', function(request, response){
console.log(request.body);
if (done == true)
response.end('file uploaded');
});
module.exports = app;
}
The problem is that when i do this i am getting the following error in the console:
Run: node ./bin/www
Result:
app.set('port', port);
TypeERror: Cannot call method 'set' of undefined at Object.<anonymous>
This happens since i placed the entire code inside the forked childs of the master. Anyone knows why this happens and how i can fix it?
EDIT:
In here you can see the bin/www file where the port and some other configurations are set:
#!/usr/bin/env node
/**
* Module dependencies.
*/
var app = require('../app');
var debug = require('debug')('dropbox:server');
var http = require('http');
/**
* Get port from environment and store in Express.
*/
var port = normalizePort(process.env.PORT || '3000');
app.set('port', port);
/**
* Create HTTP server.
*/
var server = http.createServer(app);
/**
* Listen on provided port, on all network interfaces.
*/
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
/**
* Normalize a port into a number, string, or false.
*/
function normalizePort(val) {
var port = parseInt(val, 10);
if (isNaN(port)) {
// named pipe
return val;
}
if (port >= 0) {
// port number
return port;
}
return false;
}
/**
* Event listener for HTTP server "error" event.
*/
function onError(error) {
if (error.syscall !== 'listen') {
throw error;
}
var bind = typeof port === 'string'
? 'Pipe ' + port
: 'Port ' + port
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
console.error(bind + ' requires elevated privileges');
process.exit(1);
break;
case 'EADDRINUSE':
console.error(bind + ' is already in use');
process.exit(1);
break;
default:
throw error;
}
}
/**
* Event listener for HTTP server "listening" event.
*/
function onListening() {
var addr = server.address();
var bind = typeof addr === 'string'
? 'pipe ' + addr
: 'port ' + addr.port;
debug('Listening on ' + bind);
}

Resources