I have a Node app running on AWS Elastic Beanstalk (so Node running behind Nginx). I am running socket IO with redis as the memory store and have Node running clustered using the cluster module. Generally everything works great but every now and then I get a user trying to connect that throws an undefined error on the connection.remoteAddress. My code looks like this for the connections:
if (cluster.isMaster) {
/*
------------------------------------------------------------------------------------
This stores our workers. We need to keep them to be able to reference
them based on source IP address. It's also useful for auto-restart
We also setup a message listener to every worker in order to blast AND FILTER
socket io updates to all nodes.
------------------------------------------------------------------------------------
*/
var workers = [];
var messageRelay = function(msg) {
for(var i = 0; i < workers.length; i++) {
workers[i].send(msg);
}
};
var spawn = function(i) {
workers[i] = cluster.fork();
console.log("Hello from worker %s",workers[i].process.pid);
workers[i].on('message', messageRelay);
/*
----------------------------------------
Restart worker if it gets destroyed
----------------------------------------
*/
workers[i].on('disconnect', function(worker) {
console.log('Worker disconnected');
});
workers[i].on('exit', function(worker, code, signal) {
console.log('respawning worker', i);
spawn(i);
});
};
for (var i = 0; i < cpuCount; i++) {
spawn(i);
}
/*
--------------------------------------------------------------------------------------------
Helper function for getting a worker index based on IP address (supports IPv4 AND IPv6)
This is a hot path so it should be really fast. The way it works
is by converting the IP address to a number by removing the dots (for IPv4) and removing
the :: for IPv6, then compressing it to the number of slots we have.
Compared against "real" hashing (from the sticky-session code) and
"real" IP number conversion, this function is on par in terms of
worker index distribution only much faster.
--------------------------------------------------------------------------------------------
*/
var workerIndex = function (ip, len) {
var _ip = ip.split(/['.'|':']/),
arr = [];
for (el in _ip) {
if (_ip[el] == '') {
arr.push(0);
}
else {
arr.push(parseInt(_ip[el], 16));
}
}
return Number(arr.join('')) % len;
}
/*
------------------------------------------------------------------------------------
Create the outside facing server listening on our port.
------------------------------------------------------------------------------------
*/
var server = net.createServer({ pauseOnConnect: true }, function(connection) {
/*
------------------------------------------------------------------------------------
We received a connection and need to pass it to the appropriate
worker. Get the worker for this connection's source IP and pass
it the connection.
------------------------------------------------------------------------------------
*/
if(connection.remoteAddress === undefined) {
console.log("BLEH: %o ", connection.remoteAddress);
return;
}
else {
var worker = workers[workerIndex(connection.remoteAddress, cpuCount)];
worker.send('sticky-session:connection', connection);
}
}).listen(port, function() {
console.log("Spun up worker %s", process.pid);
console.log('Server listening on *:' + port);
});
}
else {
var sio = require('socket.io');
var redis = require('socket.io-redis');
var ioEvents = require(__base + 'lib/ioEvents');
var app = new express();
/*
------------------------------------------------------------------------------------
Note we don't use a port here because the master listens on it for us.
------------------------------------------------------------------------------------
*/
var server = app.listen(0, 'localhost'),
io = sio(server);
/*
----------------------------------------------------------------------------------------------
Using Redis as the store instead of memory. This allows us to blast socket updates
to all processes (unfiltered). For example, we can do io.sockets.emit("message")
and it will be distributed to all node processes.
We cannot filter these messages to specific socket connections or specific configurations
(e.g. updateSquares(socket)), in order to do that we must blast an update to all workers
and let each process filter the request individually.
----------------------------------------------------------------------------------------------
*/
io.adapter(redis({host:'localhost', port: portRedis}));
/*
------------------------------------------------------------------------------------
Setup the socket listeners
------------------------------------------------------------------------------------
*/
ioEvents.incoming(io);
/*
------------------------------------------------------------------------------------
Listen to master for worker process updates
------------------------------------------------------------------------------------
*/
process.on('message', function(message, connection) {
/*
------------------------------------------------------------------------------------
Listen for special updates to all nodes
------------------------------------------------------------------------------------
*/
if(message.squareUpdate) {
console.log("worker %s received message %o", process.pid, message.squareUpdate);
ioEvents.squaresForceUpdate(message.squareUpdate);
}
/*
------------------------------------------------------------------------------------
If it's not a special message, then check to make sure it's just a sticky-session
Otherwise, just bail, no need to do anything else
------------------------------------------------------------------------------------
*/
if (message !== 'sticky-session:connection') {
return;
}
/*
------------------------------------------------------------------------------------
| Emulate a connection event on the server by emitting the
| event with the connection the master sent us.
------------------------------------------------------------------------------------
*/
server.emit('connection', connection);
connection.resume();
});
So the problem lies in the section above with the "BLEH" log. For some reason, remoteAddress is undefined...but only SOMETIMES. Most of the connections look just fine, but randomly I'll get a user trying to connect that throws that error. I'd like to understand what is going on here. I've read that I cannot do IP stuff when there is a proxy involved (something between Node and the User)...but 98% of the time, the connections to workers are fine and everything works as expected. Any help here is really appreciated.
Related
I count the number of online websockets via onConnection and onDisconnect events:
const socketIo = require('socket.io');
var on_connect = 0;
var on_disconnect = 0;
var port = 6001;
var io = socketIo(port, {
pingTimeout: 5000,
pingInterval: 10000
});
//I have only one NameSpace and root NS is not used
var ns1 = io.of('ns1');
ns1
.on('connection', function (socket) {
on_connect += 1;
socket.on('disconnect', function (reason) {
on_disconnect += 1;
});
});
...
var online = on_connect - on_disconnect;
...
But online value not equal io.engine.clientsCount value.
And over time the difference between online value and io.engine.clientsCount value is growing up.
Why this is happens?
What is needed to make to fix this?
on_connect and on_disconnect variables are updates in the callback events, whereas the online variable is not recalculated. So you will need to recalculate the online variable every time the other variable change.
It might be easier to use only one variable to count connections. Increment it on connection, and decrement it on disconnect. That's how I keep track of the number of connections. Then there isn't a need to calculate it every time its value is needed.
Also, the line that states var online = on_connect - on_disconnect; Is occurring before either is modified... That's what #gvmani is trying to tell you.
Here's an example of some of what I'm doing. The code below sets up to listen for connections & disconnections and maintains a count of current connections. I should note that I'm not using a namespace like the OP, but the counting portion is what's of importance. I'll also note that I use connCount > 0 in the send() function. Which in my application is used to broadcast to all connected clients.
/* ******************************************************************** */
// initialize the server
const http = require('http');
const server = http.createServer();
// Socket.io listens to our server
const io = require('socket.io').listen(server);
// Count connections as they occur, decrement when a client disconnects.
// If the counter is zero then we won't send anything over the socket.
var connCount = 0;
// A client has connected,
io.on('connection', function(socket) {
// Increment the connection counter
connCount += 1;
// log the new connection for debugging purposes.
console.log(`on connect - ${socket.id} ${connCount}`);
// The client that initiated the connection has disconnected.
socket.on('disconnect', function () {
connCount -= 1;
console.log(`on disconnect - ${socket.id} ${connCount}`);
});
});
// Start listening...
server.listen(3000);
// Send something to all connected clients (a broadcast) the
// 'channel' will indicate the destination within the client
// and 'data' becomes the payload.
function send(channel, data) {
console.log(`send() - channel = ${channel} payload = ${JSON.stringify(data)}`);
if(connCount > 0) io.emit(channel, {payload: data});
else console.log('send() - no connections');
};
I have a use case where I need to dedicate one worker to listen for incoming messages from an AWS sqs queue and process them, the rest of the workers are listening on the desired port for connection. Following is the code
var cluster = require('cluster');
if (cluster.isMaster) {
/** Master process */
/**
* spawn a worker for each available cpu
* HACK: Downsizing worker count will not restore tasks that were handled by those extra workers
*/
var cpuCount = require('os').cpus().length;
for (var i = 0; i < cpuCount; i += 1) {
cluster.fork();
}
/** re-spawn worker if dead */
cluster.on('exit', function(worker) {
console.log('worker-' + worker.id + ' died');
cluster.fork();
});
} else if (cluster.worker.id == 1) {
// Check for incoming messages from an AWS SQS queue
} else {
//Listen on the port for connections
}
If the worker listening for connection on the port dies, it is revived. But, I cannot figure out the solution of how to revive the worker which is listening for incoming messages from the SQS queue if it dies due to some error.
I'm trying to make a small WiFi control board with the use of an ESP8266 ESP-201 board.
I used the example provided for WebSocket for Arduino, with a little bit of modification to be able to handle JSON messages. This is the code I got to:
/*
* WebSocketClient.ino
*
* Created on: 24.05.2015
*
*/
#include <Arduino.h>
#include <ESP8266WiFi.h>
#include <ESP8266WiFiMulti.h>
#include <WebSocketsClient.h>
#include <ArduinoJson.h>
#include <Hash.h>
ESP8266WiFiMulti WiFiMulti;
WebSocketsClient webSocket;
#define USE_SERIAL Serial
ArduinoJson::StaticJsonBuffer<200> jsonBuffer;
void webSocketEvent(WStype_t type, uint8_t * payload, size_t lenght) {
switch(type) {
case WStype_DISCONNECTED:
USE_SERIAL.printf("[WSc] Disconnected!\n");
break;
case WStype_CONNECTED: {
USE_SERIAL.printf("[WSc] Connected to url: %s\n", payload);
// send message to server when Connected
}
break;
case WStype_TEXT: {
USE_SERIAL.printf("[WSc] get text: %s\n", payload);
String text = String((char *) &payload[0]);
USE_SERIAL.println(text);
JsonObject& root = jsonBuffer.parseObject(text);
USE_SERIAL.printf("[SETUP] BOOT WAIT %d...\n", root["r"]);
USE_SERIAL.printf("[SETUP] BOOT WAIT %d...\n", root["g"]);
USE_SERIAL.printf("[SETUP] BOOT WAIT %d...\n", root["b"]);
}
// send message to server
break;
case WStype_BIN:
USE_SERIAL.printf("[WSc] get binary lenght: %u\n", lenght);
hexdump(payload, lenght);
// send data to server
break;
}
}
void setup() {
USE_SERIAL.begin(115200);
USE_SERIAL.setDebugOutput(true);
USE_SERIAL.println();
USE_SERIAL.println();
USE_SERIAL.println();
for(uint8_t t = 4; t > 0; t--) {
USE_SERIAL.printf("[SETUP] BOOT WAIT %d...\n", t);
USE_SERIAL.flush();
delay(1000);
}
WiFiMulti.addAP("GamersHeavenLow", "nCore4Life");
while(WiFiMulti.run() != WL_CONNECTED) {
delay(100);
}
webSocket.begin("192.168.0.104", 3000);
webSocket.onEvent(webSocketEvent);
}
void loop() {
webSocket.loop();
}
And this is the server side I used to:
var express = require('express');
var app = express();
var url = require('url');
var http = require('http').Server(app);
var WebSocketServer = require('ws').Server;
var wss = new WebSocketServer({ server: http });
app.get('/', function(req, res){
res.sendFile(__dirname + '/index.html');
});
app.use(express.static('public'));
wss.on('error', function(error) {
console.log(error);
});
wss.on('connection', function connection(ws) {
var location = url.parse(ws.upgradeReq.url, true);
// you might use location.query.access_token to authenticate or share sessions
// or ws.upgradeReq.headers.cookie (see http://stackoverflow.com/a/16395220/151312)
console.log('connected');
ws.on('message', function incoming(message) {
console.log(message);
wss.clients.forEach(function each(client) {
client.send(JSON.stringify(message));
});
});
//ws.send('something');
});
http.listen(3000, function(){
console.log('listening on *:3000');
});
Before I started to work with the JSON encoding it was working for a while, but now it's not. I get the following error:
[SETUP] BOOT WAIT 4...
scandone
state: 0 -> 2 (b0)
state: 2 -> 3 (0)
state: 3 -> 5 (10)
add 0
aid 2
cnt
connected with GamersHeavenLow, channel 6
dhcp client start...
ip:192.168.0.101,mask:255.255.255.0,gw:192.168.0.1
[SETUP] BOOT WAIT 3...
[SETUP] BOOT WAIT 2...
[SETUP] BOOT WAIT 1...
[WS-Client] connect ws...
[WS-Client] connection to 192.168.0.104:3000 Faild
[WS-Client] client disconnected.
[WSc] Disconnected!
[WS-Client] connect ws...
[WS-Client] connection to 192.168.0.104:3000 Faild
[WS-Client] client disconnected.
[WSc] Disconnected!
I believed it must have been the JSON decoding, so I reverted back to the default example, but I still get the same message, connection failed.
So I tried to use the echo websocket server, with which the Arduino code started working.
So I figured out it must be a server side issue, so I tested my node.js websocket server with a pure WebSocket client, but that is also working without a problem.
So basically I have two separate sets of code, which are running in isolation from one another without a problem, but they don't want to play along together.
Any idea what might be causing this?
WebSocket server used: https://github.com/websockets/ws
WebSocket client used on ardruino: https://github.com/Links2004/arduinoWebSockets
You need to use the WS module instead of the socket.io.
Ok, I have an express-powered API where I also have socket.io running to receive/send realtime events...all works just dandy. I need to cluster my app. I set everything up based on the below code. I spin up workers, they get connections and everything works, except the fact that now I can't "blast" to all socket.io connections. Here is the setup (taken from this):
var express = require('express'),
cluster = require('cluster'),
net = require('net'),
sio = require('socket.io'),
sio_redis = require('socket.io-redis');
var port = 3000,
num_processes = require('os').cpus().length;
if (cluster.isMaster) {
// This stores our workers. We need to keep them to be able to reference
// them based on source IP address. It's also useful for auto-restart,
// for example.
var workers = [];
// Helper function for spawning worker at index 'i'.
var spawn = function(i) {
workers[i] = cluster.fork();
// Optional: Restart worker on exit
workers[i].on('exit', function(worker, code, signal) {
console.log('respawning worker', i);
spawn(i);
});
};
// Spawn workers.
for (var i = 0; i < num_processes; i++) {
spawn(i);
}
// Helper function for getting a worker index based on IP address.
// This is a hot path so it should be really fast. The way it works
// is by converting the IP address to a number by removing the dots,
// then compressing it to the number of slots we have.
//
// Compared against "real" hashing (from the sticky-session code) and
// "real" IP number conversion, this function is on par in terms of
// worker index distribution only much faster.
var workerIndex = function (ip, len) {
var _ip = ip.split(/['.'|':']/),
arr = [];
for (el in _ip) {
if (_ip[el] == '') {
arr.push(0);
}
else {
arr.push(parseInt(_ip[el], 16));
}
}
return Number(arr.join('')) % len;
}
// Create the outside facing server listening on our port.
var server = net.createServer({ pauseOnConnect: true }, function(connection) {
// We received a connection and need to pass it to the appropriate
// worker. Get the worker for this connection's source IP and pass
// it the connection.
var worker = workers[worker_index(connection.remoteAddress, num_processes)];
worker.send('sticky-session:connection', connection);
}).listen(port);
} else {
// Note we don't use a port here because the master listens on it for us.
var app = new express();
// Here you might use middleware, attach routes, etc.
// Don't expose our internal server to the outside.
var server = app.listen(0, 'localhost'),
io = sio(server);
// Tell Socket.IO to use the redis adapter. By default, the redis
// server is assumed to be on localhost:6379. You don't have to
// specify them explicitly unless you want to change them.
io.adapter(sio_redis({ host: 'localhost', port: 6379 }));
// Here you might use Socket.IO middleware for authorization etc.
// Listen to messages sent from the master. Ignore everything else.
process.on('message', function(message, connection) {
if (message !== 'sticky-session:connection') {
return;
}
// Emulate a connection event on the server by emitting the
// event with the connection the master sent us.
server.emit('connection', connection);
connection.resume();
});
}
So I connect from various machines to test concurrency, workers do their thing and all is good, but when I get an IO connection, I'm logging the TOTAL "connected" count and it's always 1 per instance. I need a way to say
allClusterForks.emit(stuff)
I get the connection on the correct worker pid, but "ALL CONNECTIONS" always returns 1.
io.on('connection', function(socket) {
console.log('Connected to worker %s', process.pid);
console.log("Adapter ROOMS %s ", io.sockets.adapter.rooms);
console.log("Adapter SIDS %s ", io.sockets.adapter.sids);
console.log("SOCKETS CONNECTED %s ", Object.keys(io.sockets.connected).length);
});
I can see the subscribe/unsubscribe coming in using Redis MONITOR
1454701383.188231 [0 127.0.0.1:63150] "subscribe" "socket.io#/#gXJscUUuVQGzsYJfAAAA#"
1454701419.130100 [0 127.0.0.1:63167] "subscribe" "socket.io#/#geYSvYSd5zASi7egAAAA#"
1454701433.842727 [0 127.0.0.1:63167] "unsubscribe" "socket.io#/#geYSvYSd5zASi7egAAAA#"
1454701444.630427 [0 127.0.0.1:63150] "unsubscribe" "socket.io#/#gXJscUUuVQGzsYJfAAAA#"
These are connections from 2 different machines, I would expect by using the socket io redis adapter that these subscriptions would be coming in on the same redis connection, but they are different.
Am I just totally missing something? There's a surprising lack of documentation/articles out there for this that aren't either completely outdated/wrong/ambiguous.
EDIT:
Node v5.3.0
Redis v3.0.6
Socket.io v1.3.7
So if anyone comes across this, I figured out that actually "looking" at the counts of connected sockets across processes is not a thing, but broadcasting or emitting to them is. So I've basically just been "testing" for no reason. All works as expected. I WILL be rewriting the socket.io-redis adapter to allow checking counts across processes.
There was a pull request a few years ago to implement support for what I was trying to do. https://github.com/socketio/socket.io-redis/pull/15 and I might try cleaning that up and re-submitting.
I am investigating options for running node in a multi-core environment.
I'm trying to determine the best approach and so far I've seen these options
Use built in cluster library to spin up works and respond to signals
Use PM but, PM2 -i is listed as beta.
Naught
Recluster
Are there other alternatives? What are folks using in production?
I've been using the default cluster library, and it works very well. I've had over 10,000 concurrents(multiple clusters on multiple servers) and it works very well.
It is suggested to use clusters with domain for error handling.
This is lifted straight from http://nodejs.org/api/domain.html I've mades some changes on how it spawns new clusters for each core of your machine. and got rid of if/else and added express.
var cluster = require('cluster'),
http = require('http'),
PORT = process.env.PORT || 1337,
os = require('os'),
server;
function forkClusters () {
var cpuCount = os.cpus().length;
// Create a worker for each CPU
for (var i = 0; i < cpuCount ; i += 1) {
cluster.fork();
}
}
// Master Process
if (cluster.isMaster) {
// You can also of course get a bit fancier about logging, and
// implement whatever custom logic you need to prevent DoS
// attacks and other bad behavior.
//
// See the options in the cluster documentation.
//
// The important thing is that the master does very little,
// increasing our resilience to unexpected errors.
forkClusters ()
cluster.on('disconnect', function(worker) {
console.error('disconnect!');
cluster.fork();
});
}
function handleError (d) {
d.on('error', function(er) {
console.error('error', er.stack);
// Note: we're in dangerous territory!
// By definition, something unexpected occurred,
// which we probably didn't want.
// Anything can happen now!Be very careful!
try {
// make sure we close down within 30 seconds
var killtimer = setTimeout(function() {
process.exit(1);
}, 30000);
// But don't keep the process open just for that!
killtimer.unref();
// stop taking new requests.
server.close();
// Let the master know we're dead.This will trigger a
// 'disconnect' in the cluster master, and then it will fork
// a new worker.
cluster.worker.disconnect();
} catch (er2) {
// oh well, not much we can do at this point.
console.error('Error sending 500!', er2.stack);
}
});
}
// child Process
if (cluster.isWorker) {
// the worker
//
// This is where we put our bugs!
var domain = require('domain');
var express = require('express');
var app = express();
app.set('port', PORT);
// See the cluster documentation for more details about using
// worker processes to serve requests.How it works, caveats, etc.
var d = domain.create();
handleError(d);
// Now run the handler function in the domain.
//
// put all code here. any code included outside of domain.run will not handle errors on the domain level, but will crash the app.
//
d.run(function() {
// this is where we start our server
server = http.createServer(app).listen(app.get('port'), function () {
console.log('Cluster %s listening on port %s', cluster.worker.id, app.get('port'));
});
});
}
We use Supervisor to manage our Node.JS process's, to start them upon boot, and to act as a watchdog in case the process's crash.
We use Nginx as a reverse-proxy to load balance traffic between the process's that listen to different ports
this way each process is isolated from the other.
for example: Nginx listens on port 80 and forwards traffic to ports 8000-8003
I was using PM2 for quite a while, but their pricing is expensive for my needs as I'm having my own analytics environment and I don't require support, so I decided to experiment alternatives. For my case, just forever made the trick, very simple one actually:
forever -m 5 app.js
Another useful example is
forever start app.js -p 8080