Can't receive redis data from socket io - node.js

I'm building a realtime visualization using redis as pubsub messenger between python and node. There's a python script always running which sets a redis hash with hmset. That side of the app is working fine, if I enter the following example command: "HGETALL 'sellers-80183917'" in a redis client I end up getting the proper data.
The problem is in the js side. I'm using socketio and redis nodejs libraries to listen to the redis instance and publish the results online through a d3js viz.
I run the following code with node:
var express = require('express');
var app = express();
var redis = require('redis');
app.use(express.static(__dirname + '/public'));
var http = require('http').Server(app);
var io = require('socket.io')(http);
var sredis = require('socket.io-redis');
io.adapter(sredis({ host: 'localhost', port: 6379 }));
redisSubscriber = redis.createClient(6379, 'localhost', {});
redisSubscriber.on('message', function(channel, message) {
io.emit(channel, message);
});
app.get('/sellers/:seller_id', function(req, res){
var seller_id = req.params.seller_id;
redisSubscriber.subscribe('sellers-'.concat(seller_id));
res.render( 'seller.ejs', { seller:seller_id } );
});
http.listen(3000, '127.0.0.1', function(){
console.log('listening on *:3000');
});
And this is the relevant part of the seller.ejs file that's receiving the user requests and outputting the viz:
var socket = io('http://localhost:3000');
var stats;
var seller_key = 'sellers-'.concat(<%= seller %>);
socket.on(seller_key, function(msg){
stats = [];
console.log('Im in');
var seller = $.parseJSON(msg);
var items = seller['items'];
for(item in items) {
var item_data = items[item];
stats.push({'title': item_data['title'], 'today_visits': item_data['today_visits'], 'sold_today': item_data['sold_today'], 'conversion_rate': item_data['conversion_rate']});
}
setupData(stats);
});
The problem is that the socket_on() method never receives anything and I don't see where the problem is as everything seems to be working fine besides this.

I think that you might be confused as to what Pub/Sub in Redis actually is. It's not a way to listen to changes on hashes; you can have a Pub/Sub channel called sellers-1, and you can have a hash with the key sellers-1, but those are unrelated to each other.
As documented here:
Pub/Sub has no relation to the key space.
There is a thing called keyspace notifications that can be used to listen to changes in the key space (through Pub/Sub channels); however, this feature isn't enabled by default because it'll take up more resources.
Perhaps an easier method would be to publish a message after the HMSET, so any subscribers would know that the hash got changed (they would then retrieve the hash contents themselves, or the published message would contain the relevant data).
This brings us to the next possible issue: you only have one subscriber connection, redisSubscriber.
From what I understand from the Node.js Redis driver, calling .subscribe() on such a connection would remove any previous subscriptions in favor of the new one. So if you were previously subscribed to the sellers-1 channel and subscribe to sellers-2, you wouldn't be receiving messages from the sellers-1 channel anymore.
You can listen on multiple channels by either passing an array of channels, or by passing them as a arguments:
redisSubscriber.subscribe([ 'sellers-1', 'sellers-2', ... ])
// Or:
redisSubscriber.subscribe('sellers-1', 'sellers-2', ... )
You would obviously have to track each "active" seller subscription. Either that, or create a new connection for each subscription, which also isn't ideal.
It's probably a better idea to have a single Pub/Sub channel on which all changes would get published, instead of a separate channel for each seller.
Finally: if your seller id's aren't hard to guess (for instance, if it's based on an incremental integer value), it would be trivial for someone to write a client that would make it possible to listen in on any seller channel they'd like. It might not be a problem, but it is something to be aware of.

Related

Socket IO Server Clusters working with Redis Pub/Sub

So firstly, I have built a microservice that fetches Football API, and thru pub/sub system of redis, it publishes any changes if there are any for livescores.
Now my server, with sockets and routes, will be in cluster mode. I already set this up with socketio-redis. Here is a snippet of this set up:
const io = require('socket.io')();
const sRedis = require('socket.io-redis');
const adapter = sRedis({ host: 'localhost', port: 6379 });
const { promisify } = require('util');
const Redis = require('ioredis');
const redis = new Redis();
redis.subscribe('livescore');
io.adapter(adapter);
const ioa = io.of('/').adapter;
ioa.clients = promisify(ioa.clients);
ioa.clientRooms = promisify(ioa.clientRooms);
ioa.remoteJoin = promisify(ioa.remoteJoin);
ioa.remoteLeave = promisify(ioa.remoteLeave);
ioa.allRooms = promisify(ioa.allRooms);
// notice this listener
redis.on('message', (channel, message) => {
io.emit('livescore', message);
})
io.on('connect', async (socket) => {
socket.clientRooms = () => ioa.clientRooms(socket.id);
socket.remoteJoin = (room) => ioa.remoteJoin(socket.id, room);
socket.remoteLeave = (room) => ioa.remoteLeave(socket.id, room);
socket.remoteDisconnect = () => ioa.remoteDisconnect(socket.id);
socket.on('join room', async (id) => {
await socket.remoteJoin(id);
socket.emit('join room', `You have joined room ${id}`)
socket.broadcast.emit('join room', `${socket.id} has joined.`)
});
socket.on('leave room', (id) => {
socket.remoteLeave(id);
});
})
module.exports = io;
So, if I run single instance of this node app, everything works perfectly.
But if I run it in cluster mode, let's say there are 4 clusters (I'm running cluster mode with pm2), the following happens:
Microservice publishes event.
Each cluster has a subscription on 'livescore' channel
Each cluster does io.emit() (to all clients)
Client get 4 same events at almost same time.
I figured out why the client gets 4 same events, but I wanna know what is the right way of handling this?
My only thought on solution is that I only do redis sub on one cluster, and publish everything from that one, but I fear that would be too much job for one cluster?
Any ideas?
There are probably multiple solutions to fix it, you could for example:
Use a message queue instead of pub/sub
Depending on the number of processing, you probably only want one node it process the message. A pub/sub is not what you want in that case. You could for example store your messages in a list and use the LPOP command to get and delete a message. Then you could say the "first one catches it" - this way only one of your servers will do the work, but a random one basically.
You could also use a distinct message queue like RabbitMQ, SQS, etc.
Use socket.io-emitter to send messages
Since you're using socket.io-redis anyway, your messages get distributed to your nodes. There's a project which is part of socket.io-redis, it's called socket.io-emitter. That can be used to send messages to all your nodes without being one itself. When you implement that in your worker microservice (the one that writes the message to "livescore" at the moment), you can send messages directly to your clients.
That might not work if you need to process the messages in your node app though.

Websockets & NodeJS - Changing Browser Tabs & Sessions

I've started writing a node.js websocket solution using socket.io.
The browsers connects to the node server successfully and I get see the socket.id and all config associated with console.log(socket). I also pass a userid back with the initial connection and can see this on the server side to.
Question: I'm not sure the best way to associate a user with a connection. I can see the socket.id changes every page change and when a tab is opened up. How can I track a user and send 'a message' to all required sockets. (Could be one page or could be 3 tabs etc).
I tried to have a look at 'express-socket.io-session' but I'm unsure how to code for it and this situation.
Question: I have 'io' and 'app' variables below. Is it possible to use the 2 together? app.use(io);
Essentially I want to be able to track users (I guess by session - but unsure of how to handle different socket id's for tabs etc) and know how to reply to user or one or more sockets.
thankyou
The best way to handle the situation is rely on SocketIO's rooms. Name the room after the user's unique ID. This will support multiple connections out of the box. Then, whenever you need to communicate with a particular user, simply call the message function and pass in their id, the event, and any relevant data. You don't need to worry about explicitly leaving a room, SocketIO does that for you whenever their session times out or they close their browser tab. (We do explicitly leave a room whenever they log out though obviously)
On the server:
var express = require('express');
var socketio = require('socket.io');
var app = express();
var server = http.createServer(app);
var io = socketio(server);
io.on('connect', function (socket) {
socket.on('userConnected', socket.join); // Client sends userId
socket.on('userDisconnected', socket.leave); // Cliend sends userId
});
// Export this function to be used throughout the server
function message (userId, event, data) {
io.sockets.to(userId).emit(event, data);
}
On the client:
var socket = io('http://localhost:9000'); // Server endpoint
socket.on('connect', connectUser);
socket.on('message', function (data) {
console.log(data);
});
// Call whenever a user logs in or is already authenticated
function connectUser () {
var userId = ... // Retrieve userId somehow
if (!userId) return;
socket.emit('userConnected', userId);
}
// Call whenever a user disconnects
function disconnectUser () {
var userId = ... // Retrieve userId somehow
if (!userId) return;
socket.emit('userDisconnected', userId);
}

Error: Redis connection to 127.0.0.1:6379 failed - connect EMFILE

So I created, an realtime application using socket.io, redis and node.js.
The problem is that with 30 users, I'm already reaching the number of connections of the server ( I'm running Ubuntu 14.04.
And I think it has something to do about the way I connect to redis.
So on one page, I have at most 12 channels to subscribe to. On two socket.io connections, one has 6 channels and the other has the other 6 channels.
Before showing my node js code, what I do, is for each channel, I create a new redis client, so let's say that I have an insert and update channel, on the node.js code, so my code wiil be:
var data = io.of('/data');
data.on('connection', function(client) {
var insert = redis.createClient();
var update = redis.createClient();
insert.subscribe('insert');
insert.on("message", function(channel, message) {
client.emit('data_insert', message);
});
update.subscribe('update');
update.on("message", function(channel, message) {
client.emit('data_update', message);
});
});
I believe that this is the problem, and that why, with 30 users I'm exceeding the limit of connections of the server, since 30 * 12 + 2 * 30 = 420, plus a few others for everything else, and it's easy to reach 1024.
So how can I optimize the code, to reduce the number of connections per client to one or two the max?
Does it help, to have the var insert = redis.createClient(); outside the data.on('connection')??
You should not be creating one (or more) redis connections per client - just keep using the same global one. Further, there's really no reason to create a client for each channel you want to subscribe to. Just use the channel argument to figure out which thing to do with the data (in this case, you're doing the same thing on each channel, so even less point in having separate ones).
var data = io.of('/data');
var redis = redis.createClient();
redis.subscribe('insert');
redis.subscribe('update');
data.on('connection', function(client) {
redis.on("message", function(channel, message) {
if(channel == 'insert')
client.emit('data_insert', message);
else if(channel == 'update')
client.emit('data_update', message);
});
});
There are probably better ways to do this, but this is a fairly direct translation of your code into something that uses only one redis connection.
Note that redis subcribers can only subscribe - you will need a second client to publish or do any other redis operations.
just for your information (I lost this evening with that)
This does not work (it defaults connecting to localhost:6379 whatever you put in the url):
const redisURL = "redis://foo.bar.org:6379"
redis.createClient( redisURL )
but this DOES WORK (it tries to connect too foo.bar.org)
redis.createClient( {url: redisURL} )
Use socket.io's namespace. Within each namespace you can also define rooms but namespace should be enough for your needs.

Socket.io with multiple Node.js hosts, emit to all clients

I am new to Socket.io and trying to get my head around the best approach to solve this issue.
We have four instances of a Node.js app running behind a load balancer.
What I am trying to achieve is for another app to POST some data to the load balancer URL which will hand if off to one of the instances.
The receiving instance will store the data, then use Socket.io to emit the data to the connected clients.
The issue is that browser/client can only be connected to a single instance at one time.
I am trying to determine if there is a way to emit to all clients at once?
Or have the clients connect to multiple servers using io.connect?
Or is this a case for Redis?
Publish/Subscribe is what you need here. Redis will give you the functionality your looking for out of the box. You just need to create a redis client and subscribe to an update channel on each of your app server nodes. Then, publish the update when a POST is successful (or whatever). Finally, have the redis client subscribe to the update chanel and on message emit a socketio event:
(truncated for brevity)
var express = require('express')
, socketio = require('socket.io')
, redis = require('redis')
, rc = redis.createClient()
;
var app = express();
var server = http.createServer(app);
var io = socketio.listen(server);
server.listen(3000);
app.post('/targets', function(req, res){
rc.publish('update', res.body);
});
rc.on('connect', function(){
// subscribe to the update channel
rc.subscribe('update');
});
rc.on('message', function(channel, msg){
// util.log('Channel: ' + channel + ' msg: ' + msg);
var msg = JSON.parse(msg);
io.sockets.in('update').emit('message', {
channel: channel,
msg: msg
});
});
Then in the JS app, listen for that emitted message:
socket.on('message', function(data){
debugger;
// do something with the updated data
});
Of course, introducing this new Redis Server adds another single point of failure. A more robust implementation may use something like a message broker with AMQP or ZeroMQ or some similar networking library which provides pub/sub capabilities.

Using (socket.io + RedisStore) to communicate across multiple servers

I am working on a multiplayer online game, using Node.js and Socket.io. I expect a lot of players to join the game, so I am hosting it on Amazon Opworks.
The problem is that the servers aren't able to send socket events to clients connected to a different server. I am using RedisStore to manage socket.io sessions. I believed RedisStore and socket.io took care of this inter-server communication under the hood in a seamless manner. Here is a reference to another question: How does socket.io send messages across multiple servers?
But that's not the case it seems. Messages do not go through to other clients if they are on different servers; the app works if there is only one server, but fails if I use multiple servers loadbalanced using ELB on Opsworks.
This is just an extract from the whole code. Please ignore syntax errors etc if any.
app.js
//Redis Client Initialization
var redis = require("redis");
redis_client = require('redis-url').connect('redis://xxxxx');
//setting RedisStore for socket.io
var RedisStore = require('socket.io/lib/stores/redis')
, redis = require('socket.io/node_modules/redis')
, pub = redis.createClient(11042,'-----')
, sub = redis.createClient(11042,'-----')
, client = redis.createClient(11042,'-----');
// using RedisStore in combo with Socket.io to allow communications across multiple servers
io.set('store', new RedisStore({
redis : redis,
redisPub : pub,
redisSub : sub,
redisClient : client
}));
//socket communication specific code
io.of('/game')
.on('connection', function (socket) {
socket.on('init' , function(data){
var user_id = data.user_id; // collecting user_id that was sent by the client
var socket_id = socket.id;
redis_client.set("user_socket:"+user_id, socket_id, function(err, reply){
//stored a referece to the socket id for that user in the redis database
});
});
socket.on('send_message', function(data){
var sender = data.sender_id;
var reciepient = data.reciepient_id ; // id of the user to whom message is to be sent to
redis_client.get("user_socket:"+reciepient, function(err,socket_id){
if(socket_id){
var socket = io.of('/game').sockets[socket_id];
socket.emit("message", {sender : sender}); // This fails. Messages to others servers dont go through.
}
})
})
});
You can't broadcast directly to socket objects on other servers. What Redis does is allows you to broadcast to 'rooms' on other servers. Thankfully with socket.io 1.x, new connections automatically join a room with the name of their socket id. To solve your problem, change:
if(socket_id){
var socket = io.of('/game').sockets[socket_id];
socket.emit("message", {sender : sender}); // This fails. Messages to others servers dont go through.
}
to emit to the room instead of calling emit on a socket object:
if(socket_id){
io.to(socket_id).emit("message", {sender : sender}); // This fails. Messages to others servers dont go through.
}
And you might have more luck.

Resources