Socket.io server ping timeout on some connections - node.js

I'm having issues with using the socket.io JS client and the server where websocket connections ping timeout at random times. I've searched high and low through SO and the rest of the interwebs.
Some context on the project - this is an Electron/ReactJS application that has two separate windows, where the socket.io client connects on application load, provided the user is logged in. If a user logs in after application load, the same function to connect to the socket is called:
const protocol = Env.isDev() ? 'http' : 'https'
const socketServer = `${protocol}://${LocalConfig.servers.websocket}:80`
const socketReceiverId = createReceiverId()
/**
* Initiates the websocket connection
*/
export function init() {
// Get authourizer ID
const authorizerId = getAuthId()
// only allow in the renderer and we have an authorizer id
// and if the socket connection is null. Socket will not be
// null if this method is called after HMR
if (Env.isRenderer() && authorizerId && !getCurrentSocket()) {
const socket = sockio(`${socketServer}/notification`, {
timeout: 5000,
reconnection: false,
transportOptions: {
polling: {
extraHeaders: {
'origin': `gui://${process.env.PLATFORM}`,
'x-receiver-token': socketReceiverId,
'x-authorization': authorizerId
}
}
}
})
socket.on('s:connected', onConnected) // called when connection made (custom event with extra data
.on('n:push', onNotification) // called when a notification is received
.on('s:close', onServerClose) // called when server shuts down
.on('disconnect', onDisconnect) // called on disconnect
.on('error', console.warn)
}
}
I had to leave reconnection to false because I need to manage reconnections manually, because if a user logs out, we won't be reconnecting, but will reconnect after successful login
I'm also using polling so I can send extra headers, including origin, authorization and the receiver-token, which is a random UUID that's stored in LocalStorage to uniquely identify the application instance, and will be the room that we join, so notifications get sent to the proper device, whether it's this app or mobile (which will all use the same websocket interface)
When the application loads initially and the websocket connection is attempted, 90% of the time, both windows connect successfully and the push notifications come through - the other 10% of the time, only one window can connect, while the other won't (even though through logging on the client, I can see that the onConnected callback is called). Eventually, both will connect successfully, but ideally, they should connect instantly at the first attempt.
What really screws things up is when the server reboots (which will be something that will happen from time to time, as new code gets pushed to the server and the service is restarted). When the server disconnects however, a message is emitted to all clients currently connected where the custom reconnect circuit is invoked.
This is where the main issues start - whether it's manual reconnection, or automatic, the polling keeps failing with a ping timeout error, even though the onConnected callback is called.
Eventually, the connections all settle and connect - sometimes after a minute or two, sometimes as much as 10 minutes to a half hour. It's all exceedingly random.
Note that everything right now is on my local machine running Ubuntu 18.04 LTS, so there should be near zero latency.
I pasted the verbose debug output here:
https://pastebin.com/TR6qPe0R
Here is the main server code:
import process from 'process'
import http from 'http'
import sockio from 'socket.io'
import { BindAll } from 'lodash-decorators'
import { Handlers } from './Handlers'
import { Receiver } from './Receiver'
import { RedisPool } from './RedisPool'
import { Log } from './Logger'
interface NamespaceGroup {
ns: SocketIO.Namespace
receiver: Receiver
}
#BindAll()
class Server {
private _port: number
private _socketServer: sockio.Server
private _pollingServer: http.Server
private _namespaces: NamespaceGroup[] = []
constructor() {
this._port = parseInt(process.env['PORT']) || 18000
this._pollingServer = http.createServer(Handlers.handleHttpRequest)
}
startServer() {
this._pollingServer.listen(this._port, this.onServerStart)
process.on('SIGINT', this.close)
process.on('SIGUSR2', this.close)
}
private async onServerStart() {
Log.info(`Server started on port ${this._port}`)
this._socketServer = sockio(this._pollingServer)
this._socketServer.origins(Handlers.handleOrigin)
this._socketServer.use(Handlers.handleAuthorization)
const nsNotify = this._socketServer.of('/notification')
.on('connection', Handlers.handleNotificationConnection)
this._namespaces.push({
ns: nsNotify,
receiver: new Receiver(nsNotify)
})
}
private close() {
Log.info('Closing server. Killing process')
for (const nsGroup of this._namespaces) {
nsGroup.ns.emit('s:close')
}
this._socketServer.close()
RedisPool.closeAll()
process.exit(0)
}
}
const server = new Server()
export default server
And the handlers:
import { IncomingMessage, ServerResponse } from 'http'
import { Socket } from 'socket.io'
import { Queries } from './Queries'
import { Log } from './Logger'
import { OutgoingMessage } from './Receiver'
export namespace Handlers {
const allowedOrigins = [
'gui://win32',
'gui://darwin',
'gui://linux',
'ast://android',
'ast://ios',
'mob://android',
'mob://ios',
]
export function handleOrigin(origin: string, callback: (error: string | null, success: boolean) => void) {
if (allowedOrigins.indexOf(origin) == -1) {
callback('Invalid Origin: ' + origin, false)
} else {
callback(null, true)
}
}
export async function handleAuthorization(socket: Socket, next: (err?: any) => void) {
const auth = socket.handshake.headers['x-authorization']
const receiverToken = socket.handshake.headers['x-receiver-token']
if (!auth) {
next(new Error('Authorization not provided'))
socket.disconnect()
} else {
try {
const credentials = await Queries.findClientCredentials(auth)
if (!!credentials) {
await Queries.saveCurrentToken(auth, receiverToken)
next()
} else {
Log.warn('Client rejected [auth:%s] (Invalid Authorization)', auth)
next(new Error('Invalid Authorization: ' + auth))
socket.disconnect()
}
} catch (err) {
next(err)
Log.error('Client auth error', err)
socket.disconnect()
}
}
}
export async function handleNotificationConnection(socket: Socket) {
const auth = socket.handshake.headers['x-authorization']
const receiverToken = socket.handshake.headers['x-receiver-token']
const namespace = socket.nsp
const ns = namespace.name
// Emit connection message
socket.emit('s:connected', ns, socket.id) // lets the client know connection is OK
// Join room specific to an application instance
socket.join(receiverToken)
Log.debug('Connect: Client connected to %s [auth:%s] - %s (%d active sockets)',
ns, auth, socket.id, socket.adapter.rooms[receiverToken].length)
// Listens to the c:close (client request close) event
socket.on('c:close', async reason => {
socket.leave(receiverToken)
socket.disconnect(true)
Log.debug('Disconnect: (%s) %s : %s', reason, receiverToken, socket.id)
})
}
// Default HTTP handler for the main Node HTTP server
export function handleHttpRequest(request: IncomingMessage, response: ServerResponse) {
response.statusCode = 200
response.setHeader('Content-Type', 'text/plain')
response.end(`${request.method} ${request.url}\n`)
}
}
The rest of the code on the server side works as it should, such as RedisPool and the Receiver class instances. The issue is with connectivity, so I only included the Server class and Handlers namespace.
And on the client, the relevant code (same namespace as the init function above):
// Flag to tell us if the disconnection
// was instigated via the client after a
// log out, so we know if we need to
// reconnect
let isManualDisconnect = false
/**
* Manually disconnect from the websocket server
* normally used when user logs out
*/
export function disconnect() {
const socket = getCurrentSocket()
isManualDisconnect = true // lets us know not to attempt reconnects
socket.emit('c:close', 'manual client disconnect', socket.id)
}
/**
* Handle a successful connection to the websocket server
* #param ns the namespace connected to (should always be /notification)
* #param socketId the socket id (SID)
*/
function onConnected(ns: string, socketId: string) {
console.log('Connection to %s Accepted. Receiver ID: %s', ns, socketId)
isManualDisconnect = false
}
/**
* Handles the disconnection from the socket server
* #param reason the reason for the disconnect
*/
function onDisconnect(reason: DisconnectReason) {
console.log('Disconnected (%s) Manual:', reason, isManualDisconnect)
if (!isManualDisconnect) {
if (getAuthId())
getCurrentSocket().connect()
}
}
/**
* Handles a socket being disconnected when the
* socket server shuts down
*/
function onServerClose() {
console.log('Socket server interrupt')
window.setTimeout(reconnect, 5000)
}
/**
* Attempts a reconnection every 5 seconds until
* a conneciton is re-established
*/
function reconnect() {
const socket = getCurrentSocket()
console.log('Attempting reconnect')
if (socket.disconnected) {
console.log('Socket confirmed disconnected. Opening...')
socket.open()
} else {
window.setTimeout(reconnect, 5000)
}
}
/**
* Gets the current authorization id from local storage
*/
function getAuthId(): string {
return window.localStorage.getItem('auth')
}
function getCurrentSocket(): SocketIOClient.Socket {
const manager = sockio.managers[socketServer]
if (manager)
return manager.nsps['/notification']
}
/**
* Gets the current receiver id (application instance id)
*/
export function getReceiverId(): string {
return socketReceiverId
}
Finally, the NGINX config for the socket server:
server {
listen 80;
server_name [server name here];
location / {
proxy_pass "http://127.0.0.1:18000";
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header Host $host;
proxy_http_version 1.1;
}
}
I realize the code examples are verbose, but I've tinkered for over a day trying to find the issue, but with no solutions. Another issue is that nearly all examples for socket.io involve chat applications that don't cover authorization, origins, or how to properly disconnect, and in which order.
Socket.io client and server versions are the same (2.2.0)

Related

websocket state 3 while sending message after reconnect

I encountered odd problem when working with websocket. I am not sure exactly what is the reason for whats happening.
I have this simple websocket client that is implemented into nestjs service
import { Injectable } from '#nestjs/common';
import * as WebSocket from 'ws';
import { timer } from 'rxjs';
#Injectable()
export class ImportFeedsGateway {
private ws: WebSocket;
constructor() {
this.connect();
}
private connect() {
this.ws = new WebSocket(process.env.URL);
this.ws.on('open', () => {
this.ws.send(Math.random()); // this message is always send properly and this.ws state is 1 when logged after reconnect
});
this.ws.on('message', (message) => {
...
});
this.ws.on('error', (message) => {
this.ws.close();
});
this.ws.on('close', (message) => {
timer(1000).subscribe(() => {
this.connect();
});
});
}
sendMessage(message) {
this.ws.send(message); // this message is never send after reconnect and this.ws state is 3 when logged
}
}
So the problem is that after reconnect Math.random() message is always properly sent while attempt to use sendMessage method results in fail because this.ws state always is 3.
It seems like this.ws points to different places depending on if its accessed from outside of class (via sendMessage method) or from inside (via on('open', callback). Inside callback its always current this.ws while sendMessage accesses old this.ws in closed state. Its like this.ws reference is not wired properly.
Does anyone know why is that and how to fix it?
---- EDIT
after transforming sendMessage method to its arrow version, its working correctly, but its more of puzzle now why
sendMessage = (message) => {
this.ws.send(message); // with arrow version this.ws is always in state 1 and its working correctly
}

create new tedious connection every time for a new API call?

I am working on creating Express JS as my API server, using tedious to connect to my SQL Server DB.
Currently, in every request logic, I'll create a new tedious Connection object, connect to the DB, execute the query, then close the connection.
import { Connection } from 'tedious';
export class Controller {
all(_: Request, res: Response): void {
const connection = new Connection(getConfig()); // create a new connection everytime
connection.on('connect', (err) => {
if (err) {
console.log('Connection Failed');
throw err;
}
getProducts(connection, _, res); // in there at the end, will call connection.close()
});
connection.connect();
}
import { Request, Response } from 'express';
import { Connection, Request as SqlReq } from 'tedious';
export default function getProducts(connection: Connection, _: Request, res: Response) {
const query = `SELECT * FROM Production.Product FOR JSON PATH;`;
let resultJson = ''; // prepare this result in return from SQL query
const sqlReq = new SqlReq(query, (err, _) => {
if (err) {
throw err;
}
// when request finished
connection.close();
res.json(JSON.parse(resultJson));
});
Is it a good or bad practice to create the connect, connect and close every time for a new API call? If there is a better way to handle the connection, may I have any reference or example?
just make sure that the connection is created only once by using this function. It will create the connection only on first call and return the previously created connection on subsequent calls.
var connection = null;
const getConnection = async () => {
if (connection) return connection;
connection = new Connection(getConfig());
return tmp;
};
Then you should leave the connection open by not calling close.
Better use connection pooling in mysql. During app startup, you can create a pool of threads used for db connecting purpose. It will be very fast, if you retrieve from the pool and establish the connection.
After your query execution/ manipulation, ensure to release the connection. So it will go to connection pool and available for further requests.
Ref : How do I create a MySQL connection pool while working with NodeJS and Express?
Ref : Release connection
node.js + mysql connection pooling

Whats the problem with the socketio connection?

Im having this alot of http petitions (6k INSIDE LAGGING) in 1-3 minutes in the console when i receive or send data to a socketio connection.
Im using node+express in the backend and vue on the front
Backend:
app.js
mongoose.connect('mongodb://localhost/app',{useNewUrlParser:true,useFindAndModify:false})
.then(result =>{
const server = app.listen(3000)
const io = require('./sockets/socket').init(server)
io.on('connection', socket =>{
// console.log('client connected')
})
if(result){console.log('express & mongo running');
}
})
.catch(error => console.log(error))
I created a io instance to use it on the routes
let io
module.exports = {
init: httpServer => {
io = require('socket.io')(httpServer)
return io;
},
getIo:()=>{
if(!io){
throw new Error('socket io not initialized')
}
return io;
}
}
Then, on the route, depending of the logic, the if,else choose what type socket response do
router.post('/post/voteup',checkAuthentication, async (req,res)=>{
//some logic
if(a.length <= 0){
io.getIo().emit('xxx', {action:'cleanAll'})
}
else if(b.length <= 0){
io.getIo().emit('xxx', {action:'cleanT',datoOne})
}
else{
io.getIo().emit('xxx', {action:'cleanX',dataTwo,dataOne,selected})
}
res.json({ serverResponse:'success'})
})
In the front (component) (activated with beforeUpdate life cycle hook)
getData(){
let socket = openSocket('http://localhost:3000')
socket.on('xxx', data => {
if(data.action === 'cleanX'){
if(this.selected === data.selected){
this.ddd = data.dataTwo
}
else if(!this.userTeamNickname){
this.qqq= data.dataOne
}
}
else if(data.action === 'cleanAll'){
this.ddd= []
this.qqq= []
}
else if(data.action === 'cleanT'){
this.ddd= data.dataOne
}
})
},
1. What kind of behavior can produce this such error?
2. Is any other most efficient way to do this?
It looks like socket.io is failing to establish a webSocket connection and has never advanced out of polling. By default, a socket.io connection starts with http polling and after a bit of negotiation with the server, it attempts to establish a webSocket connection. If that succeeds, it stops doing the polling and uses only the webSocket connection. If the the webSocket connection fails, it just keeps doing the polling.
Here are some reasons that can happen:
You have a mismatched version of socket.io in client and server.
You have some piece of infrastructure (proxy, firewall, load balancer, etc...) in between client and server that is not letting webSocket connections through.
You've attached more than one socket.io server handler to the same web server. You can't do that as the communication will get really messed up as multiple server handlers try to respond to the same client.
As a test, you could force the client to connect only with webSocket (no polling at all to start) and see if the connection fails:
let socket = io(yourURL, {transports: ["websocket"]})'
socket.on('connect', () => {console.log("connected"});
socket.on('connect_error', (e) => {console.log("connect error: ", e});
socket.on('connect_timeout', (e) => {console.log("connect timeout: ", e});

How to trigger websocket send from another function in Node?

It's been a while since I've worked with Node and Websockets. Basically how do I get socket.send() to work from another function is what I'm stuck on.
const server = new WebSocket.Server({ port: 8080 });
server.on('connection', socket => {
socket.on('message', message => {
console.log(`received from a client: ${message}`);
});
socket.send('yo world!');
});
function onMessageHandler (target, context, msg, self) {
client.say(target, response);
server.socket.send(response);
console.log(response);
}
}
How do I get my onMessageHandler to trigger a socket send, this is fail... server.socket.send(response);
Seeing your question i think there is a lack of understanding on how Websockets work. I am assuming you're using https://github.com/websockets/ws
There are two things. First is the WebSocketerver which you've named as server and then an Individual Socket which you've named as socket
Now the thing to understand is socket is not accessible outside server.on() callback The reason for this is there could be 1000 of sockets connected at a given instance and there would be no way to uniquely identify a particular socket you want to send message to.
So ask yourself the question that your application wants to send message to an individual socket to send to everyone who is connected to your server (basically broadcast)
If you want to send to an individual, you will have to uniquely identify the user
this._wss = new WebSocket.Server({
port: ENV_APP_PORT_WS
});
this._wss.on("connection", async (ws: AppWebSocket, req: IncomingMessage) => {
// const ipAddress = req.connection.remoteAddress; // IP Address of User
logger.info(req);
const queryParams = url.parse(req.url, true).query;
let authUser: User;
try {
authUser = await this._authenticateWebSocket(queryParams);
} catch (e) {
// Terminate connection and return...
}
// WS User INIT
ws.isAlive = true;
ws.userId = authUser.id;
ws.uuid = Helpers.generateUUIDV4();
ws.send(JSON.stringify({
type: "connected",
env: ENV
}));
});
The above code will add a property to each socket object that will enable it to uniquely identify a particular user/socket.
While sending =>
onMessageHandler(targetUserId: number, message: string) {
const allSockets = <AppWebSocket[]>Array.from(this._wss.clients.values());
const targetSocket = allSockets.find(w => w.userId === targetUserId);
targetSocket.send(message);
}
If You want to send to all connect users, it's quite easy:
https://github.com/websockets/ws#server-broadcast

Reconnect socket on user login

I work with a setup created by create-react-app and use flux for data management and the application needs to implement socket on the client side (I use socket.io for this purpose).
Currently the socket is initialised in a Socket.js file the following way:
import io from 'socket.io-client';
import { remoteUrl } from './constants/RemoteUrl';
import SocketWorker from './utilities/SocketWorker';
let socket = io.connect(remoteUrl + '?role=user');
socket.on('statusChange', (data) => {
return SocketWorker.receiveOrderStatusChange(data);
})
export { socket };
It does work, however the problem is that it only tries to connect to the server once, when the site is loaded. When the user opens the site unauthenticated it does not connect and misses to reconnect, thus the connection is not established and socket events are not received
I have tried to create a class instead and react an API for reconnect on the object, like:
import io from 'socket.io-client';
import { remoteUrl } from './constants/RemoteUrl';
import SocketWorker from './utilities/SocketWorker';
function Socket() {
this.socket = io.connect(remoteUrl + '?role=user');
this.reconnect = () => {
this.socket = io.connect(remoteUrl + '?role=user');
}
}
let socket = new Socket();
socket.socket.on('statusChange', (data) => {
return SocketWorker.receiveOrderStatusChange(data);
})
export { socket };
I tried to call the Socket.reconnect() method, although it did not work and connection was not established either. Any idea or alternative solution?
The way I managed to solve this if anyone face the same problem with the Socket.io API:
First, you should encapsulate your Socket into an object created by the constructor, but there is no need to create a reconnect method as the connection is present already (and the auth can be handled through emitted events I will describe below) :
import io from 'socket.io-client';
import { remoteUrl } from './constants/RemoteUrl';
import SocketWorker from './utilities/SocketWorker';
function Socket() {
this.socket = io.connect(remoteUrl + '?role=user');
this.socket.on('statusChange', (data) => {
return SocketWorker.receiveOrderStatusChange(data);
})
};
const socket = new Socket();
export { socket };
You can import the socket anywhere within your project:
import {socket} from './Socket';
And you can call:
socket.socket.emit('joinRoleRoom','user');
socket.socket.emit('joinIdRoom', _user._id);
On the server side, you just need to handled these events as follow:
socket.on('joinRoleRoom', (role) => {
socket.join(role)
console.log('Client joined to: ' + role);
});
socket.on('joinIdRoom', (id) => {
console.log('Client joined to: ' + id);
socket.join(id)
});
The socket will join the necessary rooms based on their auth info obtained during the auth process.
The original accepted answer from sznrbrt would work, but beware it has a serious security flaw.
If you do the following an attacker could join a room by just passing the desired user_id and start to receive sensitive user information. It could be private messages between two individual.
socket.socket.emit('joinRoleRoom','user');
socket.socket.emit('joinIdRoom', user_id);
Socket.io has an option to pass extraHeaders. One can use that to pass a token from the client. The server would use the desired authentication algorithm to decrypt the token and get the user_id.
Example:
socket.js
import io from 'socket.io-client';
import { remoteUrl } from './constants/RemoteUrl';
import SocketWorker from './utilities/SocketWorker';
const socket = io.connect(remoteUrl + '?role=user');
const socketAuth = () => {
socket.io.opts.extraHeaders = {
'x-auth-token': 'SET_TOKEN',
};
socket.io.opts.transportOptions = {
polling: {
extraHeaders: {
'x-auth-token': 'SET_TOKEN',
},
},
};
socket.io.disconnect();
socket.io.open();
};
export { socket, socketAuth };
client.js
import { socket, socketAuth } from './socket';
//After user logs in
socketAuth();
server.js, using a package socketio-jwt-auth
io.use(jwtAuth.authenticate({
secret: 'SECRET',
succeedWithoutToken: true
}, (payload, done) => {
if (payload && payload.id) {
return done(null, payload.id);
}
return done();
}));

Resources