Turn a net socket into an observable - node.js

I have a net tcp socket node.js section of code that I would like to convert from using a callback to rx.
I looks like this in feed.js module:
var net = require('net');
var server = net.createServer(function(socket) {
...
// Handle incoming messages from clients.
socket.on('data', function (data) {
broadcast(data, socket);
});
...
}
function broadcast(message, sender)
{
...
onChangeHandler(stock.symbol, 'stock', stock);
}
}
function start(onChange) {
onChangeHandler = onChange;
}
exports.start = start;
server.listen(....);
Then the client of the above call registers a callback:
feed.start(function(room, type, message) {
//...Do something with the message
});
I would like to convert this to use an Rx Observable/Observer. I see that there is a way to make an observable stream from the web socket (although it uses a bidirectional Subject which I don't need):
fromWebSocket(address, protocol) {
var ws = new WebSocket(address, protocol);
// Handle the data
var osbervable = Rx.Observable.create (function (obs) {
// Handle messages
ws.onmessage = obs.onNext.bind(obs);
ws.onerror = obs.onError.bind(obs);
ws.onclose = obs.onCompleted.bind(obs);
// Return way to unsubscribe
return ws.close.bind(ws);
});
var observer = Rx.Observer.create(function (data) {
if (ws.readyState === WebSocket.OPEN) { ws.send(data); }
});
return Rx.Subject.create(observer, observable);
}
var socketSubject = fromWebSocket('ws://localhost:9999', 'sampleProtocol');
// Receive data
socketSubject.subscribe(
function (data) {
// Do something with the data
},
function (error) {
// Do something with the error
},
function () {
// Do something on completion
});
// Send data
socketSubject.onNext(42);
What is the equivalent for a net socket? If there is a standard library to use that is ok.
My initial attempt is thus, but I don't know how to tie Rx and and the socket functions together into an onnext:
var net = require('net');
fromNetSocket(address, protocol) {
var ns = net.createServer(function(socket) {
socket.on('disconnect', function () { // This seems like it maps to onclose
console.log('User disconnected. %s. Socket id %s', socket.id);
});
// Handle incoming messages from clients.
socket.on('data', function (data) { //this should map to onnext
});
// Handle the data
var osbervable = Rx.Observable.create (function (obs) {
// Handle messages
ns.onmessage = obs.onNext.bind(obs);
ns.onerror = obs.onError.bind(obs);
ns.onclose = obs.onCompleted.bind(obs);
// Return way to unsubscribe
return ns.close.bind(ns);
});
});
};

Try the following
const createSubject = () => {
return Rx.Observable.create((observer) => {
const socket = net.connect({port: 1705}, () => {
log.i('Connected to Server!');
let socketObservable = Rx.Observable.create((observer) => {
socket.on('data', (data) => observer.next(JSON.parse(data)));
socket.on('error', (err) => observer.error(err));
socket.on('close', () => observer.complete());
});
let socketObserver = {
next: (data) => {
if (!socket.destroyed) {
socket.write(`${JSON.stringify(data)}\r\n`);
}
}
};
const subject = Rx.Subject.create(socketObserver, socketObservable);
observer.next(subject);
observer.complete();
});
});
};
Then you can use the subject like this
createSubject().subscribe((con) => {
con.subscribe((data) => console.log(data));
con.next({
id: utils.UUID(),
jsonrpc: '2.0',
method: 'Server.GetRPCVersion'
});
});

Related

Node.js net socket

I'm using the net module to create a listener but I've experienced some issues. I'm trying to make it wait till it's done writing the "text" to the client before the client can type again. If I'm not doing this and I hold in enter it'll just make you able to write enters between text leading to weird formatting etc.
So how could I make it wait till it's written to the client?
Code:
const net = require('net');
const server = new net.Server();
server.on('connection', async function (socket) {
console.log("Client connected!");
socket.on('data', async function (data) {
socket.setEncoding('utf8');
let input = data.toString().replace(/(\r\n|\n|\r)/gm, "");
if (input == "echo")
socket.write("$ ");
else
socket.write("invalid command");
});
});
server.listen(1337, function() {
console.log("listening");
});
Picture:
https://imgur.com/a/lc21Y13
Edit:
This is on localhost, let's say I'd host it on a server so there's a higher ping it's way worse.
Edit:
Here a picture from when it's hosted on a server:
https://imgur.com/a/LIKRRr9
Edit:
I've tried using SSH instead of telnet and raw and got basically the same result now.
Picture:
https://imgur.com/a/XJmpGSa
Code:
var fs = require('fs');
var username = null;
var ssh2 = require('ssh2');
new ssh2.Server({
hostKeys: [fs.readFileSync('ssh.key')]
}, function (client) {
console.log('Client connected!');
client.on('authentication', function (ctx) {
if (ctx.method !== 'password') return ctx.reject(['password']);
if (ctx.method === 'password') {
username = ctx.username;
console.log(username);
console.log(ctx.password);
ctx.accept();
}
else {
console.log("rejected.");
ctx.reject();
}
}).on('ready', function () {
console.log('Client authenticated!');
client.on('session', function (accept, reject) {
var session = accept();
session.once('shell', function (accept, reject, info) {
var stream = accept();
stream.write("$ ");
stream.on('data', function (data) {
var args = data.toString().split(" ");
console.log(args);
switch (args[0]) {
case "echo":
args.shift();
stream.write(args.join(" ") + "\r\n");
break;
case "whoami":
stream.write(username + "\r\n");
break;
case "exit":
stream.exit(0);
stream.end();
stream = undefined;
break;
default:
stream.stderr.write(args[0] + ": No such command!\r\n");
break;
}
if (typeof stream != 'undefined') {
stream.write("$ ");
}
});
});
});
}).on('end', function () {
console.log('Client disconnected');
});
}).listen(1337, function () {
console.log('Listening on port ' + this.address().port);
});
Try This. What this code does is simply buffering until \n enter is received from a client.
const net = require("net");
const readline = require("readline");
const execCommand = (command, args, socket) => {
return new Promise((res, rej) => {
setTimeout(() => {
// to clear the terminal
socket.write("\u001B[2J\u001B[0;0f");
socket.write(
`Executed command: ${command} with args: ${args} and result was: ${Math.random()}`
);
socket.write('\n>')
res();
}, 3000);
});
};
const server = net.createServer((socket) => {
socket.write("Connected");
// nice prompt
socket.write("\n>");
const rl = readline.createInterface({
input: socket,
output: socket,
});
rl.on("line", (line) => {
if (line.length === 0) {
socket.write("No command to execute!");
socket.write('\n>')
return;
}
// destructuring command and args
// E.g. command arg1 arg2 ....
const [command, ...args] = line.split(" ");
execCommand(command, args, socket);
});
});
server.listen(1337, "127.0.0.1");

New connection cause the current connections to stop working

I am making the chat application using socket (which I'm new at) with multiple tenants structure and using namespaces. Here's my code:
Socket server:
index.js
class Server {
constructor() {
this.port = process.env.PORT || 3000;
this.host = process.env.HOST || `localhost`;
this.app = express();
this.http = http.Server(this.app);
this.rootSocket = socketio(this.http);
}
run() {
new socketEvents(this.rootSocket).socketConfig();
this.app.use(express.static(__dirname + '/uploads'));
this.http.listen(this.port, this.host, () => {
console.log(`Listening on ${this.host}:${this.port}`);
});
}
}
const app = new Server();
app.run();
socket.js
var redis = require('redis');
var redisConnection = {
host: process.env.REDIS_HOST,
password: process.env.REDIS_PASS
};
var sub = redis.createClient(redisConnection);
var pub = redis.createClient(redisConnection);
class Socket {
constructor(rootSocket) {
this.rootIo = rootSocket;
}
socketEvents() {
/**
* Subscribe redis channel
*/
sub.subscribe('visitorBehaviorApiResponse');
//TODO: subscribe channel..
// Listen to redis channel that published from api
sub.on('message', (channel, data) => {
data = JSON.parse(data);
console.log(data);
const io = this.rootIo.of(data.namespace);
if (channel === 'visitorBehaviorApiResponse') {
io.to(data.thread_id).emit('receiveBehavior', data);
io.to('staff_room').emit('incomingBehavior', data);
}
})
sub.on('error', function (error) {
console.log('ERROR ' + error)
})
var clients = 0;
this.rootIo.on('connection', (rootSocket) => {
clients++;
console.log('root:' + rootSocket.id + ' connected (total ' + clients + ' clients connected)');
const ns = rootSocket.handshake['query'].namespace;
// Dynamic namespace for multiple tenants
if (typeof (ns) === 'string') {
const splitedUrl = ns.split("/");
const namespace = splitedUrl[splitedUrl.length - 1];
const nsio = this.rootIo.of(namespace);
this.io = nsio;
this.io.once('connection', (socket) => {
var visitors = [];
console.log('new ' + socket.id + ' connected');
// once a client has connected, we expect to get a ping from them saying what room they want to join
socket.on('createChatRoom', function (data) {
socket.join(data.thread_id);
if (typeof data.is_staff !== 'undefined' && data.is_staff == 1) {
socket.join('staff_room');
} else {
if (visitors.some(e => e.visitor_id === data.visitor_id)) {
visitors.forEach(function (visitor) {
if (visitor.visitor_id === data.visitor_id) {
visitor.socket_ids.push(socket.id);
}
})
} else {
data.socket_ids = [];
data.socket_ids.push(socket.id);
visitors.push(data);
}
socket.join('visitor_room');
}
//TODO: push to redis to check conversation type
});
socket.on('sendMessage', function (data) {
console.log(data);
pub.publish('chatMessage', JSON.stringify(data));
this.io.in(data.thread_id).emit('receiveMessage', data);
this.io.in('staff_room').emit('incomingMessage', data);
// Notify new message in room
data.notify_type = 'default';
socket.to(data.thread_id).emit('receiveNotify', data);
}.bind(this))
socket.on('disconnect', (reason) => {
sub.quit();
console.log('client ' + socket.id + ' left, ' + reason);
});
socket.on('error', (error) => {
console.log(error);
});
});
}
// Root disconnect
rootSocket.on('disconnect', function () {
clients--;
console.log('root:' + rootSocket.id + ' disconnected (total ' + clients + ' clients connected)');
});
});
}
socketConfig() {
this.socketEvents();
}
}
module.exports = Socket;
Client:
const server = 'https://socket-server'
const connect = function (namespace) {
return io.connect(namespace, {
query: 'namespace=' + namespace,
resource: 'socket.io',
transports: ['websocket'],
upgrade: false
})
}
const url_string = window.location.href
const url = new URL(url_string)
const parameters = Object.fromEntries(url.searchParams)
const socket = connect(`${server}/${parameters.shopify_domain}`)
var handleErrors = (err) => {
console.error(err);
}
socket.on('connect_error', err => handleErrors(err))
socket.on('connect_failed', err => handleErrors(err))
socket.on('disconnect', err => handleErrors(err))
The problem that I met is when socket server got a new connection, the existing connections will be stopped working util they make a page refreshing to reconnect a new socket.id.
And when a namespace's client emit data, it sends to other namespaces, seem my code is not work correctly in a namespace.
Could you take a look at my code and point me where I'm wrong?
Thanks
1) Get UserId or accessToken while handshaking(in case of accessToken decrypt it).
and store userID: socketId(in Redis or in local hashmap) depends upon the requirement .
2) When u are going to emit to particular user fetch the socketid to that userid from redis or local hashmap
and emit to it.
**io.to(socketId).emit('hey', 'I just met you');**
3) If you are using multiple servers use sticky sessions
4) Hope this will help you

share mqtt client object between files

I connect to MQTT this way:
//mqtt.js
const mqtt = require('mqtt');
var options = {
//needed options
};
var client = mqtt.connect('mqtt://someURL', options);
client.on('connect', () => {
console.log('Connected to MQTT server');
});
I want to export the client object this way:
//mqtt.js
module.exports = client;
So that I can import it in other files and make use of it this way:
//anotherFile.js
const client = require('./mqtt');
client.publish(...)
However, we all know that this will not work! How can I achieve this ?
Update
I tried promise and get a very strange behavior. When I use the promise in the same file (mqtt.js) like the code below, everything is OK:
//mqtt.js
const mqtt = require('mqtt');
var mqttPromise = new Promise(function (resolve, reject) {
var options = {
//needed options
};
var client = mqtt.connect('mqtt://someURL', options);
client.on('connect', () => {
client.subscribe('#', (err) => {
if (!err) {
console.log('Connected to MQTT server');
resolve(client);
} else {
console.log('Error: ' + err);
reject(err);
}
});
});
});
mqttPromise.then(function (client) {
//do sth with client
}, function (err) {
console.log('Error: ' + err);
});
But when I export the promise and use it in another file, like this:
//mqtt.js
//same code to create the promise
module.exports = mqttPromise;
//anotherFile.js
const mqttPromise = require('./mqtt');
mqttPromise.then(function (client) {
//do sth with client
}, function (err) {
console.log('Error: ' + err);
});
I get this error:
TypeError: mqttPromise.then is not a function
You can probably achieve your goal creating 2 files, one for handling mqtt methods and another to manage the connection object.
Here's the file for the mqtt handler:
//mqttHandler.js
const mqtt = require('mqtt');
class MqttHandler {
constructor() {
this.mqttClient = null;
this.host = 'YOUR_HOST';
this.username = 'YOUR_USER';
this.password = 'YOUR_PASSWORD';
}
connect() {
this.mqttClient = mqtt.connect(this.host, {port: 1883});
// Mqtt error calback
this.mqttClient.on('error', (err) => {
console.log(err);
this.mqttClient.end();
});
// Connection callback
this.mqttClient.on('connect', () => {
console.log(`mqtt client connected`);
});
this.mqttClient.on('close', () => {
console.log(`mqtt client disconnected`);
});
}
// // Sends a mqtt message to topic: mytopic
sendMessage(message, topic) {
this.mqttClient.publish(topic, JSON.stringify(message));
}
}
module.exports = MqttHandler;
Now lets use the exported module to create a mqtt client connection on another file:
//mqttClient.js
var mqttHandler = require('./mqttHandler');
var mqttClient = new mqttHandler();
mqttClient.connect();
module.exports = mqttClient;
With this exported module you can now call your client connection object and use the methods created in the mqttHandler.js file in another file :
//main.js
var mqttClient = require('./mqttClient');
mqttClient.sendMessage('<your_topic>','<message>');
Although there may be a better method to perform your task, this one worked pretty well for me...
Hope it helps!
cusMqtt.js
const mqtt = require("mqtt");
function prgMqtt() {
const options = {
port: 1883,
host: "mqtt://xxxxxxx.com",
clientId: "mqttjs_" + Math.random().toString(16).substr(2, 8),
username: "xxxxxx",
password: "xxxxxx",
keepalive: 60,
reconnectPeriod: 1000,
protocolId: "MQIsdp",
protocolVersion: 3,
clean: true,
encoding: "utf8",
};
prgMqtt.client = mqtt.connect("mqtt://xxxxxxxx.com", options);
prgMqtt.client.on("connect", () => {
prgMqtt.client.subscribe("Room/Fan");
console.log("connected MQTT");
});
prgMqtt.client.on("message", (topic, message) => {
console.log("message is " + message);
console.log("topic is " + topic);
// client.end();
});
}
exports.prgMqtt = prgMqtt;
index.js/main program call
const { prgMqtt } = require("./startup/cusMqtt");
prgMqtt();
another .js
const { prgMqtt } = require("../startup/cusMqtt");
router.get("/:id", async (req, res) => {
prgMqtt.client.publish("Room/Reply", "Replied Message");
});

Node.js memory usage remains high even after destroying sockets

I've created a simple nodejs server and client. They interact with each other via 15000 tcp sockets.
client code:
'use strict';
const net = require('net');
for (let i = 0; i < 15000; ++i) {
let socket = new net.Socket();
socket.connect(6000, '127.0.0.1', () => {
console.log('Connected');
socket.write('data to server');
});
socket.on('data', data => {
console.log(data);
});
socket.on('close', () => {
console.log('Connection closed');
});
}
server code:
'use strict';
const net = require('net');
let sockets = [];
let server = net.createServer(socket => {
socket.write('blabla from server');
socket.on('data', data => {
console.log(data);
});
sockets.push(socket);
if (sockets.length >= 15000) {
setTimeout(() => {
console.log('cleanup start');
for (let socket of sockets) {
socket.end();
socket.destroy();
socket.unref();
}
console.log('cleaned up and ready');
}, 80000);
}
});
if (global.gc) {
setInterval(() => {
global.gc();
}, 15000);
}
setInterval(() => {
console.log(process.memoryUsage());
}, 5000);
server.listen(6000, '127.0.0.1');
They send and receive messages. During creation of sockets the memory usage gets high. But after destroying the sockets I expect the memory usage to get low, which doesn't happen.
You have an array which is full with null values.
sockets.push(socket);
if (sockets.length >= 15000) { // The length is never decreasing
setTimeout(() => {
for (let socket of sockets) {
[...]
socket = null; // Because you just null the value but the array has still the same length.
}
}, 80000);
}
Have a look into this example:
var arr=[];
setInterval(function(){
arr.push(null);
}, 1);
setInterval(() => {
console.log(process.memoryUsage());
console.log(arr.length); // You will see the array is growing continously
}, 5000);
So the solution is to choose a different iteration numeric and slice they keys free the array index instead of overwriting it with another property.
But why are you deleting all connections regardles if these are in use or not. You can destroy the socket better on clients disconnect.
sockets.slice(index,1);
Ok it seems I found the solution. Nodejs caches the Socket objects to be used in future. Here is the updated code.
server code:
'use strict';
const net = require('net');
let sockets = [];
let numberOfSockets = 2000;
let server = net.createServer(socket => {
sockets.push(socket);
if (sockets.length >= numberOfSockets) {
console.log(process.memoryUsage());
console.log('cleanup start');
for (let socket of sockets) {
socket.end();
socket.destroy();
socket.unref();
}
sockets = [];
global.gc();
setTimeout(() => {
console.log(process.memoryUsage());
}, 1000);
console.log('cleaned up and ready');
}
});
if (global.gc) {
setInterval(() => {
global.gc();
}, 15000);
}
server.listen(6000, '127.0.0.1');
client code:
'use strict';
const net = require('net');
let numberOfClosedSockets = 0;
let numberOfSockets = 2000;
function test() {
for (let i = 0; i < numberOfSockets; ++i) {
let socket = new net.Socket();
socket.connect(6000, '127.0.0.1', () => {
console.log('Connected');
});
socket.on('data', data => {
console.log(data);
});
socket.on('close', () => {
console.log('Connection closed ' + numberOfClosedSockets);
numberOfClosedSockets++;
if (numberOfClosedSockets >= numberOfSockets) {
numberOfClosedSockets = 0;
setTimeout(() => {
test();
}, 2000);
}
socket.destroy();
socket.unref();
});
}
}
test();
In this example the client creates new sets of sockets after the first sets of sockets has been destroyed. And it does it over and over again. If you look at memory usage it doesn't increase so there is no memory leak.
github issue link

Custom TCP-Protocol on node.js

how i can implement a custom protocol on a Node.js NET-Client?
The problem is:
I want to connect to a server. That server has an simple protocol.
Each packet has a length-prefix, it means the first byte say how long the packet is.
How i can implement that?
How i can read for example the first byte to get the packet-length to read the other stuff?
var net = require('net');
var client = new net.Socket();
client.connect(2204, 'myexampledomain.com', function() {
console.log('Connecting...', protocol);
});
client.on('data', function(data) {
console.log('DATA: ' + data);
});
client.on('end', function() {
console.log('end');
});
client.on('timeout', function() {
console.log('timeout');
});
client.on('drain', function() {
console.log('drain');
});
client.on('error', function() {
console.log('error');
});
client.on('close', function() {
console.log('Connection closed');
});
client.on('connect', function() {
console.log('Connect');
client.write("Hello World");
});
You'll have to maintain an internal buffer holding received data and check if it has length bytes before slicing the packet from it. The buffer must be concatenated with received data until it has length bytes and emptied on receiving a full packet. This can be better handled using a Transform stream.
This is what I used in my json-rpc implementation. Each JSON packet is lengthPrefixed. The message doesn't have to be JSON – you can replace the call to this.push(JSON.parse(json.toString()));.
var Transform = require('stream').Transform;
function JsonTransformer(options) {
if (!(this instanceof JsonTransformer)) {
return new JsonTransformer(options);
}
Transform.call(this, {
objectMode: true
});
/*Transform.call(this);
this._readableState.objectMode = false;
this._writableState.objectMode = true;*/
this.buffer = new Buffer(0);
this.lengthPrefix = options.lengthPrefix || 2;
this._readBytes = {
1: 'readUInt8',
2: 'readUInt16BE',
4: 'readUInt32BE'
}[this.lengthPrefix];
}
JsonTransformer.prototype = Object.create(Transform.prototype, {
constructor: {
value: JsonTransformer,
enumerable: false,
writable: false
}
});
function transform() {
var buffer = this.buffer,
lengthPrefix = this.lengthPrefix;
if (buffer.length > lengthPrefix) {
this.bytes = buffer[this._readBytes](0);
if (buffer.length >= this.bytes + lengthPrefix) {
var json = buffer.slice(lengthPrefix, this.bytes + lengthPrefix);
this.buffer = buffer.slice(this.bytes + lengthPrefix);
try {
this.push(JSON.parse(json.toString()));
} catch(err) {
this.emit('parse error', err);
}
transform.call(this);
}
}
}
JsonTransformer.prototype._transform = function(chunk, encoding, next) {
this.buffer = Buffer.concat([this.buffer, chunk]);
transform.call(this);
next();
}
JsonTransformer.prototype._flush = function() {
console.log('Flushed...');
}
var json = new JsonTransformer({lengthPrefix: 2});
var socket = require('net').createServer(function (socket) {
socket.pipe(json).on('data', console.log);
});
socket.listen(3000);
module.exports = JsonTransformer;
json-transformer

Resources