using WiFly with ws websockets - node.js

I have been trying to figure out a way to connect a WiFly from an Arduino to send some accelerometer data to my node.js server. Currently the way that I have it worked out is having three servers:
Http >> This is for clients purposes
Net server >> This is basically for TCP request, this is how my server receives the information from 3. WS websockets >> this takes the data from the Net server and streams it to the client side.
Here is the code:
var http = require('http');
var fs = require('fs');
var path = require('path');
var url = require('url');
var net = require('net');
var sensorData;
var message = {
"data": ''
}
var newValue,
oldValue,
diff;
//Settings
var HTTP_PORT = 9000;
var NET_PORT = 9001;
var WS_PORT = 9002;
//Server
var mimeTypes = {
"html": "text/html",
"jpeg": "image/jpeg",
"jpg": "image/jpeg",
"png": "image/png",
"js": "text/javascript",
"css": "text/css"
};
http.createServer(function (req, res) {
var fileToLoad;
if (req.url == '/') {
fileToLoad = 'index.html';
} else {
fileToLoad = url.parse(req.url).pathname.substr(1);
}
console.log('[HTTP] :: Loading :: ' + 'frontend/' + fileToLoad);
var fileBytes;
var httpStatusCode = 200;
fs.exists('frontend/' + fileToLoad, function (doesItExist) {
if (!doesItExist) {
console.log('[HTTP] :: Error loading :: ' + 'frontend/' + fileToLoad);
httpStatusCode = 404;
}
var fileBytes = fs.readFileSync('frontend/' + fileToLoad);
var mimeType = mimeTypes[path.extname(fileToLoad).split('.')[1]];
res.writeHead(httpStatusCode, {
'Content-type': mimeType
});
res.end(fileBytes);
});
// console.log("[INIT] Server running on HTTP Port");
}).listen(HTTP_PORT);
proxy.on("close", function(){
console.log("Connection has closed");
});
proxy.on("end", function(){
console.log("Connection has ended");
});
var socket;
var clients = [];
var socketObject;
var server = net.createServer(function (socket) {
socketObject = socket;
socket.name = socket.remoteAddress + ":" + socket.remotePort;
clients.push(socket);
console.log(socket);
socket.write("HTTP/1.1 101", function () {
console.log('[CONN] New connection: ' + socket.name + ', total clients: ' + clients.length);
});
socket.setEncoding('utf8');
socket.on('error', function (data) {
console.log(data);
});
socket.on('end', function () {
console.log('[END] Disconnection: ' + socket.name + ', total clients: ' + clients.length);
});
socket.on('data', function (data) {
console.log('[RECV from ' + socket.remoteAddress + "] " + data);
oldValue = newValue;
newValue = data;
diff = Math.abs(newValue) - Math.abs(oldValue);
console.log(Math.abs(newValue) + '-' + Math.abs(oldValue));
message.data = Math.abs(diff);
console.log('[SAVED] ' + message.data);
});
});
server.listen(NET_PORT, function () {
console.log("[INIT] Server running on NET server port", NET_PORT);
});
var WebSocketServer = require('ws').Server,
wss = new WebSocketServer({
port: WS_PORT
});
wss.on('connection', function (ws) {
// ws.send(JSON.stringify(message));
setInterval(function () {
updateXData(ws)
}, 500);
});
function updateXData(ws) {
var newMessage = {
"data": ""
}
newMessage.data = message.data
ws.send(JSON.stringify(newMessage));
}
So the question is: Is there a cleaner way to do this just by using ws to handle the data from the WiFly and then sending it to the client?
Thanks in advance!

Not sure whether this will suit you and might be new to you but you could make use of MQTT, there are free brokers available which are very good and its relatively easy to set up and implement with Arduino equipped with WiFly Shield.
http://mqtt.org/
Hope this helps somewhat!

Related

Dynamically Create Kafka Consumers That Send Data Through a Websocket

I am using kafka-node to read stream data and pass it to my web app with Web Sockets using NodeJS. This works fine if I able to define the kafka producer server and the topic I am interested in, however for my use case the end users will input the kafka producer server and the topic and my NodeJS backend will be responsible to receive that request and to create the appropriate kafka/websocket connections.
My idea was the following:
Create a rest API to which the web app could send requests to in order to create a new kafka consumer/web socket connection (/registerTopic)
Save the new kafka consumers in a global array when I create a new kafka consumer so that I can later pause or resume the stream with another rest API call (/pauseTopic and /resumeTopic)
I ran into problems trying to move the WebSocket code into /registerTopic...Whenever I do this everything acts very strangely and I suddenly get 1000x messages at once and then 40-50x messages every second even though the kafka producer is only sending 1 message per second. Any ideas on how I can get this working?
const express = require("express");
const app = express();
const cors = require('cors');
const bodyParser = require('body-parser');
const fs = require('fs');
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
var WebSocketServer = require('websocket').server;
var http = require('http');
var https = require('https');
var kafka = require('kafka-node');
var topics = [];
var privateKey = fs.readFileSync('PATH', 'utf8');
var certificate = fs.readFileSync('PATH', 'utf8');
var credentials = { key: privateKey, cert: certificate };
var consumers = new Set();
// This was working without any issues before I tried to make this dynamic!
/* var Consumer = kafka.Consumer,
client = new kafka.KafkaClient('localhost:9092'),
consumer = new Consumer(
client, [{ topic: 'numtest', partition: 0 }], { autoCommit: false }); */
var server = http.createServer(function (request, response) {
console.log(' Request recieved : ' + request.url);
response.writeHead(404);
response.end();
});
server.listen(8080, function () {
console.log('Listening on port : 8080');
});
webSocketServer = new WebSocketServer({
httpServer: server,
autoAcceptConnections: false
});
function iSOriginAllowed(origin) {
return true;
}
// This was working without any issues before I tried to make this dynamic!
/* webSocketServer.on('request', function (request) {
if (!iSOriginAllowed(request.origin)) {
request.reject();
console.log(' Connection from : ' + request.origin + ' rejected.');
return;
}
let connection = request.accept('echo-protocol', request.origin);
console.log(' Connection accepted : ' + request.origin);
connection.on('message', function (message) {
if (message.type === 'utf8') {
console.log('Received Message: ' + message.utf8Data);
}
});
consumer.on('message', function (message) {
console.log('msg');
connection.sendUTF(message.value);
});
connection.on('close', function (reasonCode, description) {
console.log('Connection ' + connection.remoteAddress + ' disconnected.');
});
}); */
var httpsServer = http.createServer(credentials, app);
httpsServer.listen(3000, () => {
console.log("Server running on port 3000");
});
app.get("/getTopics", (req, res, next) => {
res.json(topics);
});
app.post("/registerTopic", (req, res) => {
try {
var client = new kafka.KafkaClient(req.body.host);
var Consumer = kafka.Consumer;
consumer = new Consumer(
client, [{ topic: req.body.topic, partition: 0 }], { autoCommit: false });
let consumerExists = false;
for (let c = 0; c < [...consumers].length; c++) {
if ([...consumers][c].topic == req.body.topic && [...consumers][c].sessionId == req.body.sessionId) {
consumerExists = true;
}
}
if (!consumerExists) {
consumers.add({ 'topic': req.body.topic, 'sessionId': req.body.sessionId, 'consumer': consumer });
}
client.loadMetadataForTopics([], function (error, results) {
Object.keys(results[1].metadata).forEach(function (key) {
var value = results[1].metadata[key];
if (!value['0'].topic.includes('__') && !value['0'].topic.includes('offset')) {
topics.push({ 'producer': req.body.host, 'topic': value['0'].topic });
}
});
});
webSocketServer.on('request', function (request) {
if (!iSOriginAllowed(request.origin)) {
request.reject();
console.log(' Connection from : ' + request.origin + ' rejected.');
return;
}
let connection = request.accept('echo-protocol', request.origin);
console.log(' Connection accepted : ' + request.origin);
connection.on('message', function (message) {
if (message.type === 'utf8') {
console.log('Received Message: ' + message.utf8Data);
}
});
consumer.on('message', function (message) {
console.log('msg');
connection.sendUTF(message.value);
});
connection.on('close', function (reasonCode, description) {
console.log('Connection ' + connection.remoteAddress + ' disconnected.');
});
});
res.json("Working");
} catch (error) {
console.error(error);
res.status(400).send('Unable to register new topic')
}
});
app.post("/pauseTopic", (req, res) => {
try {
console.log(req.body);
let filteredConsumer = [...consumers].filter(function (item) {
console.log(req.body.topic, item.sessionId);
if (item.topic == req.body.topic && item.sessionId == req.body.sessionId) {
return c;
}
});
console.log(filteredConsumer);
//filteredConsumer[0].consumer.pause();
res.json("Working");
} catch (error) {
console.error(error);
res.status(400).send('Unable to register new topic')
}
});
app.post("/resumeTopic", (req, res) => {
try {
let filteredConsumer = [...consumers].filter(function (item) {
if (item.topic == req.body.topic && item.sessionId == req.body.sessionId) {
return item;
}
});
filteredConsumer[0].consumer.resume();
res.json("Working");
} catch (error) {
console.error(error);
res.status(400).send('Unable to register new topic')
}
});

Send message synchronously with Nodejs Websocket

My app has three modules:
WebSocket Module: resides on the remote server (internet) and it acts as a Websocket
server that entertain the connection between Client Module and
Webhook module.
Webhook Module: resides on the remote server (internet) and it acts as
a webhook to answer the HTTP post request from the user. It is connected to Websocket Module via websocket as well.
Client Module: resides on my local machine and it is connected to
the
Webhook model via websocket. This client responsible to get query
from my local backend.
When a user call Webhook Module through HTTP Post request, Webhook module initiate a connection to WebSocket module via websocket. Then, the WebSocket module initiate the connection to Client module and response back with the necessary information. Actually I have to do this to eliminate the HTTP tunnel that is blocked in my company.
The problem is, when I open two browser windows to perform the HTTP Post request at the same time with different parameter, for example, param "A", I would expect to get return "A", with param "B", I expect to get "B" not "A". But There is something wrong with my codes/design. If I executed at the same time, I throw "A" then I got "B" which is wrong. How do I overcome this.
This is a simple diagram to illustrate it.
WebSocket Module:
'use strict'
//This is WebSocket Server
const clients = {};
const SocketIO = require('socket.io');
const express = require('express');
const http = require('http');
const app = express();
const server = http.createServer(app);
const ws = SocketIO(server);
const port = 3000;
var clientid;
ws.on('connection', (client) => {
clients[client.id] = client;
console.log('new connection', client.id);
clientid = client.id;
client.emit('message', { message: 'welc' })
client.on('disconnect', () => {
delete clients[client.id];
console.log('Client ' + client.id + ' disconnected. Deleted');
});
client.on('WH', function (from, msg) {
console.log('Message from Webhook', from, ' : ', msg);
client.broadcast.emit('message', { message: msg });
//console.log('send to: ' + clientid);
//ws.to(clientid).emit('hey', { message: msg });
//client.emit('message', { message: msg })
});
client.on('CL', function (from, msg) {
console.log('Message from Client', from, ' : ', msg);
client.broadcast.emit('message', 'me', msg);
//ws.to(client.id).emit('message', 'me', msg);
//client.emit('message', 'me', msg);
});
});
server.listen(process.env.PORT || port);
console.log('WebSocket Server is running on port ' + port);
Webhook Module
'use strict'
//This is WebHook Server
const express = require('express');
const bodyParser = require('body-parser');
const request = require('request');
const http = require('http');
const io = require('socket.io-client');
const app = express()
app.use(bodyParser.json())
const clients = {};
const SocketIO = require('socket.io');
const server = http.createServer(app);
const ws = SocketIO(server);
const port = 5000;
let Res;
let httpreq = false;
let nctid;
let ts;
const socket = io.connect('http://localhost:3000', {reconnect: true});
// Add a connect listener
socket.on('connect', function() {
console.log('Connected to WebSocket server!');
});
socket.on('message', function(from, msg) {
//console.log('Message from ', from, ' : ', msg);
console.log('nctid: ' + nctid + ', ts: ' + ts);
//Get the message from Client
if (httpreq) {
Res.send({
replies: [{
type: 'text',
content: msg,
}],
conversation: {
memory: {
key: msg
}
}
})
httpreq = false;
}
});
app.listen(process.env.PORT || port, () => {
console.log('Webhook server is running on port ' + port);
})
app.post('/', (req, res) => {
//console.log(req.body)
let query = req.body.nlp.entities.query[0].value;
nctid = req.body.nlp.entities.nctid[0].value;
ts = Math.floor(Date.now() / 1000);
console.log("query: " + query + '|' + nctid + '|' + ts);
//Send message to WebSocket server with parameter query and NCTID
socket.emit('WH', 'me', query + '|' + nctid);
Res = res;
httpreq = true;
})
app.post('/errors', (req, res) => {
console.log(req.body);
res.send();
})
Client Module
'use strict'
//This is client app running on client premise
const request = require('request');
const parser = require('xml2json');
const io = require('socket.io-client');
const socket = io.connect('http://localhost:3000', {reconnect: true});
// Add a connect listener
socket.on('connect', function(socket) {
console.log('Connected to WebSocket server!');
});
socket.on('message', function(from, msg) {
//console.log('MSG', from, ' : ', msg);
console.log(from);
let param = from.message.split('|');
let query = param[0];
let nctid = param[1];
if (typeof nctid != 'undefined') {
getNCTID(nctid, function(returnValue) {
//console.log(returnValue);
try {
let json = parser.toJson(returnValue);
json = JSON.parse(json);
if (query == 'title')
socket.emit('CL', 'me', 'Title is ' + json.clinical_study.brief_title);
else
socket.emit('CL', 'me', 'Status is ' + json.clinical_study.overall_status);
} catch (e) {
console.log(e);
socket.emit('CL', 'me', 'No NCTID ' + nctid);
}
});
}
});
function getNCTID(nctid, callback) {
let url = "https://clinicaltrials.gov/ct2/show/" + nctid + "?displayxml=true";
let options = {
url: url,
method: 'GET'
}
//console.log(options);
let requestWithEncoding = function(options, callback) {
let req = request.get(options);
req.on('response', function(res) {
let chunks = [];
res.on('data', function(chunk) {
chunks.push(chunk);
});
res.on('end', function() {
let buffer = Buffer.concat(chunks);
let encoding = res.headers['content-encoding'];
if (encoding == 'gzip') {
zlib.gunzip(buffer, function(err, decoded) {
callback(err, decoded && decoded.toString());
});
} else if (encoding == 'deflate') {
zlib.inflate(buffer, function(err, decoded) {
callback(err, decoded && decoded.toString());
})
} else {
callback(null, buffer.toString());
}
});
});
req.on('error', function(err) {
callback(err);
});
}
requestWithEncoding(options, function(err, data) {
if (err) {
console.log('err:' + err);
callback('error');
} else
//console.log(data);
callback(data);
})
}

Why i can't use Primus.connect()?

I download a example at https://github.com/strongloop/strong-pubsub-example
When I use Primus.connect(url). It's occurs error that: "Primus.connect is not a function".
This is my pub.js
var PORT = process.env.PORT;
var TOPIC = process.env.TOPIC;
var Client = require('strong-pubsub');
var Adapter = require('strong-pubsub-mqtt');
var readline = require('readline');
var client = new Client({port: PORT}, Adapter);
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
(function prompt() {
rl.question('to "' + TOPIC + '": ', function(answer) {
client.publish(TOPIC, answer);
prompt();
});
})();
And this is my sub.js
var PORT = process.env.PORT;
var TOPIC = process.env.TOPIC;
var Client = require('strong-pubsub');
var Adapter = require('strong-pubsub-mqtt');
var client = new Client({port: PORT}, Adapter);
client.subscribe(TOPIC);
var duplex = require('duplex');
var Primus = require('primus');
Primus.Stream = require('stream');
var server = new Client({port: PORT, host: 'localhost'}, Adapter, {
createConnection: function(port, host) {
var connection = duplex();
var primus = Primus.connect('http://' + host + ':' + port, {
transformer: 'engine.io',
parser: 'binary'
});
connection.on('_data', function(chunk) {
// someone called `connection.write(buf)`
primus.write(chunk);
});
primus.on('data', function(chunk) {
// chunk is an arrayBuffer
connection._data(toBuffer(chunk));
});
primus.on('open', function() {
connection.emit('connect');
});
connection.on('_end', function() {
primus.end();
this._end();
});
return connection;
}
});
client.on('message', function(topic, msg) {
console.log(msg.toString());
server.publish(topic,msg);
});
And errors

Nodejs - data transfer between server and client

I was given a task to send JSON string from client to server and from server to client, whenever there is a new record found to send.
I decided to build TCP connection(suggest me if there is any other better way in Node.js) between server and client to transfer data.
The problem is, I was supposed to use a delimiter to separate JSON strings one from another. I am afraid what if the json string contains the delimiter string inside the object. I am looking for a better way to separate two JSON strings.
Below is my code. Please help me.
Client
var net = require('net')
, client = new net.Socket();
var chunk = ''
, dlim_index = -1
, delimit = '~~';
client.connect(config.Port, config.IpAddress, function () {
console.log('Server Connected');
client.write('CLIENTID:' + process.argv[2]);
client.write(delimit);
});
client.on('data', function (data) {
var recvData = data.toString().trim();
chunk += recvData;
dlim_index = chunk.indexOf(recvData);
console.log(data);
while (dlim_index > -1) {
var useData = chunk.substring(0, dlim_index);
if (useData == 'SUCCESS') {
controller.listenOutQueue(function (dataToSend) {
var object = JSON.parse(dataToSend);
client.write(dataToSend);
client.write(delimit);
});
}
else {
var record = JSON.parse(useData);
controller.insertIntoQueue(record, function (status) {
});
}
chunk = chunk.substring(dlim_index + 2);
dlim_index = chunk.indexOf(delimit);
}
});
client.on('close', function () {
console.log('Connection closed');
});
client.setTimeout(50000, function () {
//client.destroy();
});
Server
var net = require('net')
, server = net.createServer()
, delimit = '~~'
, clients = [];
controller.listenOutQueue(function (dataToSend) {
client.write(dataToSend);
client.write(delimit);
});
server.on('connection', function (socket) {
var chunk = '';
var dlim_index = -1;
socket.on('data', function (data) {
var recvData = data.toString().trim();
chunk += recvData;
dlim_index = chunk.indexOf(delimit);
while (dlim_index > -1) {
var useData = chunk.substring(0, dlim_index);
if (useData.substring(0, 9) == 'CLIENTID:') {
socket.clientid = useData.replace('CLIENTID:', '');
console.log('Client Id: ' + socket.clientid);
clients.push(socket);
var successMessage = "SUCCESS";
socket.write(successMessage);
socket.write(delimit);
}
else {
controller.insertIntoQueue(JSON.parse(useData), function (status) {
});
}
chunk = chunk.substring(dlim_index + 2);
dlim_index = chunk.indexOf(delimit);
}
});
socket.on('end', function () {
console.log('Connection Closed (' + socket.clientid + ')');
});
socket.on('error', function (err) {
console.log('SOCKET ERROR:', err);
});
});
server.listen(config.Port, config.IpAddress);

How to create a node js server socket to receive data in different ports?

I need a socket server that must receive data from 100 clientes in different ports on the server. I've created an array of server sockets and I don't know if this is correct.
I also identified a slowness to receive the data when I have about 100 clients sending data and for some reason the server stops receiving from 10 clients.
Here is my code for 3 ports as an example.
var net = require('net');
var rl = require('readline');
var fs = require('fs');
var ports = [60001, 60002, 60003];
var server = new Array();
ports.forEach(function(value) {
server[value] = net.createServer(function (socket) { // array of socket servers
socket.on('error', function(err) {
console.log(err.stack);
server[value].close();
server[value].listen(value); //start listen again
});
socket.on('end', function() {
console.log('client disconnected: ' + value);
server[value].close();
server[value].listen(value); //start listen again
});
console.log('client connected: ' + value);
var intf = rl.createInterface(socket, socket);
intf.on('line', function (line) {
fs.exists(__dirname + "\\file" + value.toString() + ".txt", function (exists) {
if(exists){
var stream = fs.createWriteStream(__dirname + "\\file" + value.toString() + ".txt", {'flags': 'a'});
} else {
var stream = fs.createWriteStream(__dirname + "\\file" + value.toString() + ".txt", {'flags': 'w'});
}
try {
stream.once('open', function(fd) {
console.log(value.toString() + " - " + line);
stream.write(line + "\r\n");
stream.end();
});
} catch (x) {
console.log(x.stack);
}
});
});
});
server[value].listen(value); // listen many ports
});

Resources