I want to send the same messages many times in a row, but i need to use a loop. When I use a loop though, no messages are sent. I am using amqp in Nodejs.
Here is the working code for sending a single messages. What should I do to send many. I have already tried just wrapping a while loop around the connection.publish part and nothing was sent.
var amqp = require('amqp');
var connection = amqp.createConnection({url: "amqp://tester:tstpsswrd#10.4.52.115:5672"});
connection.on('ready', function () {
connection.queue('my-queue', function (q) {
connection.publish('my-queue', 'hi');
});
});
I'm positive that I am doing something stupid wrong here, or maybe missing something. First time with rabbitmq.
Update, Loop example
var amqp = require('amqp');
var connection = amqp.createConnection({url: "amqp://tester:tstpsswrd#10.4.52.115:5672"});
connection.on('ready', function () {
connection.queue('my-queue', function (q) {
while(true){
connection.publish('my-queue', 'hi');
}
});
});
In practical scenario you can not and should not be having a infinite loop as such for writing to a message broker. There have to be some event based thing or a proper defined number.
Try this code you can use the for loop according to your requirement:
var amqp = require('amqp');
var connection = amqp.createConnection({ host: 'localhost', port: 5672});
connection.on('ready', function () {
for(var i=0; i<1000; i++){
var status = writeOnQueue("testing the queue"+i);
}
});
function writeOnQueue(xml){
var msg = xml;
console.log(msg);
try{
connection.exchange('test-exchange', {confirm: true},function(exchange) {
publish = exchange.publish('my-queue',msg, { mandatory: false });
console.log('sent the message success test-exchange');
return true;
});
}
catch(e){
console.log('Some error occured.'+ e);
}
}
Related
I'm currently having problems figuring out how to capture my MQTT message event back to my REST API body which is written in NodeJS. My current setup is App -> NodeJS REST API -> MQTT broker inside RPi 3.
This is my MQTTHandler.js class where I have put all my reusable MQTT functions
const mqtt = require('mqtt')
class MQTTHandler {
constructor (host) {
this.client = null
this.host = host
}
connect () {
this.client = mqtt.connect(this.host)
this.client.on('error', function (err) {
console.log(err)
this.client.end()
})
this.client.on('connect', function () {
console.log('MQTT client connected...')
})
// I need this to send message back to app.js
this.client.on('message', function (topic, message) {
if (!message.toString()) message = 'null'
console.log(JSON.parse(message.toString()))
})
this.client.on('close', function () {
console.log('MQTT client disconnected...')
})
}
subscribeTopic (topic) {
this.client.subscribe(topic)
}
unsubscribeTopic (topic) {
this.client.unsubscribe(topic)
}
sendMessage (topic, message) {
this.client.publish(topic, message)
}
}
module.exports = MQTTHandler
And below is a short snippet of my app.js
const MQTTHandler = require('./mqtt.handler')
...
var mqttClient = new MQTTHandler('mqtt://127.0.0.1')
mqttClient.connect()
app.get('/hello', function (req, res) {
mqttClient.subscribeTopic('topic')
mqttClient.sendMessage('topic', 'hello world')
// I need to return the MQTT message event here
// res.json(<mqtt message here>)
res.end()
})
I have already tried using NodeJS' event emitter but it doesn't seem to work. Any help or suggestions would be much appreciated, thank you!
You are trying to mix a synchronous protocol (HTTP) with and asynchronous protocol (MQTT). These 2 paradigm don't easily mix.
When you publish an MQTT message you have no idea how many clients may be subscribed to that topic, it could be zero, it could be many. There is also no guarantee that any of them will send a reply so you need to include a timeout. (You also need to include a request id in the payload so you can coordinate any response with the request as you can't say anything about what order responses may come in.)
Your example code is only using 1 topic, this is very bad as you will end up needing to filter out request messages from response messages. Better to use 2 different topics (MQTT v5 even has a msg header to specify the topic the response should be sent on).
Having said all that it is possible to build something that will work (I will use request and reply topics.
var inflightRequests = {};
// interval to clear out requests waiting for a response
// after 3 seconds
var timer = setInterval(function() {
var now = new Date.now();
var keys = Object.keys(inflightRequests);
for (var key in keys) {
var waiting = inflightRequests[keys[key]];
var diff = now = waiting.timestamp;
// 3 second timeout
if (diff > 3000) {
waiting.resp.status(408).send({});
delete(inflightRequests[keys[key]]);
}
}
},500);
// on message handler to reply to the HTTP request
client.on('message', function(topic, msg){
if (topic.equals('reply')) {
var payload = JSON.parse(msg);
var waiting = inflightRequest[payload.requestId];
if (waiting) {
waiting.res.send(payload.body);
delete(inflightRequests[payload.requestId]);
} else {
// response arrived too late
}
}
});
// HTTP route handler.
app.get('/hello', function(req, res) {
//using timestamp as request Id as don't have anything better in this example.
var reqId = Date.now();
var waiting = {
timeStamp: reqId,
res: res
}
inflightRequests[reqId] = waiting;
var message = {
requestId: reqId,
payload: 'hello world'
}
client.publish('request',JSON.stringify(message));
});
I am making use of "socket.io-client" and "socket.io stream" to make a request and then stream some data. I have the following code that handles this logic
Client Server Logic
router.get('/writeData', function(req, res) {
var io = req.app.get('socketio');
var nameNodeSocket = io.connect(NAMENODE_ADDRESS, { reconnect: true });
var nameNodeData = {};
async.waterfall([
checkForDataNodes,
readFileFromS3
], function(err, result) {
if (err !== null) {
res.json(err);
}else{
res.json("Finished Writing to DN's");
}
});
function checkForDataNodes(cb) {
nameNodeSocket.on('nameNodeData', function(data) {
nameNodeData = data;
console.log(nameNodeData);
cb(null, nameNodeData);
});
if (nameNodeData.numDataNodes === 0) {
cb("No datanodes found");
}
}
function readFileFromS3(nameNodeData, cb) {
for (var i in nameNodeData['blockToDataNodes']) {
var IP = nameNodeData['blockToDataNodes'][i]['ipValue'];
var dataNodeSocket = io.connect('http://'+ IP +":5000");
var ss = require("socket.io-stream");
var stream = ss.createStream();
var byteStartRange = nameNodeData['blockToDataNodes'][i]['byteStart'];
var byteStopRange = nameNodeData['blockToDataNodes'][i]['byteStop'];
paramsWithRange['Range'] = "bytes=" + byteStartRange.toString() + "-" + byteStopRange.toString();
//var file = require('fs').createWriteStream('testFile' + i + '.txt');
var getFileName = nameNodeData['blockToDataNodes'][i]['key'].split('/');
var fileData = {
'mainFile': paramsWithRange['Key'].split('/')[1],
'blockName': getFileName[1]
};
ss(dataNodeSocket).emit('sendData', stream, fileData);
s3.getObject(paramsWithRange).createReadStream().pipe(stream);
//dataNodeSocket.disconnect();
}
cb(null);
}
});
Server Logic (that gets the data)
var dataNodeIO = require('socket.io')(server);
var ss = require("socket.io-stream");
dataNodeIO.on('connection', function(socket) {
console.log("Succesfully connected!");
ss(socket).on('sendData', function(stream, data) {
var IP = data['ipValue'];
var blockName = data['blockName'];
var mainFile = data['mainFile'];
dataNode.makeDir(mainFile);
dataNode.addToReport(mainFile, blockName);
stream.pipe(fs.createWriteStream(mainFile + '/' + blockName));
});
});
How can I properly disconnect the connections in function readFileFromS3. I have noticed using dataNodeSocket.disconnect() at the end does not work as I cannot verify the data was received on the 2nd server. But if I comment it out, I can see the data being streamed to the second server.
My objective is to close the connections in Client Server side
It appears that the main problem with closing the socket is that you weren't waiting for the stream to be done writing before trying to close the socket. So, because the writing is all asynchronous and finishes sometime later, you were trying to close the socket before the data had been written.
Also because you were putting asynchronous operations inside a for loop, you were also running all your operations in parallel which may not be exactly what you want as it makes error handling more difficult and server load more difficult.
Here's the code I would suggest that does the following:
Create a function streamFileFromS3() that streams a single file and returns a promise that will notify when it's done.
Use await in a for loop with that streamFileFromS3() to serialize the operations. You don't have to serialize them, but then you would have to change your error handling to figure out what to do if one errors while the others are already running and you'd have to be more careful about concurrency issues.
Use try/catch to catch any errors from streamFileFromS3().
Add error handling on the stream.
Change all occurrences of data['propertyName'] to data.propertyName. The only time you need to use brackets is if the property name contains a character that is not allowed in a Javascript identifier or if the property name is in a variable. Otherwise, the dot notation is preferred.
Add socket.io connection error handling logic for both socket.io connections.
Set returned status to 500 when there's an error processing the request
So, here's the code for that:
const ss = require("socket.io-stream");
router.get('/writeData', function(req, res) {
const io = req.app.get('socketio');
function streamFileFromS3(ip, data) {
return new Promise((resolve, reject) => {
const dataNodeSocket = io.connect(`http://${ip}:5000`);
dataNodeSocket.on('connect_error', reject);
dataNodeSocket.on('connect_timeout', () {
reject(new Error(`timeout connecting to http://${ip}:5000`));
});
dataNodeSocket.on('connection', () => {
// dataNodeSocket connected now
const stream = ss.createStream().on('error', reject);
paramsWithRange.Range = `bytes=${data.byteStart}-${data.byteStop}`;
const filename = data.key.split('/')[1];
const fileData = {
'mainFile': paramsWithRange.Key.split('/')[1],
'blockName': filename
};
ss(dataNodeSocket).emit('sendData', stream, fileData);
// get S3 data and pipe it to the socket.io stream
s3.getObject(paramsWithRange).createReadStream().on('error', reject).pipe(stream);
stream.on('close', () => {
dataNodeSocket.disconnect();
resolve();
});
});
});
}
function connectError(msg) {
res.status(500).send(`Error connecting to ${NAMENODE_ADDRESS}`);
}
const nameNodeSocket = io.connect(NAMENODE_ADDRESS, { reconnect: true });
nameNodeSocket.on('connect_error', connectError).on('connect_timeout', connectError);
nameNodeSocket.on('nameNodeData', async (nameNodeData) => {
try {
for (let item of nameNodeData.blockToDataNodes) {
await streamFileFromS3(item.ipValue, item);
}
res.json("Finished Writing to DN's");
} catch(e) {
res.status(500).json(e);
}
});
});
Other notes:
I don't know what paramsWithRange is as it is not declared here and when you were doing everything in parallel, it was getting shared among all the connections which is asking for a concurrency issue. In my serialized implementation, it's probably safe to share it, but the way it is now bothers me as it's a concurrency issue waiting to happen.
Trying to write a TCP client in Node v0.10.15 and I am having a little trouble getting data back from the server. I know that the server is working properly because I have 3-4 different clients written in different languages communicating with it.
Below is a snippet of a larger piece of code but this should get the point across.
The problem is: I'm expecting 2 packets coming back after writing to the socket (this part is not included in this example). I'm only seeing the "data" event being fired once. Is there something that I need to do to get node to resume reading from the Tcp stream? I can confirm that the server is sending 2 packets(The length and then the actual data) Any help would be appreciated.
var dc = require('./DataContracts.js');
var net = require('net');
require('buffertools').extend();
var client = net.Socket();
var isConnected = false;
var serverHost = '10.2.2.21';
var dataCallback;
var receivedBuffer = new Array();
function InitComm(buffer) {
if (!isConnected) {
client.connect(4987, serverHost, function() {
client.on('data', function(data) {
console.log('Received server packet...');
var buf = new Buffer(data);
receivedBuffer.push(buf);
client.resume();
});
client.on('end', function() {
if (receivedBuffer.length > 1) {
if (dataCallback !== undefined)
dataCallback(receivedBuffer);
}
});
client.on('close', function() {
//clean up
});
client.on('error', function(err) {
console.log('Error!: ' + err);
});
Communicate(buffer);
});
} else {
Communicate(buffer);
}
}
Turns out that node was combining both of the packets together. I must be missing a Carriage return on the first packet
Pretty simple node.js question. I want to extend the stream object to re-chunk the data that is coming in from a remote connection. I'm doing multiple telnets and sending commands to other servers, and they send back responses. It looks something like this.
> Hello, this is a command
This is the response to the command.
Sometimes it pauses here (which triggers the 'data' event prematurely).
But the message isn't over until you see the semicolon
;
What I'd like to do is instead of triggering the 'data' event at the pause, is wait for the ; and trigger a custom 'message' event.
I've read and reread this question, but I don't quite get it yet (partially because it's about a writable stream, and partially because I don't yet grok CoffeeScript).
EDIT: I guess I'm asking two things here:
How do I extend/inherit the stream object that net.CreateConnection uses?
Can I just extend the prototype.write to do a 'split' and re-'emit' each part?
Here's a snip of what I'm doing so far, but the chunking should be part of the stream, not part of the 'data' listener:
var net = require('net');
var nodes = [
//list of ip addresses
];
function connectToServer(ip) {
var conn = net.createConnection(3083, ip);
conn.on('connect', function() {
conn.write ("login command;");
});
conn.on('data', function(data) {
var read = data.toString();
var message_list = read.split(/^;/m);
message_list.forEach (function(message) {
console.log("Atonomous message from " + ip + ':' + message);
//I need to extend the stream object to emit these instead of handling it here
//Also, sometimes the data chunking breaks the messages in two,
//but it should really wait for a line beginning with a ; before it emits.
});
});
conn.on('end', function() {
console.log("Lost conncection to " + ip + "!!");
});
conn.on('error', function(err) {
console.log("Connection error: " + err + " for ip " + ip);
});
}
nodes.forEach(function(node) {
connectToServer(node);
});
If I was using a raw stream, I guess it would be something like this (based on code I found elsewhere)?
var messageChunk = function () {
this.readable = true;
this.writable = true;
};
require("util").inherits(messageChunk, require("stream"));
messageChunk.prototype._transform = function (data) {
var regex = /^;/m;
var cold_storage = '';
if (regex.test(data))
{
var message_list = read.split(/^;/m);
message_list.forEach (function(message) {
this.emit("data", message);
});
}
else
{
//somehow store the data until data with a /^;/ comes in.
}
}
messageChunk.prototype.write = function () {
this._transform.apply(this, arguments);
};
But I'm not using a raw stream, I'm using the stream object in the net.createConnection object returns.
Don't use the _transform,_read,_write,or _flush functions you implement directly, those are for the internals of node to use.
Emit a custom event when you see the character ";" in your stream:
var msg = "";
conn.on("data",function(data) {
var chunk = data.toString();
msg += chunk;
if(chunk.search(";") != -1) {
conn.emit("customEvent",msg);
msg = "";
}
});
conn.on("customEvent",function(msg) {
//do something with your message
});
With node.js, I'm trying to send the current server_time to all clients in every second.
Therefore, I wanted to use setInterval() to emit an event to all clients and sending the time, but it doesn't work. Did I define the setInterval function at the right place or did missed something else?
var http = require("http");
var socketIO = require('socket.io');
var connect = require('connect');
//keep track of every connected client
var clients = {};
//create Server
var httpServer = connect.createServer(
connect.static(__dirname)
).listen(8888);
//socket
var io = socketIO.listen(httpServer);
io.sockets.on('connection', function (socket) {
//add current client id to array
clients[socket.id] = socket;
socket.on('close', function() {
delete clients[socket.fd]; // remove the client.
});
//send news on connection to client
socket.emit('news', { hello: 'world' });
//this one works fine!
//send server time on connection to client
socket.emit("server_time", { time: new Date().toString() });
});
//this doesn't work!
// Write the time to all clients every second.
setInterval(function() {
var i, sock;
for (i in clients) {
sock = clients[i];
if (sock.writable) { // in case it closed while we are iterating.
sock.emit("server_time", {
console.log("server_time sended");
time: new Date().toString()
});
}
}
}, 1000); //every second
May I suggest a workaround/improvement that should fix the problem. Add the clients to a chat room. Somewhere in:
io.sockets.on('connection', function (socket) {
add a
socket.join('timer');
Then the setIntervall would be
setInterval(function() {
io.sockets.in('timer').emit("server_time", { time: new Date().toString() })
}, 1000);
Hope this works for you!
The problem is the following function:
if (sock.writable) { // in case it closed while we are iterating.
sock.emit("server_time", {
// console.log("server_time sended"); // get rid of this line -> invalid code
time: new Date().toString()
});
}
sock.writable is undefined and therefore the emit event is never sent. Set the property to true on connection and to false on close.