I have simple sample program for pub/sub pattern in nodejs which is as shown below
publisher.js
var zmq = require('zmq');
var pub = zmq.socket('pub');
pub.bind('tcp://127.0.0.1:6666');
console.log("Current 0MQ version is " + zmq.version);
var loop =0
pub.send(['t',loop++ + ' pub msg']);
pub.send(['t',loop++ + ' pub msg']);
subber.js
var zmq = require('zmq');
var sub = zmq.socket('sub');
sub.connect('tcp://127.0.0.1:6666');
sub.subscribe('t'); //herein lies the question
console.log('Received msg:');
sub.on('message',function(topic, msg){
console.log('Received msg:',msg.toString());
})
there are some unexpected behaviour which i am falling to understand from two days
1:for the first time when i run subber.js first and then
publisher.js i will get the messages as expected
2:Again if i run in the same order i am not getting any messages (subber.js and publisher.js)
3: for first i run publisher.js and then subber.js none of the messages i will get
really i am not understanding that sample program working or not please help me to understand publisher subscriber pattern of zmq with some sample code (**In the sample i have not used setinterval bcoz once the publisher send the message i have send to subscriber please provide the some sample to understand and accomplish this)
Thanks
When you start publisher first, publisher send msg (in nothing) so when subber starts, it can't get any msg.
First of all. You should use sync bind. Socket not created yet, but you already try to send something.
pub.bindSync('tcp://127.0.0.1:6666');
Nevertheless it didn't help me. I think bindSync work not sync (((. So I have to use timer to send something if application starts.
setTimeout( () => {
pub.send(['t',loop++ + ' pub msg']);
pub.send(['t',loop++ + ' pub msg']);
}, 1000);
Related
var express = require('express');
var app = express();
var server = app.listen(3000);
var replyFromBot;
app.use(express.static('public'));
var socket = require('socket.io');
var io = socket(server);
io.sockets.on('connection' , newConnection);
function newConnection(socket) {
console.log(socket.id);
listen = true;
socket.on('Quest' ,reply);
function reply(data) {
replyFromBot = bot.reply("local-user", data);
console.log(socket.id+ " "+replyFromBot);
socket.emit('Ans' , replyFromBot);
}
}
i've created a server based chat-bot application using node.js socket.io and express but the thing is for first time when i call socket.on it gets executed once and for 2nd time it gets executed twice for 3rd thrice and so on i've tackled this issue by setting a flag on my client so that it would display only once. i just wants to know is my code logically correct i mean is this a good code? because if the client ask a question for 10th time than listeners array will have 10+9+8....+1 listeners it would go on increasing depending upon number of questions clients asked. which is not good
i tried using removeListener it just removes listener once and it dosent call back for 2nd time. what do you guys recommend? do i go with this or is there any other way to add the listener when socket.on called and remove it when it gets executed and again add listener for the next time it gets called
thank-you.
client code:
function reply() {
socket.emit('Quest' , Quest);
flag = true;
audio.play();
socket.on('Ans', function(replyFromBot) {
if(flag) {
console.log("hi");
var para = document.createElement("p2");
x = document.getElementById("MiddleBox");
para.appendChild(document.createTextNode(replyFromBot));
x.appendChild(para);
x.scrollTop = x.scrollHeight;
flag = false;
}
});
}
The problem is caused by your client code. Each time you call the reply() function in the client you set up an additional socket.on('Ans', ...) event handler which means they accumulate. You can change that to socket.once() and it will remove itself each time after it get the Ans message. You can then also remove your flag variable.
function reply() {
socket.emit('Quest' , Quest);
audio.play();
// change this to .once()
socket.once('Ans', function(replyFromBot) {
console.log("hi");
var para = document.createElement("p2");
x = document.getElementById("MiddleBox");
para.appendChild(document.createTextNode(replyFromBot));
x.appendChild(para);
x.scrollTop = x.scrollHeight;
});
}
Socket.io is not really built as a request/response system which is what you are trying to use it as. An even better way to implement this would be to use the ack capability that socket.io has so you can get a direct response back to your Quest message you send.
You also need to fix your shared variables replyFromBot and listen on your server because those are concurrency problems waiting to happen as soon as you have multiple users using your server.
Better Solution
A better solution would be to use the ack capability that socket.io has to get a direct response to a message you sent. To do that, you'd change your server to this:
function newConnection(socket) {
console.log(socket.id);
socket.on('Quest', function(data, fn) {
let replyFromBot = bot.reply("local-user", data);
console.log(socket.id+ " "+replyFromBot);
// send ack response
fn(replyFromBot);
});
}
And, change your client code to this:
function reply() {
audio.play();
socket.emit('Quest', Quest, function(replyFromBot) {
console.log("hi");
var para = document.createElement("p2");
x = document.getElementById("MiddleBox");
para.appendChild(document.createTextNode(replyFromBot));
x.appendChild(para);
x.scrollTop = x.scrollHeight;
});
}
Doing it this way, you're hooking into a direct reply from the message so it works as request/response much better than the way you were doing it.
Instead of socket.on('Quest' ,reply); try socket.once('Quest' ,reply);
The bug in your code is that each time newConnection() is called node registers a event listener 'Quest'. So first time newConnection() is called the number of event listener with event 'Quest' is one, the second time function is called, number of event listener increases to two and so on
socket.once() ensures that number of event listener bound to socket with event 'Quest' registered is exactly one
I can't make following code to work:
"use strict";
let kafka = require('kafka-node');
var conf = require('./providers/Config');
let client = new kafka.Client(conf.kafka.connectionString, conf.kafka.clientName);
let consumer = new kafka.HighLevelConsumer(client, [ { topic: conf.kafka.readTopic } ], { groupId: conf.kafka.clientName, paused: true });
let threads = 0;
consumer.on('message', function(message) {
threads++;
if (threads > 10) consumer.pause();
if (threads > 50) process.exit(1);
console.log(threads + " >>> " + message.value);
});
consumer.resume();
I see 50 messages in console and process exits by termination statement.
What I'm trying to understand, is that is it my code broken or package broken? Or maybe I'm just doing something wrong? Does anyone was able to make kafka consumer work with pause/resume? I tried several versions of kafka-node, but all of them behave same way. Thanks!
You are already using pause and resume in your code, so obviously they work. ;)
It's because pause doesn't pause the consumption of messages. It pauses the fetching of messages. I'm guessing you already fetched the first 50 in one throw before you receive the first message and call pause.
For kicks, I just tested pause() and resume() in the Node REPL and they work as expected:
var kafka = require('kafka-node');
var client = new kafka.Client('localhost:2181');
var consumer = new kafka.HighLevelConsumer(client, [{topic: 'MyTest'}]);
consumer.on('message', (msg) => { console.log(JSON.stringify(msg)) });
Then I go into another window and run:
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic MyTest
And type some stuff, and it shows up in the first window. Then in the first window, I type: consumer.pause(); And type some more in the second window. Nothing appears in the first window. Then I run consumer.resume()in the first window, and the delayed messages appear.
BTW, you should be able to play with the Kafka config property fetch.message.max.bytes and control how many messages can be fetched at one time. For example, if you had fixed-width messages of 500 bytes, set fetch.message.max.bytes to something less than 1000 (but greater than 500!) to only receive a single message per fetch. But note that this might not fix the problem entirely -- I am fairly new to Node, but it is asynchronous, and I suspect a second fetch could get kicked off before you processed the first fetch completely (or at all).
I got this code for monitoring sockets in the zmq bindings for nodejs. So far it works but my problem is I dnt know what events the monitoring socket has. The code I got only did that, I will continue looking for more code but this is what I have so far..
``
var zmq = require('zmq');
var socket = zmq.socket('pub');
socket.connect('tcp://127.0.0.1:10001');
socket.monitor();
I tried adding an "onmessage" event handler but it showed nothing, so.. I dnt know whats up..
socket.on("message",function(msg){
console.log(msg);
});
I printed the object that I got back from the monitor() function and from it I was able to get some monitor events, I think it is unelegant though, I got this link that tests the monitor function of the socket ( https://github.com/JustinTulloss/zeromq.node/blob/master/test/socket.monitor.js ) but some things are not working but...
mon.monitor();
console.log(mon);
mon.on("message",function(msg){
console.log(msg);
});
mon.on('close',function(){console.log("Closed");});
mon._zmq.onMonitorEvent = function(evt){
if (evt == 1)
console.log("Should be 1 : "+ evt);
else
console.log(evt);
};
I haven't worked with the PUB/SUB handlers in 0mq. I have used some of the other types and am fairly familiar. Having not tested this code, my recommendation would be
SCRIPT 1: Your existing PUB script, needs to send a message
socket.send('TEST_MESSAGES', 'BLAH')
SCRIPT 2: This needs to be added:
var zmq = require('zmq');
var sub_socket = zmq.socket('sub');
sub_socket.connect('tcp://127.0.0.1:10001');
sub_socket.subscribe('TEST_MESSAGES')
sub_socket.on("message",function(msg){
console.log(msg);
});
The trick here is timing. 0mq doesn't give you retries or durable messages. You need to build those elements on your own. Still if you put your publish in a timer (for the sake of getting an example running) you should see the messages move.
Hello i am running a basic code for a chatserver on node.js. This is almost lifted from the book - Node:Up and Running. The problem is when a client types a message, his message is transmitted at each keystroke, than after a complete line and pressing enter, resulting in output like this -
client1: (Sends) Hello
Client2: (Recieves)127.0.0.1:50672>h 127.0.0.1:50672>e 127.0.0.1:50672>l 127.0.0.1:50672>l 127.0.0.1:50672>o
But This is how it SHOULD come
Client2: (Recieves) 127.0.0.1:50672>hello
What is happening here is messages are being transmitted at each keystroke, than after pressing enter. I lifted another example code from git by someone and same thing happened! here is my code :
var net = require ('net');
var chatServer = net.createServer(),
clientlist =[];
chatServer.on('connection',function(client){
client.name = client.remoteAddress+':'+client.remotePort;
client.write('Welcome '+ client.name + '!\n');
clientlist.push(client);
client.on('data',function(data){
broadcast(data,client);
});
});
function broadcast(message, client){
for(var i=0;i<clientlist.length;i+=1){
if(client !== clientlist[i]){
clientlist[i].write(client.name + ":"+ message);
}
}
}
console.log("Chatserver Started")
chatServer.listen(9000);
Here is the github example which yielded same results:
GITHUB
HOW TO CORRECT THIS BEHAVIOR?
The server event data is fired every time a block of data is received. Probably it fires for every character from the client.
The solution would be to create, in the server's connection callback, a buffer string/array. The data callback adds the received data to that buffer. When the data contains a newline character, the buffer is broadcasted and emptied.
I was using zeroMQ in nodeJS. But it seems that while sending the data from producer to worker, if I do not put it in setInterval, then it does not send the data to the worker. My example code is as follows:
producer.js
===========
var zmq = require('zmq')
, sock = zmq.socket('push');
sock.bindSync('tcp://127.0.0.1:3000');
console.log('Producer bound to port 3000');
//sock.send("hello");
var i = 0;
//1. var timer = setInterval(function() {
var str = "hello";
console.log('sending work', str, i++);
sock.send(str);
//2. clearTimeout(timer);
//3. }, 150);
sock.on('message', function(msg) {
console.log("Got A message, [%s], [%s]", msg);
});
So in the above code, if I add back the lines commented in 1, 2 and 3, then I do receive the message to the worker side, else it does not work.
Can anyone throw light why to send message I need to put it in setInterval? Or am I doing something wrong way?
The problem is hidden in the zmq bindings for node.js . I've just spent some time digging into it and it basically does this on send():
Enqueue the message
Flush buffers
Now the problem is in the flushing part, because it does
Check if the output socket is ready, otherwise return
Flush the enqueued messages
In your code, because you call bind and immediately send, there is no worker connected at the moment of the call, because they simply didn't have enough time to notice. So the message is enqueued and we are waiting for some workers to appear. Now the interesting part - where do we check for new workers? In the send function itself! So unless we call send() later, when there are actually some workers connected, our messages are never flushed and they are enqueued forever. And that is why setInterval works, because workers have enough time to notice and connect and you periodically check if there are any.
You can find the interesting part at https://github.com/JustinTulloss/zeromq.node/blob/master/lib/index.js#L277 .
Cheers ;-)