how to minimize the massive message delay paho mqtt - python-3.x

I have seven raspberry pi's as clients a and one raspberry pi as client b which are transferring messages between each other via mqtt. It's working well so far, but there is a massive delay of over 2s sometimes. Most of the times the the delay is arround 0,02-0,2s but sometimes there are some delay-peaks with over 2s of delay.
I think it appears as soon as client b gets too many messages at the same time from the a clients.
Has anybody of you guys an idea what's the reason for this and how I can solve this?
Thanks and stay healthy!
When the run() is called, client a publishes to client b
client a
from datetime import datetime
import time
import paho.mqtt.client as mqtt
class CarClient:
def __init__(self, id):
self.client = mqtt.Client(id)
self.client.connect("192.168.178.41", 1883, 60) # connect to broker
self.newSpeed = 0
self.holdSpeedTime = 5
self.speedDuration = 0
self.id = id
self.time_sent = 0
def on_connect(self, mqttc, obj, flags, rc):
print("connected" + str(rc))
def on_message(self, mqttc, obj, msg):
answerTuple = eval(msg.payload)
self.newSpeed = int(answerTuple[0])
self.holdSpeedTime = answerTuple[1]
self.client.unsubscribe("test_channel")
time_feedback = time.time()
delay = time_feedback - self.time_sent
print(str(delay))
def on_disconnect(self, mqttc, flags, rc):
print("disconnected" + str(rc))
def on_publish(self, mqttc, obj, mid):
print("Data published " + str(mid))
print("published")
def on_subscribe(self, mqttc, obj, mid, granted_qos):
print("Subscribed: " + str(mid) + " " + str(granted_qos))
def on_unsubscribe(self, mqttc, flags, rc):
print("unsubscribed" + str(rc))
def on_log(self, mqttc, obj, level, string):
print(string)
def run(self, carID, connectionPoint):
self.time_sent = time.time()
self.tuple = (carID, connectionPoint)
# assining functions to callback
self.client.on_disconnect = self.on_disconnect
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
# self.client.on_subscribe = self.on_subscribe
# self.client.on_unsubscribe = self.on_unsubscribe
self.client.loop_start()
self.client.subscribe(str(self.id), 0) # subscribe topic
self.client.publish("test_channel1", str(self.tuple))
time.sleep(1)
carFeedback = (self.newSpeed, self.holdSpeedTime)
self.newSpeed = 0
self.holdSpeedTime = 5
return carFeedback
client b
import paho.mqtt.client as mqtt
import time
crossingList = []
def on_subscribe(client, userdata, mid, granted_qos): # create function for callback
print("subscribed with qos", granted_qos, "\n")
def on_disconnect(client, userdata, rc=0):
print("disconnected")
def on_connect(client, userdata, flags, rc):
print("connected")
def on_message(client, userdata, message):
dataTuple = eval(message.payload) # message is a tuple of (carID, connectionPoint)
print(str(dataTuple))
addToList(dataTuple)
def on_publish(client, userdata, mid):
print("published")
def addToList(dataTuple):
carID = dataTuple[0]
connectionPoint = dataTuple[1]
crossingList.append((carID, connectionPoint))
def logic():
carID = crossingList.__getitem__(0)[0]
connectionPoint = crossingList.__getitem__(0)[1]
if connectionPoint == 600:
newSpeed = 400
holdTime = 5
client.publish( carID, str((newSpeed, holdTime)))
del crossingList[0]
else:
newSpeed = 500
holdTime = 3
client.publish(carID, str((newSpeed, holdTime)))
del crossingList[0]
client = mqtt.Client("Logic") # create client object
client.on_subscribe = on_subscribe # assign function to callback
client.on_disconnect = on_disconnect # assign function to callback
client.on_connect = on_connect # assign function to callback
client.on_message = on_message
client.connect("192.168.178.41", 1883, 60) # establish connection
time.sleep(0.1)
client.loop_start()
client.subscribe("test_channel1")
count = 1
while True: # runs forever break with CTRL+C
if len(crossingList) != 0:
logic()
else:
count = count + 1

Related

How to process many messages in parallel in mqtt

I want to receive messages from many sensors (in the code I've tried from two) and process these messages in parallel by sending a command to the lights at the same time if the two sensors return "1". So I've done that in a separate thread but it doesn't work. It stops sending to the bedroom once someone enters the kitchen!
from class_light import Light
from test_class import Mqtt
import paho.mqtt.client as mqtt
import sys
import threading
import time
from MongoDB import is_circadian_Light
Broker = "broker.mqttdashboard.com"
username = "growthtechnology"
password = "growthtechnology"
PortaBroker = 1883
KeepAliveBroker = 60
client_name = "Local"
topic_sub = "testtopic/1"
topic_pub = "testtopic/3"
light = Mqtt()
occupancy_sensor = [("testtopic/4", 0), ("testtopic/5", 0)]
tunable_light = Light("deviceID")
def on_connect(client, userdata, flags, rc):
print("Connection to the broker. Result : "+str(rc))
client.subscribe(topic_sub)
client.subscribe(occupancy_sensor)
def on_publish(client,userdata,result):
print("Message published")
def on_disconnect(client, userdata, rc):
if rc != 0:
print("Unexpected disconnection.")
try:
print("[STATUS] Inicializando MQTT...")
#inicializing MQTT:
client = mqtt.Client()
client.username_pw_set(username, password)
client.on_connect = on_connect
client.on_message = light.message()
client.on_disconnect = on_disconnect
client.on_publish = on_publish
client.connect(Broker, PortaBroker, KeepAliveBroker)
def publishing():
while True:
msg = light.messages # calling for the variable "messages" to get PUBLISH messages
print(msg)
time.sleep(3)
topic = light.topic
if msg == "1" and topic == "testtopic/4":
if is_circadian_Light("device2") == 1 :
client.publish("testtopic/Bedroom",tunable_light.circadianLight())
time.sleep(10)
def publishing_kitchen():
while True:
msg_kitchen = light.messages
topic = light.topic
if msg_kitchen == "1" and topic == "testtopic/5":
if is_circadian_Light("device2") == 1 :
client.publish("testtopic/Kitchen",tunable_light.circadianLight())
time.sleep(10)
#code to turn off lights after 1 min
pub = threading.Thread(target= publishing)
pub.start()
pub_k = threading.Thread(target= publishing_kitchen)
pub_k.start()
client.loop_forever()
except KeyboardInterrupt:
print ("\nCtrl+C pressionado, encerrando aplicacao e saindo...")
sys.exit(0)
This the class Mqtt that I have created:
import paho.mqtt.client as mqtt
import time
class Mqtt():
def __init__(self, messages = "", topic = "", topic_1 = ""):
self._messages = messages
self._topic = topic
#property
def messages(self):
return self._messages
#messages.setter
def messages(self, messages):
self._messages = messages
#property
def topic(self):
return self._topic
#topic.setter
def topic(self, topic):
self._topic = topic
def message(self):
def on_message(client, userdata, msg):
Message = str(msg.payload.decode("utf-8"))
self._messages = Message
if msg.topic == "testtopic/4":
self._topic = msg.topic
print("[Receiving msg] Topic:" +msg.topic+" / Message: "+Message)
return on_message

RabbitMQ Pika connection reset , (-1, ConnectionResetError(104, 'Connection reset by peer'))

searched through stackoverflow and posting this question because no solution worked for me and my question might be different from other question.
I am writing a script which gets an article from rabbitMQ queue and process the article to count words and extract key words from it and dump it in db. my script is working fine but after some time of execution i get this exception
(-1, "ConnectionResetError(104, 'Connection reset by peer')")
I have no idea why am I getting this. I have tried a lot of solutions available on stackover flow none is working for me. I havr written my script and tried it in two different ways. both work fine but after some time same exception occurs.
here is my first code:
def app_main():
global channel, results, speedvars
Logger.log_message('Starting app main')
# Edit 4
def pika_connect():
connection = pika.BlockingConnection(pika.ConnectionParameters(
host=Config.AMQ_DAEMONS['base']['amq-host']))
channel = connection.channel()
print ("In pika connect")
Logger.log_message('Setting up input queue consumer')
channel.queue_declare(Config.AMQ_DAEMONS['consumer']['input'], durable=True)
channel.basic_consume(on_message, queue=Config.AMQ_DAEMONS['consumer']['input'], no_ack=True)
Logger.log_message('Starting loop')
channel.start_consuming()
#########
speedvars = SpeedVars()
speedtracker = SpeedTracker(speedvars)
speedtracker.start()
sender = ResultsSender(results, speedvars)
sender.start()
# Edit 5 starting 10 threads to listen to pika
for th in range(qthreads):
Logger.log_message('Starting thread: '+str(th))
try:
t = Thread(target=pika_connect, args=())
t.start()
except Exception as e:
Logger.error_message("Exception in starting threads " + str(e))
try:
app_main()
except Exception as e:
Logger.error_message("Exception in APP MAIN " + str(e))
here is my second code:
def app_main():
global channel, results, speedvars
Logger.log_message('Starting app main')
speedvars = SpeedVars()
speedtracker = SpeedTracker(speedvars)
speedtracker.start()
sender = ResultsSender(results, speedvars)
sender.start()
connection = pika.BlockingConnection(pika.ConnectionParameters(
host=Config.AMQ_DAEMONS['base']['amq-host']))
channel = connection.channel()
print ("In app main")
Logger.log_message('Setting up input queue consumer')
channel.queue_declare(Config.AMQ_DAEMONS['consumer']['input'], durable=True)
channel.basic_consume(on_message, queue=Config.AMQ_DAEMONS['consumer']['input'], no_ack=True)
Logger.log_message('Starting loop')
try:
channel.start_consuming()
except Exception as e:
Logger.error_message("Exception in start_consuming in main " + str(e))
raise e
try:
app_main()
except Exception as e:
Logger.error_message("Exception in APP MAIN " + str(e))
in my first code i used threading because i want to speed up the process of processing articles.
this is my call back fuction
def on_message(ch, method, properties, message):
Logger.log_message("Starting parsing new msg ")
handle_message(message)
EDIT: Full Code
import os
abspath = os.path.abspath(__file__)
dname = os.path.dirname(abspath)
os.chdir(dname)
from Modules import Logger
import pika
import Config
import json
import pickle
import Pipeline
import sys
import time
import datetime
import threading
import queue
import functools
from pid.decorator import pidfile
Logger.log_init(Config.AMQ_DAEMONS['consumer']['log-ident'])
#qthreads = Config.AMQ_DAEMONS['consumer']['threads']
results = queue.Queue()
channel = None
speedvars = None
SPD_RECEIVED = 'received'
SPD_DISCARDED = 'discarded'
SPD_SENT = 'sent'
class SpeedVars(object):
vars = {}
lock = None
def __init__(self):
self.lock = threading.Lock()
def inc(self, var):
self.lock.acquire()
try:
if var in self.vars:
self.vars[var] += 1
else:
self.vars[var] = 1
finally:
self.lock.release()
def dec(self, var):
self.lock.acquire()
try:
if var in self.vars:
self.vars[var] -= 1
else:
Logger.error_message('Cannot decrement ' + var + ', not tracked')
finally:
self.lock.release()
def get(self, var):
out = None
self.lock.acquire()
try:
if var in self.vars:
out = self.vars[var]
else:
Logger.error_message('Cannot get ' + var + ', not tracked')
finally:
self.lock.release()
return out
def get_all(self):
out = None
self.lock.acquire()
try:
out = self.vars.copy()
finally:
self.lock.release()
return out
class SpeedTracker(threading.Thread):
speedvars = None
start_ts = None
last_vars = {}
def __init__(self, speedvars):
super(SpeedTracker, self).__init__()
self.start_ts = time.time()
self.speedvars = speedvars
Logger.log_message('Setting up speed tracker')
def run(self):
while True:
time.sleep(Config.AMQ_DAEMONS['consumer']['speed-tracking-interval'])
prev = self.last_vars
cur = self.speedvars.get_all()
now = time.time()
if len(prev) > 0:
q = {}
for key in cur:
qty = cur[key] - prev[key]
avg = qty / Config.AMQ_DAEMONS['consumer']['speed-tracking-interval']
overall_avg = cur[key] / (now - self.start_ts)
Logger.log_message('Speed-tracking (' + key + '): total ' + str(cur[key])
+ ', delta ' + str(qty) + ', speed ' + '%0.2f' % avg + '/sec, '
+ ', overall speed ' + '%0.2f' % overall_avg + '/sec')
pending = cur[SPD_RECEIVED] - cur[SPD_DISCARDED] - cur[SPD_SENT]
pending_avg = pending / (now - self.start_ts)
Logger.log_message('Speed-tracking (pending): total ' + str(pending)
+ ', overall speed ' + '%0.2f' % pending_avg + '/sec')
self.last_vars = cur
class ResultsSender(threading.Thread):
channel = None
results = None
speedvars = None
def __init__(self, results, speedvars):
super(ResultsSender, self).__init__()
connection = pika.BlockingConnection(pika.ConnectionParameters(
host=Config.AMQ_DAEMONS['base']['amq-host']))
self.channel = connection.channel()
Logger.log_message('Setting up output exchange')
self.channel.exchange_declare(exchange=Config.AMQ_DAEMONS['consumer']['output'], exchange_type='direct')
self.results = results
self.speedvars = speedvars
def run(self):
while True:
item = self.results.get()
self.channel.basic_publish(
exchange=Config.AMQ_DAEMONS['consumer']['output'],
routing_key='',
body=item)
self.speedvars.inc(SPD_SENT)
def parse_message(message):
try:
bodytxt = message.decode('UTF-8')
body = json.loads(bodytxt)
return body
except Exception as e:
Logger.error_message("Cannot parse message - " + str(e))
raise e
def get_body_elements(body):
try:
artid = str(body.get('article_id'))
article_dt = datetime.datetime.fromtimestamp(body.get('pubTime'))
date = article_dt.strftime(Config.DATE_FORMAT)
article = "\n".join([body.get('title', ''), body.get('subheading', ''), body.get('content', '')])
return (artid, date, article)
except Exception as e:
Logger.error_message("Cannot retrieve article attributes " + str(e))
raise e
def process_article(id, date, text):
global results, speedvars
try:
Logger.log_message('Processing article ' + id)
keywords = Pipeline.extract_keywords(text)
send_data = {"id": id, "date": date, "keywords": keywords}
results.put(pickle.dumps(send_data))
# print('Queue Size:',results.qsize())
except Exception as e:
Logger.error_message("Problem processing article " + str(e))
raise e
def ack_message(ch, delivery_tag):
"""Note that `channel` must be the same pika channel instance via which
the message being ACKed was retrieved (AMQP protocol constraint).
"""
if channel.is_open:
channel.basic_ack(delivery_tag)
else:
Logger.error_message("Channel is already closed, so we can't ACK this message" + str(e))
# Channel is already closed, so we can't ACK this message;
# log and/or do something that makes sense for your app in this case.
#pass
def handle_message(connection, ch, delivery_tag, message):
global speedvars
start = time.time()
thread_id = threading.get_ident()
try:
speedvars.inc(SPD_RECEIVED)
body = parse_message(message)
(id, date, text) = get_body_elements(body)
words = len(text.split())
if words <= Config.AMQ_DAEMONS['consumer']['word-count-limit']:
process_article(id, date, text)
else:
Logger.log_message('Ignoring article, over word count limit')
speedvars.inc(SPD_DISCARDED)
except Exception as e:
Logger.error_message("Could not process message - " + str(e))
cb = functools.partial(ack_message, ch, delivery_tag)
connection.add_callback_threadsafe(cb)
Logger.log_message("Thread id: "+str(thread_id)+" Delivery tag: "+str(delivery_tag))
Logger.log_message("TOtal time taken to handle message : "+ str(time.time()-start))
# CALL BACK
## def on_message(ch, method, properties, message):
## global executor
## executor.submit(handle_message, message)
def on_message(ch, method, header_frame, message, args):
(connection, threads) = args
delivery_tag = method.delivery_tag
t = threading.Thread(target=handle_message, args=(connection, ch, delivery_tag, message))
t.start()
threads.append(t)
####################################################
#pidfile(piddir=Config.AMQ_DAEMONS['base']['pid-dir'], pidname=Config.AMQ_DAEMONS['consumer']['pid-file'])
def app_main():
global channel, results, speedvars
speedvars = SpeedVars()
speedtracker = SpeedTracker(speedvars)
speedtracker.start()
sender = ResultsSender(results, speedvars)
sender.start()
# Pika Connection
connection = pika.BlockingConnection(pika.ConnectionParameters(
host=Config.AMQ_DAEMONS['base']['amq-host']))
channel = connection.channel()
Logger.log_message('Setting up input queue consumer')
channel.queue_declare(Config.AMQ_DAEMONS['consumer']['input'], durable=True)
#channel.basic_consume(on_message, queue=Config.AMQ_DAEMONS['consumer']['input'], no_ack=True)
channel.basic_qos(prefetch_count=1)
threads = []
on_message_callback = functools.partial(on_message, args=(connection, threads))
channel.basic_consume(on_message_callback, Config.AMQ_DAEMONS['consumer']['input'])
Logger.log_message('Starting loop')
## channel.start_consuming()
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
Wait for all to complete
for thread in threads:
thread.join()
connection.close()
app_main()
pika is not taking a lot of time to process message still i am facing connection reset issue.
**TOtal time taken to handle message : 0.0005991458892822266
**
Your handle_message method is blocking heartbeats because all of your code, including the Pika I/O loop, is running on the same thread. Check out this example of how to run your work (handle_message) on a separate thread from Pikas I/O loop and then acknowledge messages correctly.
NOTE: the RabbitMQ team monitors the rabbitmq-users mailing list and only sometimes answers questions on StackOverflow.
I was getting the same issue . Increasing the duration of heart-beat & connection timeouts configuration didn't work out for me. I finally figured out that, if you have
already created a channel and you are not publishing anything on it for
several minutes(20 mins in my case) ,in that case we get this error.
The Solutions which worked for me:
Create channel immediately just before publishing any message. OR
Use try-except and if you get an exception , create another channel and republish. ie.
try:
channel.basic_publish(exchange='', routing_key='abcd', body=data)
except Exception as e1:
connection=pika.BlockingConnection(pika.ConnectionParameters(host='1.128.0.3',credentials=credentials))
channel = connection.channel()
channel.basic_publish(exchange='', routing_key='abcd', body=data)
This will atleast keep the things running and prevent from losing any data. I'm not an expert in this, but hope this helps someone!
I also faced the same issue and resolved by increasing the duration for heart-beat & connection timeouts configuration.
Many thanks to #LukeBakken who has actually identified the root cause.
Here is how you can configure the timeouts:
import pika
def main():
# NOTE: These parameters work with all Pika connection types
params = pika.ConnectionParameters(heartbeat=600, blocked_connection_timeout=300)
conn = pika.BlockingConnection(params)
chan = conn.channel()
chan.basic_publish('', 'my-alphabet-queue', "abc")
# If publish causes the connection to become blocked, then this conn.close()
# would hang until the connection is unblocked, if ever. However, the
# blocked_connection_timeout connection parameter would interrupt the wait,
# resulting in ConnectionClosed exception from BlockingConnection (or the
# on_connection_closed callback call in an asynchronous adapter)
conn.close()
if __name__ == '__main__':
main()
Reference: https://pika.readthedocs.io/en/stable/examples/heartbeat_and_blocked_timeouts.html

multiple events in paho mqtt with python

#!/usr/bin/env python
import sys
from arduino.Arduino import Arduino
import paho.mqtt.client as mqtt
import serial
import time
pin = 13
broker_adress = "10.0.2.190"
sys.path.append("/home/hu/Schreibtisch/Arduino_BA_2.0/Probe_Programmierung/Python-Arduino-Proto-API-v2/arduino")
ser = serial.Serial('/dev/ttyACM0', 9600)
b = Arduino('/dev/ttyACM0')
b.output([pin])
b.setLow(pin)
gassensor_value = "no default_value"
sensor_value = [['/Deutschland/Osnabrueck/Coffee-bike-1/Sensor_1',gassensor_value]]
#########################################################################
# Callback_1 for relay
#on_connect1,on_disconnect1,on_subscribe1on_message_1
#########################################################################
def on_connect(mqttrelay, obj, flags, rc):
if rc != 0:
exit(rc)
else:
mqttrelay.subscribe("qos0/test", 0)
def on_disconnect(mqttrelay, obj, rc):
obj = rc
def on_subscribe(mqttrelay, obj, mid, granted_qos):
print(mqttrelay.subscribe("qos0/test", 0))
print("Waiting for the subscribed messages")
def on_message(mqttrelay,userdata, message):
a = str(message.payload.decode("utf-8"))
print(a)
if (a == "1" or a == "0"):
if (a == "1"):
b.setHigh(13)
time.sleep(10)
else:
b.setLow(13)
time.sleep(10)
else:
print("please publish the message 1 or 0")
#########################################################################
# Callback_2 for gassensor
# on_connect2,on_publish2
#########################################################################
def on_publish(mqttgassensor, obj, mid):
print("mid: " + str(mid))
def on_connect(mqttgassensor, userdata, flags, rc):
print("Connected with result code " + str(rc))
#create new instance to subscribe the sitution of relay
mqttrelay = mqtt.Client("relay_K_12", 1)
#create new instance to publish the situation of gassensor
mqttgassensor = mqtt.Cleint("gassensor",1)
#the events and callbacks of instance mqttrelais associate with each other:
mqttrelay.on_message = on_message
mqttrelay.on_connect = on_connect
mqttrelay.on_subscribe = on_subscribe
mqttrelay.on_disconnect = on_disconnect
mqttrelay.connect(broker_adress)
#the events and callbacks of instance gassensor associate with each other:
mqttgassensor.on_connect = on_connect
mqttgassensor.on_publish = on_publish
mqttgassensor.connect(broker_adress)
while True:
mqttrelay.loop_start()
time.sleeps(2)
mqttrelay.loop_stop()
print("relay 開始循環")
mqttgassensor.loop_start()
mqttgassensor.loop()
time.sleep(1)
sensor_value[0][1] = ser.readline()
if (sensor_value[0][1] != "no default_value" or sensor_value[0][1] != b''):
print(sensor_value[0])
mqttgassensor.publish("/Deutschland/Osnabrueck/Coffee-bike-1/Sensor_1", sensor_value[0][1])
mqttgassensor.loop_stop()
Hello, everyone. I want to accomplish two instances in this script.
Through the publish we can get the data from the gassensor. Because at the top of this script i have imported the serial modul with that we can accomplish the communication between the arduino and raspberry pi.
I want to use the subscribe to get the command(1 or 0) from the server. the number 1 can active the Relay and the 0 can deactive the relay.
I have tried to accomplish the two thoughts lonely and successfully. But the Combination gives me no reply.
You only need 1 instance of the MQTT client since there is only 1 broker. You can subscribe and publish to multiple topics from a single client.
You should connect this 1 instance and then start the network loop in the background with client.start_loop()
You can then run your own loop to read from the serial port.

Add several individual threads to MQTT Python code

this is my code and I need add,
(1) loop(thread1) which can publish data when receive specific data by UART(ser.read())
(2) loop(thread2) which can run report_temp_humidity() with timed delay
so I put thread for test but this runs only one time, not loop.
Is this cause by "mqttclient.loop_forever()" at the end of line?
Need some help on this.
import paho.mqtt.client as mqtt
import Adafruit_DHT
import Adafruit_BBIO.UART as UART
import serial
import threading
import time
# DHT-11 Temperature & Humidity
sensor = Adafruit_DHT.DHT11
sensor_pin = 'P8_11'
# UART
UART.setup("UART1")
ser = serial.Serial(port = "/dev/ttyO1", baudrate=9600)
ser.close()
ser.open()
# Define event callbacks
def on_connect(self, client, userdata, rc):
if rc == 0:
print("Connected successfully.")
else:
print("Connection failed. rc= "+str(rc))
def on_publish(client, userdata, mid):
print("Message "+str(mid)+" published.")
def on_subscribe(client, userdata, mid, granted_qos):
print("Subscribe with mid "+str(mid)+" received.")
def on_message(client, userdata, msg):
print("Message received on topic "+msg.topic+" with QoS "+str(msg.qos)+"and payload "+msg.payload)
if msg.payload == "check_status":
report_temp_humidity()
if msg.payload == "motor_run!":
print "Send command to RF"
ser.write("11.1111[S]")
# Define my function prototypes
def report_temp_humidity():
hum, temp = Adafruit_DHT.read_retry(sensor, sensor_pin)
mqttclient.publish("/my_topic1/", "Temperature "+ str(temp) + "`C", qos=0)
mqttclient.publish("/my_topic1/", "Humidity " + str(hum) + "%", qos=0)
# Define thread(loop) test
def loop_test(delay):
print(loop!)
time.sleep(1)  
t1 = threading.Thread(target=loop_test, args=('1,',))
t1.start()
mqttclient = mqtt.Client()
# Assign event callbacks
mqttclient.on_connect = on_connect
mqttclient.on_publish = on_publish
mqttclient.on_subscribe = on_subscribe
mqttclient.on_message = on_message
# Connect
mqttclient.username_pw_set("dioty_user", "1234567")
mqttclient.connect("mqtt.dioty.co", 1883)
# Start subscription
mqttclient.subscribe("/my_topic1/")
# Publish a message
mqttclient.publish("/my_topic1/", "Hello World Message!")
# Loop
mqttclient.loop_forever()
This has nothing to do with the MQTT library.
Your thread t1 will run the loop_test function, which only has 2 statements in it. It will run these (once) then exit, there is nothing to tell it to continue to do anything else.
If you want that thread to continue to run for ever then you will need to include a loop statement of some sort (e.g. while True).

Pass asynchronous websocket.send() to stdout/stderr wrapper class

I have a class function which unbuffers stdout and stderr, like so:
class Unbuffered:
def __init__(self, stream):
self.stream = stream
def write(self, data):
data = data.strip()
if data.startswith("INFO: "):
data = data[6:]
if '[' in data:
progress = re.compile(r"\[(\d+)/(\d+)\]")
data = progress.match(data)
total = data.group(2)
current = data.group(1)
data = '{0}/{1}'.format(current, total)
if data.startswith("ERROR: "):
data = data[7:]
self.stream.write(data + '\n')
self.stream.flush()
def __getattr__(self, attr):
return getattr(self.stream, attr)
The output is from a function run in ProcessPoolExecutor when inbound from websocket arrives.
I want the output printed in console as well as sent to my websocket client. I tried asyncing Unbuffered and passing websocket instance to it but no luck.
UPDATE: The essentials of run() and my websocket handler() look something like this:
def run(url, path):
logging.addLevelName(25, "INFO")
fmt = logging.Formatter('%(levelname)s: %(message)s')
#----
output.progress_stream = Unbuffered(sys.stderr)
stream = Unbuffered(sys.stdout)
#----
level = logging.INFO
hdlr = logging.StreamHandler(stream)
hdlr.setFormatter(fmt)
log.addHandler(hdlr)
log.setLevel(level)
get_media(url, opt)
async def handler(websocket, path):
while True:
inbound = json.loads(await websocket.recv())
if inbound is None:
break
url = inbound['url']
if 'path' in inbound:
path = inbound['path'].rstrip(os.path.sep) + os.path.sep
else:
path = os.path.expanduser("~") + os.path.sep
# blah more code
while inbound != None:
await asyncio.sleep(.001)
await loop.run_in_executor(None, run, url, path)
run(), handler() and Unbuffered are separate from each other.
Rewriting get_media() to use asyncio instead of running it in a different thread would be the best. Otherwise, there are some options to communicate between a regular thread and coroutines, for example, using a socketpair:
import asyncio
import socket
import threading
import time
import random
# threads stuff
def producer(n, writer):
for i in range(10):
# print("sending", i)
writer.send("message #{}.{}\n".format(n, i).encode())
time.sleep(random.uniform(0.1, 1))
def go(writer):
threads = [threading.Thread(target=producer, args=(i + 1, writer,))
for i in range(5)]
for t in threads:
t.start()
for t in threads:
t.join()
writer.send("bye\n".encode())
# asyncio coroutines
async def clock():
for i in range(11):
print("The time is", i)
await asyncio.sleep(1)
async def main(reader):
buffer = ""
while True:
buffer += (await loop.sock_recv(reader, 10000)).decode()
# print(len(buffer))
while "\n" in buffer:
msg, _nl, buffer = buffer.partition("\n")
print("Got", msg)
if msg == "bye":
return
reader, writer = socket.socketpair()
reader.setblocking(False)
threading.Thread(target=go, args=(writer,)).start()
# time.sleep(1.5) # socket is buffering
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait([clock(), main(reader)]))
loop.close()
You can also try this 3rd-party thread+asyncio compatible queue: janus

Resources