how to communicate between regular MQTT and websocket MQTT? - python-3.x

I have a main App that initializes fastAPI and uses fastAPI MQTT. Now, this file is running in an event loop and is looking for regular MQTT port.
I have a second app that uses MQTT Websocket and is running in an event loop. Now, I want the two apps to exchange information with each other. How can I make them interchange the information.One is using websocket MQTT and another one is using fast API MQTT. What is the way to make them talk to each other. Here is the
main.py - which runs fastapi and mqtt - regular one.
from fastapi import FastAPI
from fastapi_mqtt import FastMQTT, MQTTConfig
from pydantic import BaseModel
from ipaddress import IPv4Address
import jsonpickle
app = FastAPI()
class Nmap(BaseModel):
host: IPv4Address
portRange: str
class Config:
schema_extra = {
"example" : {
"host": "10.0.2.15",
"portRange": "22-80",
"description": "Scan the port from 22 to 80 of the ip address 10.0.2.15"
}
}
## for docker - compose using mqtt - broker name from docker compose.
## for normal leave that blank to fall off to localhost.
mqtt_config = MQTTConfig(host = "mqtt")
mqtt = FastMQTT(config=mqtt_config)
mqtt.init_app(app)
#mqtt.on_connect()
def connect(client, flags, rc, properties):
mqtt.client.subscribe("/mqtt/toModel/#") # subscribing mqtt topic wildcard- multi-level
print("connected: ", client, flags, rc, properties)
#mqtt.on_message()
async def message(client, topic, payload, qos, properties):
print("received message: ", topic, jsonpickle.decode(payload.decode()), qos, properties)
return 0
#mqtt.on_disconnect()
def disconnect(client, packet, exc=None):
print("Disconnected")
#mqtt.on_subscribe()
def subscribe(client, mid, qos, properties):
print("subscribed", client, mid, qos, properties)
#app.get("/")
async def func():
mqtt.client.publish("/mqtt", "Hello from fastApi")
return {"result": True, "message": "Published"}
#app.post("/scan/{host}")
async def scan_host_port(nmap_details : Nmap):
results = {"got_val" : nmap_details}
print(type(nmap_details))
mqtt.client.publish("/mqtt/fromModel/nmap", jsonpickle.encode(nmap_details))
return results
and here is the websocket based one mqtt.
import paho.mqtt.client as paho
import time
broker="10.0.2.15"
#port= 80
#port=1883
port= 9001
sub_topic="ws/pythonApp"
def on_subscribe(client, userdata, mid, granted_qos): #create function for callback
print("subscribed with qos",granted_qos, "\n")
pass
def on_message(client, userdata, message):
print("message received " ,str(message.payload.decode("utf-8")))
def on_publish(client,userdata,mid): #create function for callback
print("data published mid=",mid, "\n")
pass
def on_disconnect(client, userdata, rc):
print("client disconnected ok")
client= paho.Client("client-socks",transport='websockets') #create client object
#client= paho.Client("control1")
client.on_subscribe = on_subscribe #assign function to callback
client.on_publish = on_publish #assign function to callback
client.on_message = on_message #assign function to callback
client.on_disconnect = on_disconnect
print("connecting to broker ",broker,"on port ",port)
client.connect(broker,port) #establish connection
client.loop_start()
print("subscribing to ",sub_topic)
client.subscribe(sub_topic)
time.sleep(3)
client.publish("ws/jsclient","on") #publish
time.sleep(4)
client.disconnect()

You don't have to do anything, the MQTT broker will bridge both input protocols (native MQTT & MQTT over WebSockets) into the same topic space.
Anything published will be published to any subscribers no matter if they are connected via native MQTT or MQTT over WebSockets.

Related

how to communicate between regular MQTT and tornado websocket?

Is there any way to make a mqtt client send messages to tornado websocket?
Using this code, tornado works. While from MQTT no message passes.
import os.path
import tornado.httpserver
import tornado.web
import tornado.ioloop
import tornado.options
import tornado.httpclient
import tornado.websocket
import json
import random
from paho.mqtt import client as mqtt_client
broker = 'xxxxxxx'
port = 1883
topic = "xxx/xxxx"
client_id = f'xxxx-{random.randint(0, 100)}'
clients = []
def connect_mqtt() -> mqtt_client:
def on_connect(clientMQTT, userdata, flags, rc):
if rc == 0:
print("Connected to MQTT Broker!")
else:
print("Failed to connect, return code %d\n", rc)
clientMQTT = mqtt_client.Client(client_id)
# clientMQTT.username_pw_set(username, password)
clientMQTT.on_connect = on_connect
clientMQTT.connect(broker, port)
return clientMQTT
def subscribe(clientMQTT: mqtt_client):
def on_message(clientMQTT, userdata, msg):
print(f"Received `{msg.payload.decode()}` from `{msg.topic}` topic")
send_to_all_clients(msg.payload.decode())
clientMQTT.subscribe(topic)
clientMQTT.on_message = on_message
def runMQTT():
clientMQTT = connect_mqtt()
subscribe(clientMQTT)
clientMQTT.loop_start()
# clientMQTT.loop_forever()
def send_to_all_clients(message):
for clientws in clients:
clientws.write_message(message)
class SocketHandler(tornado.websocket.WebSocketHandler):
def open(self):
print("new client")
clients.append(self)
def on_close(self):
clients.remove(self)
print("removing client")
def on_message(self, message):
pass
# for client in utility.clients:
# if client != self:
# client.write_message(msg)
if __name__ == '__main__':
app = tornado.web.Application(
handlers = [
(r"/monitor", SocketHandler)
],
debug = True,
template_path = os.path.join(os.path.dirname(__file__), "templates"),
static_path = os.path.join(os.path.dirname(__file__), "static")
)
runMQTT()
app.listen(8200)
tornado.ioloop.IOLoop.instance().start()
I hope someone can give me some help.
Thank you all
If I try to listen on mqtt, with no tornado, it works.
I would like to be able to send messages from mqtt to browsers via tornado websocket.

Python flask mqtt socketio and subscribing on startup

When starting a flask, mqtt and socketio app how exactly can I subscribe to topics when the app starts but before the browser connects for the first time?
I had assumed I could use before_first_request but that only gets called on first request, also assume I could use mqtt.on_connect but that is never called when using socket io and if I subscribe before starting the app in __main__ then I get two threads subscribed rather than one.
#!/usr/bin/env python3
import json
from flask import Flask, render_template
from flask_mqtt import Mqtt
from flask_socketio import SocketIO
from flask_bootstrap import Bootstrap
# Flask App
app = Flask(__name__)
app.config['MQTT_BROKER_URL'] = '192.168.109.135'
print('Configured MQTT IP Address: ' + app.config['MQTT_BROKER_URL'])
mqtt = Mqtt(app)
socketio = SocketIO(app)
#app.route("/")
def roots():
return render_template('index.html')
#app.route('/mqtt')
def mqtt_index():
return render_template('mqtt.html')
#socketio.on('subscribe')
def handle_subscribe(json_str):
print('Subscribe ' + json_str)
#socketio.on('unsubscribe_all')
def handle_unsubscribe_all():
print('Socket IO unsubscribe all')
mqtt.unsubscribe_all()
#socketio.on('connect')
def handle_connect():
print('Socket IO Connected')
#socketio.on('discconnect')
def handle_connect():
print('Socket IO Discconnect')
#mqtt.on_connect()
def handle_mqtt_connect():
print('MQTT Connected')
#mqtt.on_message()
def handle_mqtt_message(client, userdata, message):
print('MQTT Message')
data = dict(
topic=message.topic,
payload=message.payload.decode(),
qos=message.qos,
)
print(mqttresponse)
socketio.emit('mqtt_message', data=data)
#mqtt.on_log()
def handle_logging(client, userdata, level, buf):
print('MQTT log', level, buf)
pass
#app.before_first_request
def before_first_request():
print("before_first_request")
mqtt.subscribe('homeassistant/+/+/set', 0)
if __name__ == "__main__":
# Main http web server for firmware downloading and the main frontend.
socketio.run(app, host='0.0.0.0', port='6080', use_reloader=True)
Any ideas where the mqtt.subscribe should go so it subscribes to the topics I want before the first connect to the webserver?
I found there is already a related issue which is the on_connect callback doesn't get called, and a question on here too. So this is a duplicate.
Flask MQTT on_connect is never called when used with SocketIO
https://github.com/stlehmann/Flask-MQTT/issues/82

Python Paho-Mqtt: What should I pay attention to when subscribing to more topics?

With mqtt subscribe client I am subscribing to lots of threads (over 6000) but not getting results that change on the fly. I'm lagging. Does mqtt give possibility to subscribe too many threads in parallel in background? loop_start would that be enough?
What should I pay attention to when subscribing to more topics?
import logging
import paho.mqtt.client as mqtt
import requests
import zmq
import pandas as pd
PORT=1351
def set_publisher():
context = zmq.Context()
socket_server = context.socket(zmq.PUB)
socket_server.bind(f"tcp://*:{PORT}")
return socket_server
# The callback for when the client receives a CONNACK response from the server.
def on_connect(client, userdata, flags, rc):
#logging.warning(f"Connected with result code :: code : {rc}")
print(f"Connected with result code :: code : {rc}")
# Subscribing in on_connect() means that if we lose the connection and
# reconnect then subscriptions will be renewed.
client.subscribe(topics)
# The callback for when a PUBLISH message is received from the server.
def on_message(client, userdata, msg):
msg = msg.payload
# logging.info(f"message:: {msg}\n")
print(f"message:: {msg}\n")
if msg:
publisher.send(f"{msg}")
def on_disconnect(client, userdata, rc):
if rc != 0:
# logging.warning(f"Unexpected disconnection :: code: {rc}")
print(f"Unexpected disconnection :: code: {rc}")
#todo: if rc is change hostname raise except
client = mqtt.Client(protocol=mqtt.MQTTv31, transport="tcp")
client.username_pw_set(******, password=******)
topics = [(f"topic{i}", 0) for i in 6000]
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
if client.connect(hostname= *****, port= **** , keepalive=300) != 0:
# logging.info("Could not connect to MQTT Broker !")
print("Could not connect to MQTT Broker !")
client.loop_forever(timeout=3000)
You are describing a situation of compute power (either at the client or the broker or in-between) not sufficient to handle your scenario. It is a common occurrance and that is what performance testing is for: does your setup handle your scenario for your requirements? Capacity planning then expands that question to: ... in the future.

Combine paho MQTT with another asyncio process in Python

I have a basic MQTTListener class in Python which listens for messages on certain topics and should start or stop an async process imported from another script. This process runs forever, unless it is manually stopped. Let's assume the Listener looks like this:
import paho.mqtt.client as mqtt
import json
from python_file.py import async_forever_function
class MqttListener:
def __init__(self, host, port, client_id):
self.host = host
self.port = port
self.client_id = client_id
self.client = mqtt.Client(client_id=self.client_id)
self.client.connect(host=self.host, port=self.port)
def on_connect(self, client, userdata, flags, rc):
self.client.subscribe(topic=[("start", 1), ])
self.client.subscribe(topic=[("stop", 1), ])
logging.info(msg="MQTT - connected!")
def on_disconnect(client, userdata, rc):
logging.info(msg="MQTT - disconnected!")
def on_message(self, client, userdata, message, ):
print('PROCESSING MESSAGE', message.topic, message.payload.decode('utf-8'), )
if message.topic == 'start':
async_forever_function(param='start')
print('process started')
else:
async_forever_function(param='stop')
print('process removed')
def start(self):
self.client.on_connect = lambda client, userdata, flags, rc: self.on_connect(client, userdata, flags, rc)
self.client.on_message = lambda client, userdata, message: self.on_message(client, userdata, message)
self.client.on_disconnect = lambda client, userdata, rc: self.on_disconnect(client, userdata, rc)
self.client.loop_start()
def stop(self):
self.client.loop_stop()
Now, this works for starting a new async process. That is, async_function is correctly triggered when a message is posted on the start MQTT topic. However, once this async process is started, the listener is no longer able to receive/process messages from the stop MQTT topic and the async process will continue to run forever, when in fact it should have been stopped.
My question: how can I adapt the code of this class such that it can also process messages when an active async process is running in the background?
You can not do blocking tasks in the on_message() callback.
This call back runs on the MQTT client thread (the one started by the loop_start() function. This thread handles all network traffic and message handling, if you block it then it can't do any of that.
If you want to call long running tasks from the on_message() callback you need to start a new thread for the long running task so it doesn't block the MQTT client loop.

Flask-socketio doesn't recieve message from client

I'm trying to write a basic Socket.io program where the python client (python-socketio[asyncio_client] 4.6.0) emits a single string message to the flask server (with Flask-SocketIO 4.3.1 and eventlet).
The client appears to connect and send the message properly, but there is no output seen at the Flask server.
Server code:
from flask import Flask
from flask_socketio import SocketIO, emit
app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app)
#socketio.on('connect')
def test_connect():
print('connected')
#socketio.on('disconnect')
def test_disconnect():
print('Client disconnected')
#socketio.on('message')
def handle_message(msg):
print('Recieved',msg)
#socketio.on('json')
def handle_json(json):
print(str(json))
if __name__ == '__main__':
socketio.run(app,debug=True)
Client code:
import asyncio
import socketio
sio = socketio.AsyncClient()
#sio.event
def connect():
print('connection established')
#sio.event
def disconnect():
print('disconnected from server')
async def main():
await sio.connect('http://localhost:5000')
await sio.emit('message',data='detection')
print('message sent')
await sio.disconnect()
if __name__ == '__main__':
asyncio.run(main())
Server output:
PS C:\Users\daksh\sih\sihPython> python .\test_socketio.py
* Restarting with stat
* Debugger is active!
* Debugger PIN: 101-561-255
(16664) wsgi starting up on http://127.0.0.1:5000
(16664) accepted ('127.0.0.1', 59497)
connected
127.0.0.1 - - [23/Jul/2020 20:38:42] "GET /socket.io/?transport=polling&EIO=3&t=1595516920.71801 HTTP/1.1" 200 367 0.004934
Client disconnected
127.0.0.1 - - [23/Jul/2020 20:38:42] "GET /socket.io/?transport=websocket&EIO=3&sid=88790300120f4b899e019ae7cc16ee87&t=1595516922.7757218 HTTP/1.1" 200 0 0.010027
Client output:
PS C:\Users\daksh\sih\sihPython> python .\socketio-client.py
connection established
message sent
The print statement from handle_message() is missing in the server output.
I've gone through multiple tutorials online, and I've tried it with and without namespaces. Have not been able to figure out what's wrong.
Any help is appreciated.
(I'm using Python 3.8.3 on Windows 10)
UPDATE: It works if I change the client code to use socketio.Client() instead of AsyncClient(), however I need the client to connect using AsyncClient.
The problem is that your async client is obviously asynchronous, you can't just send and exit because you don't give the background tasks that support the Socket.IO protocol time to do their thing.
Here is a more robust version of the client that lets the event go through before exiting:
import asyncio
import socketio
sio = socketio.AsyncClient()
#sio.event
async def connect():
print('connection established')
await sio.emit('message',data='detection', callback=done)
print('message sent')
#sio.event
def disconnect():
print('disconnected from server')
async def done():
await sio.disconnect()
async def main():
await sio.connect('http://localhost:5000')
await sio.wait()
if __name__ == '__main__':
asyncio.run(main())
The trick here is to use a callback on the emit. When the callback is invoked you are sure the message was delivered, so at that point it is safe to disconnect.

Resources