Greatings everyone,
I have a main that runs a thread. This thread manages a HTTP server using http.server like this:
import binascii, threading
from http.server import BaseHTTPRequestHandler, HTTPServer
class Com(threading.Thread):
def __init__(self, queue):
MyHandler.queue = queue
threading.Thread.__init__(self)
def run(self):
httpd = HTTPServer(('10.0.0.254', 8000), MyHandler)
httpd.serve_forever()
class MyHandler(BaseHTTPRequestHandler):
queue = None
def do_POST(self):
" Receive a POST and put the received frame in the queue "
def send(self, toSend):
self.send_response(200, "OK")
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write(toSend)
The do_POST method works well, it receive data and send it to the main thanks to the queue.
What I want is: sometimes when I want, the main orders the server to send some data. Normally, the code to send is good. My problem is that I can't, from the main, access to the HTTP server instance to call the send method, because it is managed by http.server.
In Com class (instance in the main), I tried something like:
def sendThroughServer(self, toSend):
MyHandler.send(toSend)
Also using class method, but it doesn't seems to work, because it requieres "self".
How can I do that pls ?
Hope it is clear. Thanks in advance.
Related
I'm trying to implement an async RPC client within a Flask server.
The idea is that each request spawn a thread with an uuid, and each request is going to wait until there is a response in the RpcClient queue attribute object with the correct uuid.
The problem is that one request out of two fails. I think that might be a problem with multi-threading, but I don't see where it comes from.
Bug can be seen here.
Using debug print, it seems that the message with the correct uuid is received in the _on_response callback and update the queue attribute in this instance correctly, but the queue attribute within the /rpc_call/<payload> endpoint doesn't synchronize (so queue[uuid] has a value of response in the RpcClient callback but still None in the scope of the endpoint).
My code:
from flask import Flask, jsonif
from gevent.pywsgi import WSGIServer
import sys
import os
import pika
import uuid
import time
import threading
class RpcClient(object):
"""Asynchronous Rpc client."""
internal_lock = threading.Lock()
queue = {}
def __init__(self):
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host='rabbitmq'))
self.channel = self.connection.channel()
self.channel.basic_qos(prefetch_count=1)
self.channel.exchange_declare(exchange='kaldi_expe', exchange_type='topic')
# Create all the queue and bind them to the corresponding routing key
self.channel.queue_declare('request', durable=True)
result = self.channel.queue_declare('answer', durable=True)
self.channel.queue_bind(exchange='kaldi_expe', queue='request', routing_key='kaldi_expe.web.request')
self.channel.queue_bind(exchange='kaldi_expe', queue='answer', routing_key='kaldi_expe.kaldi.answer')
self.callback_queue = result.method.queue
.
thread = threading.Thread(target=self._process_data_events)
thread.setDaemon(True)
thread.start()
def _process_data_events(self):
self.channel.basic_consume(self.callback_queue, self._on_response, auto_ack=True)
while True:
with self.internal_lock:
self.connection.process_data_events()
time.sleep(0.1)
def _on_response(self, ch, method, props, body):
"""On response we simply store the result in a local dictionary."""
self.queue[props.correlation_id] = body
def send_request(self, payload):
corr_id = str(uuid.uuid4())
self.queue[corr_id] = None
with self.internal_lock:
self.channel.basic_publish(exchange='kaldi_expe',
routing_key="kaldi_expe.web.request",
properties=pika.BasicProperties(
reply_to=self.callback_queue,
correlation_id=corr_id,
),
body=payload)
return corr_id
def flask_app():
app = Flask("kaldi")
#app.route('/', methods=['GET'])
def server_is_up():
return 'server is up', 200
#app.route('/rpc_call/<payload>')
def rpc_call(payload):
"""Simple Flask implementation for making asynchronous Rpc calls. """
corr_id = app.config['RPCclient'].send_request(payload)
while app.config['RPCclient'].queue[corr_id] is None:
#print("queue server: " + str(app.config['RPCclient'].queue))
time.sleep(0.1)
return app.config['RPCclient'].queue[corr_id]
if __name__ == '__main__':
while True:
try:
rpcClient = RpcClient()
app = flask_app()
app.config['RPCclient'] = rpcClient
print("Rabbit MQ is connected, starting server", file=sys.stderr)
app.run(debug=True, threaded=True, host='0.0.0.0')
except pika.exceptions.AMQPConnectionError as e:
print("Waiting for RabbitMq startup" + str(e), file=sys.stderr)
time.sleep(1)
except Exception as e:
worker.log.error(e)
exit(e)
I found where the bug came from:
Thedebug=True of the line app.run(debug=True, threaded=True, host='0.0.0.0') restart the server at the beginning.
The whole script is then restarted from the beginning. Because of it, another rpcClient is initialized and consume from the same queue. Problem is that the previous thread is also running. This cause two rpcClient to consume from the same thread, with one that is virtually useless.
I have this part of code which is doing psubscribe to redis. I want to run this part of code in a thread an working in the background while the other part of code will check some notifications from this below.
def psubscribe(context, param1, param2, param3):
context.test_config = load_config()
RedisConnector(context.test_config["redis_host"],
context.test_config["redis_db_index"])
redis_notification_subscriber_connector = RedisConnector(context.test_config["notification__redis_host"],
int(param3),
int(context.test_config[
"notification_redis_port"]))
context.redis_connectors = redis_notification_connector.psubscribe_to_redis_event(param1,
timeout_seconds=int(
param2)
)
what I have done till now: but its not running :(
context.t = threading.Thread(target=psubscribe, args=['param1', 'param2', 'param3'])
context.t.start()
It is actually working. I think you didn't need actually to pass context variable to your psubscribe function.
Here is an example:
Start http server that listens on port 8000 as a background thread
Send http requests to it and validate response
Feature scenario:
Scenario: Run background process and validate responses
Given Start background process
Then Validate outputs
background_steps.py file:
import threading
import logging
from behave import *
from features.steps.utils import run_server
import requests
#given("Start background process")
def step_impl(context):
context.t = threading.Thread(target=run_server, args=[8000])
context.t.daemon = True
context.t.start()
#then("Validate outputs")
def step_impl(context):
response = requests.get('http://127.0.0.1:8000')
assert response.status_code == 501
utils.py file
from http.server import HTTPServer, BaseHTTPRequestHandler
def run_server(port, server_class=HTTPServer, handler_class=BaseHTTPRequestHandler):
server_address = ('', port)
httpd = server_class(server_address, handler_class)
httpd.serve_forever()
I'm creating a python3 tornado web server that may listen to an MQTT broker and whenever listens a new message from it, broadcasts it to the connected browsers, through web sockets. However, seems that Tornado doesn't like calls to its API from a thread different to IOLoop.current() and I can't figure out another solution...
I've already tried to write some code. I've put the whole MQTT client (in this case called PMCU client), on a separated thread which loops and listens to MQTT notifications.
def on_pmcu_data(data):
for websocket_client in websocket_clients:
print("Sending websocket message")
websocket_client.write_message(data) # Here it stuck!
print("Sent")
class WebSocketHandler(tornado.websocket.WebSocketHandler):
def open(self):
websocket_clients.append(self)
def on_close(self):
websocket_clients.remove(self)
def make_app():
return tornado.web.Application([
(r'/ws', WebSocketHandler)
])
if __name__ == "__main__":
main_loop = IOLoop().current()
pmcu_client = PMCUClient(on_pmcu_data)
threading.Thread(target=lambda: pmcu_client.listen("5.4.3.2")).start()
app = make_app()
app.listen(8080)
main_loop.start()
However as I said, seems that calls to Tornado API outside the IOLoop.current() blocks: the code above only prints Sending websocket message.
My intent is to run websocket_client.write_message(data) on IOLoop.current() event loop. But seems that the function IOLoop.current().spawn_callback(lambda: websocket_client.write_message(data)) not works after IOLoop.current() has started. How could I achieve that?
I know that I have a huge misunderstanding of IOLoop, asyncio, on which it depends, and python3 async.
on_pmcu_data is being called in a separate thread but the websocket is controlled by Tornado's event loop. You can't write to a websocket from a thread unless you have access to the event loop.
You'll need to ask the IOLoop to write the data to websockets.
Solution 1:
For simple cases, if you don't want to change much in the code, you can do this:
if __name__ == "__main__":
main_loop = IOLoop().current()
on_pmcu_data_callback = lambda data: main_loop.add_callback(on_pmcu_data, data)
pmcu_client = PMCUClient(on_pmcu_data_callback)
...
This should solve your problem.
Solution 2:
For more elaborate cases, you can pass the main_loop to PMCUClient class and then use add_callback (or spawn_callback) to run on_pmcu_data.
Example:
if __name__ == "__main__":
main_loop = IOLoop().current()
pmcu_client = PMCUClient(on_pmcu_data, main_loop) # also pass the main loop
...
Then in PMCUCLient class:
class PMCUClient:
def __init__(self, on_pmcu_data, main_loop):
...
self.main_loop = main_loop
def lister(...):
...
self.main_loop.add_callback(self.on_pmcu_data, data)
I'm trying to start a long blocking function after receiving an HTTP request. The request must be responded inmediately (200 OK or 500 Internal Error), but the process should run in the background and send a notification to a WebSocket after finished.
Also, the application should receive other requests for processing and these must also be responded inmediately, without blocking the previous ones.
I'm using add_callback, but I'm not sure if it's the correct way to use tornado, since it's blocking the incoming HTTP requests. I've tried using different threads, but I got exceptions when trying to call the send_message method inside the WebSocket handler.
import time
from tornado import gen
from tornado.ioloop import IOLoop
from tornado.web import Application, RequestHandler, asynchronous
from tornado.websocket import WebSocketHandler
def long_process(id):
time.sleep(5)
class RequestWeb(RequestHandler):
#gen.coroutine
def process(self, id):
# Trying to call long_process, just like
# yield gen.Task(IOLoop.current().add_timeout, time.time() + 10)
# The response must be sent inmediately, but the process should run in the background
IOLoop.current().add_callback(callback=lambda: long_process(id))
#asynchronous
#gen.coroutine
def get(self, id):
IOLoop.current().add_future(self.process(id), self.process_complete)
self.write("OK")
def process_complete(self, future):
SocketHandler.send_message('Processing complete')
class SocketHandler(WebSocketHandler):
connections = set()
def open(self):
SocketHandler.connections.add(self)
#classmethod
def send_message(cls, message):
for ws in cls.connections:
ws.write_message(message)
def make_app():
return Application([
(r'/api/(?P<id>[a-zA-Z0-9]+)$', RequestWeb),
(r'/ws', SocketHandler)
])
if __name__ == "__main__":
app = make_app()
app.listen(8000)
IOLoop.current().start()
I want to be able to add an instance variable to my subclass of http.server.BaseHTTPRequestHandler.
Here's my code:
from http.server import BaseHTTPRequestHandler, HTTPServer
import urllib
class Server(BaseHTTPRequestHandler):
def __init__(self, request, client_addr, server):
super().__init__(request, client_addr, server)
self.pathobj = urllib.parse.urlparse(self.path)
def do_HEAD(self):
self.send_response(200)
def do_GET(self):
print(self.pathobj)
self.send_response(200)
self.end_headers()
def do_POST(self):
print(self.pathobj)
self.send_response(405)
self.end_headers()
def run(server_class=HTTPServer, handler_class=Server, port=8080):
server_address = ("", port)
httpd = server_class(server_address, handler_class)
print("Starting httpd on port {}...".format(port))
httpd.serve_forever()
if __name__ == "__main__":
run()
I want to be able to access the ParseResult object returned by urllib.parse.urlparse in each class method without needing to rewrite self.pathobj = urllib.parse.urlparse(self.path) at the beginning of every class method.
The above code does not work -- when do_GET or do_POST are called it complains that 'Server' object has no attribute 'pathobj'.
The above docs for http.server.BaseHTTPRequestHandler say:
All of the relevant information is stored in instance variables of the handler. Subclasses should not need to override or extend the __init__() method.
But I don't see another way to do this. Is it possible?
The docs say
The handler will parse the request and the headers, then call a method specific to the request type.
As it turns out, http.server.BaseHTTPRequestHandler ultimately inherits from socketserver.BaseRequestHandler, and socketserver.BaseRequestHandler.__init__() (defined here) calls do_GET(). So the issue is that the instance variable is actually being set after do_GET() has already been called.
So in order to make this work, you'll need to move the self.pathobj line above the super() line and then rewrite it to do the parsing that BaseHTTPRequestHandler does to construct self.path.