I need Threadsafe primitives (locking, conditions Semaphore), do they exist in the asyncio ecosystem?
I created some code myself, but it feels a bit sluggish:
import asyncio
from threading import get_ident,Lock,Event
class AsyncThreadsafeEvent():
def __init__(self,*args,**kwargs):
super(AsyncThreadsafeEvent, self).__init__()
self.waiters = {}
self.event = Event()
def set(self):
self.event.set()
events = self.waiters.values()
for loop,event in events:
loop.call_soon_threadsafe(event.set)
async def wait(self):
event = asyncio.Event()
#not strictly necessary but could provide tiny speedup
if self.event.is_set():
return
self.waiters[get_ident()] = (asyncio.get_event_loop(),event)
#to ensure thread safty
if self.event.is_set():
return
await event.wait()
Any help would be appreciated!
Related
I am trying to get websockets, asyncio and multiprocess to work together. I have been stuck on this for 2 days and could appreciate some help.
I have searched for websockets asyncio and multiprocessing on stackoverflow as well as general internet searches. I have found threading examples, which I can make work.
import asyncio
import websockets
import threading
class Connection():
def __init__(self):
self.loop = asyncio.new_event_loop()
sock_thread = threading.Thread(target=self.new_loop)
sock_thread.start()
self.x = 0
async def connect_to_socket(self):
self.websocket = await websockets.connect('ws://demos.kaazing.com/echo')
await self.websocket.send("hello")
response = await self.websocket.recv()
print(response)
async def listen_to_socket(self):
while True:
await asyncio.sleep(0)
print('Listening for a message...')
while self.x < 5:
message = await self.websocket.recv()
print("< {}".format(message))
print('\n\n')
print(self.x)
self.x += 1
self.task.cancel()
self.loop.close()
def stop(self):
print('canceling task\n\n')
self.x = 0
self.task.cancel()
def new_loop(self):
self.task = self.loop.create_task(self.connect_to_socket())
self.loop.run_forever()
def make_task(self):
self.task = self.loop.create_task(self.listen_to_socket())
if __name__ == '__main__':
conn=Connection()
This works with no issues. I have seen examples where multiprocessing opens a process in an event loop, this is not what I want. I want to ope However, this is not what I want. I want to open a new process and run an event loop in the new process. Inside the event loop, I want to run my sockets. I want to free my main process from listening to sockets and use a child process to listen to the sockets while I do computationally expensive work on my main process.
When I try the following code. I get nothing.
import asyncio
import websockets
import multiprocessing
class Connection(multiprocessing.Process):
def __init__(self, tasks, results):
super().__init__()
self.tasks = tasks
self.results = results
self.loop = asyncio.new_event_loop()
print('create event loop')
self.x = 0
self.task = self.loop.create_task(self.test())
print('done with connecting')
#connect to socket and get response
async def test(self):
self.ws = await websockets.connect('ws://demos.kaazing.com/echo')
await self.websocket.send("hello")
response = await self.websocket.recv()
print(response)
#listen to socket long term after connection
async def listen_to_socket(self):
while True:
await asyncio.sleep(0)
print('Listening for a message...')
while self.x < 5:
await self.websocket.send("hello")
message = await self.websocket.recv()
print("< {}".format(message))
print('\n\n')
print(self.x)
self.x += 1
self.results.put(message)
self.task.cancel()
self.loop.close()
#stop task
def stop(self):
print('canceling task\n\n')
self.x = 0
self.task.cancel()
# listen to socket long term
#I have not called this as I can't even get a response from test()
def make_task(self):
self.task = self.loop.create_task(self.listen_to_socket())
if __name__ == '__main__':
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
process = Connection(tasks, results)
if tasks.empty():
print('empty')
else:
print(tasks.get())
I expect to connect with the socket and receive a response. However, I get nothing. I get no error messages,no printout from the connection, I get an empty queue and that's all. How do I get the return values from my websocket?
I am still new enough, I am not sure what I am doing wrong. Any advice would help me out.
Thank you
Anyone interested, I got this to work. It is very much a work in progress and I am adding to it, and since this is for me and relatively simple, I didn't comment it.
I started with the code from this answer and built on it.
Python3 Websockets / Multithreading issue
import asyncio
import websockets
import sys
import time
import multiprocessing
class connect():
def __init__(self, results, tasks):
self.x = 0
self.results = results
self.tasks = tasks
self.loop = asyncio.new_event_loop()
async def commander_start(self):
while not self.tasks.empty():
self.uri = self.tasks.get()
self.tasks.task_done()
self.ws = await websockets.connect(self.uri)
while True:
await asyncio.sleep(0.1)
print('Listening for a message...')
while self.x < 5:
await self.ws.send("hello")
message = await self.ws.recv()
message = message+str(self.x)
print("< {}".format(message))
print('\n\n')
print(self.x)
self.x += 1
self.results.put(message)
self.ws.close()
self.x = 0
print('ws clsed')
self.task.cancel()
await asyncio.sleep(1)
self.loop.close()
def run_commander(self):
self.task = self.loop.create_task(self.commander_start())
self.loop.run_forever()
def main(self):
self.commander = multiprocessing.Process(target=self.run_commander)
self.commander.start()
time.sleep(3)
self.commander.kill()
print('is alive:', self.commander, self.commander.is_alive())
if __name__ == "__main__":
size_q = 10
tasks = multiprocessing.JoinableQueue(maxsize=size_q)
results = multiprocessing.Queue(maxsize=size_q)
conn = connect(results,tasks)
tasks.put('ws://demos.kaazing.com/echo')
conn.main()
print('tasks2 put')
tasks.put('wss://echo.websocket.org')
conn.main()
if not results.empty():
for x in range(size_q):
print(results.get())
There is a bunch I am going to change and improve, I just wanted the base system to work so I could build from there, so that anyone that uses this will need to modify it to suit their needs. For instance, I spawn a new process and kill it, instead of running a continuous process and giving it work to do, I also am trying to figure out the specifics of the joinable queue and how to use it to add jobs after the process and event loop has been created.
I'm looking to mock a set of REST APIs for some tests. The following main() function works fine (i.e. it returns {"some-data": 1234} as json to the browser when I GET localhost:8099). The issue is it blocks the main thread:
from gevent import monkey, sleep, pywsgi
monkey.patch_all()
import flask
from flask_restful import reqparse, abort, Api, Resource
import queue
import sys
import threading
STUFFS = {"some-data": 1234}
class Stuff(Resource):
def get(self):
return flask.jsonify(STUFFS)
class ControlThread(threading.Thread):
def __init__(self, http_server, stop_event):
threading.Thread.__init__(self)
self.stop_event = stop_event
self.http_server = http_server
self.running = False
def run(self):
try:
while not self.stop_event.is_set():
if not self.running:
self.http_server.start()
self.running = True
sleep(0.001)
except (KeyboardInterrupt, SystemExit):
pass
self.http_server.stop()
class StuffMock:
def __init__(self, port, name=None):
if name is None:
name = __name__
self.app = flask.Flask(name)
self.api = Api(self.app)
self.api.add_resource(Stuff, "/stuff/")
self.stop_event = threading.Event()
self.http_server = pywsgi.WSGIServer(('', port), self.app)
self.serving_thread = ControlThread(self.http_server,
self.stop_event)
self.serving_thread.daemon = True
def start(self):
self.serving_thread.start()
def stop(self):
self.stop_event.set()
self.serving_thread.join()
def main():
mocker = StuffMock(8099)
mocker.start()
try:
while True:
sleep(0.01)
except (KeyboardInterrupt, SystemExit):
mocker.stop()
sys.exit()
if __name__ == "__main__":
main()
Without the sleep() call in the while loop above, nothing resolves. Here is a more succinct usage to demonstrate:
import time
from stuff_mock import StuffMock
mocker = StuffMock(8099)
mocker.start()
while True:
user_text = input("let's do some work on the main thread: ")
# will only resolve the GET request after user input
# (i.e. when the main thread executes this sleep call)
time.sleep(0.1)
if user_text == "q":
break
mocker.stop()
The gevent threading module seems to work differently from the core one. Does anyone have any tips or ideas about what's going on under the hood?
Found that if I switch out threading for multiprocessing (and threading.Thread for multiprocessing.Process), everything works as expected, and I can spin up arbitrary numbers of mockers without blocking.
I'm using tornado with threads.
In short, each time the websocket handler receives a requests, it start to execute a task, which might take a few minutes.
However, once a client is connected, no other client can be connected, until the first one disconnects.
Any ideas?
I've attached a minimal example that uses time.sleep to simulate long running tasks.
import tornado.web
import tornado.websocket
import tornado.httpserver
import tornado.ioloop
import time
import json
import threading
class TaskHandler(tornado.websocket.WebSocketHandler):
def open(self):
pass
def check_origin(self, origin):
return True
def on_message(self, message):
try:
print 'received: ', message
self.write_message(json.dumps({'status': 'running'}))
def worker_A(kwargs):
time.sleep(100)
pass
def worker_B(kwargs):
time.sleep(100)
pass
threads = []
for target in [worker_A, worker_B]:
t = threading.Thread(target = target, args = ({'xxx': 'yyy'}, ))
t.daemon = True
t.start()
threads.append(t)
for t in threads:
t.join()
except Exception, e:
print 'TaskHandler: exception: ', e
pass
self.write_message(json.dumps({'status': 'done'}))
def on_close(self):
pass
class Server(tornado.web.Application):
def __init__(self):
handlers = [
('/task', TaskHandler),
]
tornado.web.Application.__init__(self, handlers)
if __name__ == '__main__':
server = tornado.httpserver.HTTPServer(Server())
server.listen(8765, address = '127.0.0.1')
tornado.ioloop.IOLoop.instance().start()
You block the whole Tornado event loop for 100 seconds in t.join. Unless you have a yield statement or schedule a callback and exit a function, then your function is not asynchronous. Notice how your function "on_message" begins two threads and then calls t.join on each -- how can Tornado's event loop accomplish any other work while your function is waiting for t.join?
Instead, use a ThreadPoolExecutor something like this:
thread_pool = ThreadPoolExecutor(4)
class TaskHandler(tornado.websocket.WebSocketHandler):
# Make this an asynchronous coroutine
#gen.coroutine
def on_message_coroutine(self, message):
print 'received: ', message
self.write_message(json.dumps({'status': 'running'}))
def worker_A(kwargs):
time.sleep(100)
pass
def worker_B(kwargs):
time.sleep(100)
pass
futures = []
for target in [worker_A, worker_B]:
f = thread_pool.submit(target, {'xxx': 'yyy'})
futures.append(future)
# Now the event loop can do other things
yield futures
def on_message(self, message):
IOLoop.current().spawn_callback(self.on_message_coroutine,
message)
I have been experimenting with QRunnable in order to get some service calls up and running. I have stumbled across the following piece of information from the Qt documentation:
QThreadPool supports executing the same QRunnable more than once by
calling tryStart(this) from within QRunnable::run(). If autoDelete is
enabled the QRunnable will be deleted when the last thread exits the
run function. Calling start() multiple times with the same QRunnable
when autoDelete is enabled creates a race condition and is not
recommended.
Can someone explain what this means? I've written the following code and it allows me to execute a QRunnable object multiple times sequentially:
#!/usr/bin/env python
from PyQt4.QtCore import QRunnable, pyqtSlot, pyqtSignal, QObject, QThread, QThreadPool
from PyQt4.QtGui import QApplication, QWidget, QPushButton, QHBoxLayout, QLabel
from sys import exit, argv
from random import getrandbits
class ServiceCallSignals(QObject):
srv_status = pyqtSignal(bool)
srv_running = pyqtSignal(bool)
class ServiceCall(QRunnable):
def __init__(self):
super(ServiceCall, self).__init__()
self.signals = ServiceCallSignals()
def run(self):
self.signals.srv_running.emit(True)
call = bool(getrandbits(1))
print('QRunnable Thread ID: %d' % int(QThread.currentThreadId()))
if call: QThread.sleep(5)
self.signals.srv_status.emit(call)
self.signals.srv_running.emit(False)
class Test(QWidget):
def __init__(self):
super(Test, self).__init__()
self.initUI()
def initUI(self):
layout = QHBoxLayout(self)
self.cb = QPushButton('Send request', self)
self.cb.clicked.connect(self.srv_send)
layout.addWidget(self.cb)
self.lbl = QLabel('Waiting...', self)
layout.addWidget(self.lbl)
self.srv = ServiceCall()
self.srv.setAutoDelete(False)
self.srv.signals.srv_status.connect(self.srv_receive)
self.srv.signals.srv_running.connect(self.srv_block)
self.tp = QThreadPool(self)
self.setGeometry(300, 300, 250, 150)
self.setWindowTitle('QRunnable and ROS service calls')
self.show()
#pyqtSlot()
def srv_send(self):
print('Main Thread ID: %d' % int(QThread.currentThreadId()))
self.tp.start(self.srv)
self.cb.setText('Running for reply')
#pyqtSlot(bool)
def srv_block(self, state):
self.cb.setEnabled(not state)
#pyqtSlot(bool)
def srv_receive(self, srv_res):
if srv_res: self.lbl.setText('Success')
else: self.lbl.setText('Failed')
self.cb.setText('Send request')
def main():
app = QApplication(argv)
t = Test()
exit(app.exec_())
if __name__ == '__main__':
main()
Does the quote from the documentation mean that I'm doing it wrong? If I put my QThreadPool and use tryStart(self) inside my run I get many, many threads running...
The documentation is saying that it supports calling tryStart inside run(), not that it is required.
If you sequentially call start() outside of run() it will re-use the same thread. But if you call tryStart() within run() it may reserve additional threads if necessary.
you need to .setAutoDelete(0) in order to use trystart().
Autodelete(0) means that after execution of a thread it wont be deleted automatically and you should delete it manually when you want.
then when launching/ relaunching the thread by using trystart() instead of start() a race condition is avoided, as the same thread continues.
it's on the documentation.
good luck
Me and a friend are having a programming challenge to who can make a good VOS (Virtual Operating System) and currently mine is running custom programs from Threads within the program, I am using Tkinter currently so the separate Threads have their own self.master.mainloop(). I have all the Threads stored in a list but I was wondering whether I could call a function in the Thread which would call a subroutine in the program telling it to do self.master.destroy(). Is there any way to do this?
I would like something along the lines of
class ToBeThread():
def __init__(self):
self.master = Tk()
self.master.mainloop()
def on_stop(self, reason):
self.master.destroy()
Then in my main class
from threading import Thread
thread = Thread(ToBeThread())
thread.setDaemon(True)
thread.on_stop += ToBeThread.on_stop # Similar to how it is done in c#
thread.start()
...
...
thread.stop() # This calls the functions related to the "on_stop"
I have found a way to do this, so for any wondering I did:
from threading import Thread
class MyThread(Thread):def __init__(self, method, delay=-1):
Thread.__init__(self)
self.method = method
self._running = False
self.delay = delay
self.setDaemon(True)
def run(self):
self._running = True
while self._running == True:
self.method()
if self.delay != -1:
time.sleep(self.delay)
def stop(self):
self._running = False
This allows me to write pass a function in through the initialiser, and it will run it ever x seconds or as many times as possible until I do thread.stop()