I am trying to run some concurrent tasks with asyncio. Currently, i got the following example:
import asyncio
from time import sleep
from signal import SIGINT, SIGTERM, signal
async def f():
print("Got in F")
await asyncio.sleep(10)
print("Finished Sleep in F")
return "f"
async def g():
print("Got in G")
await asyncio.sleep(20)
print("Finished Sleep in G")
return "g"
async def count_timer():
for i in range(20):
print(i)
sleep(1)
async def main():
task_g = asyncio.create_task(g())
task_f = asyncio.create_task(f())
await task_g
await task_f
task_counter = asyncio.create_task(count_timer())
await task_counter
return
if __name__ == "__main__":
import time
s = time.perf_counter()
asyncio.run(main())
elapsed = time.perf_counter() - s
print(f"{__file__} executed in {elapsed:0.2f} seconds.")
What Im trying to do is to call the counter_timer function after the f and g functions were called, but still run all three concurrently.
Thank you in advance,
Lucas Delfino Nogueira.
Related
I need an asynchronous parent process to handover function calls to a process pool.
The imports are to time consuming to spawn a new worker/process every time. So I thought to put tasks in an asyncio.queue and have a consumer listen to it and hand them off to the workers. (Sort of like how Gunicorn works but I don't want to run a webserver in order to make the calls).
However the function call seems to only be executed if I call res.get() on the response of pool.apply_async() but then it just runs as if I would call a normal synchronous for-loop.
This is my code:
#!/usr/bin/env python
import os
import time
import multiprocessing as mp
import asyncio
def f(x: list) -> int:
print(f'the pid of this process is: {os.getpid()}')
time.sleep(1)
return len(x)
def callback_func(x):
print(f'this is the callback function')
print(x)
async def consumer(queue):
with mp.Pool(processes=4) as pool:
while True:
x = await queue.get()
if x == 'stop':
break
# this makes it seem to run synchronous:
res = pool.apply_async(f, (x,))
print(res.get(), x, os.getpid())
# if I run this instead, both f() and callback_func
# are not executed.
#res = pool.apply_async(f, (x,), callback_func)
#print(x, os.getpid())
queue.task_done()
print(f'consumed')
async def producer(queue):
for i in range(20):
await queue.put([i,i+1,i+2])
# await asyncio.sleep(0.5)
await queue.put('stop')
async def main():
queue = asyncio.Queue()
input_coroutines = [consumer(queue), producer(queue)]
for f in asyncio.as_completed(input_coroutines):
try:
result = await f
print(result)
except Exception as e:
print('caught exception')
print(e)
if __name__ == "__main__":
asyncio.run(main())
What am I doing wrong?
During the asyncio learning and tests, I've wrote the code below with 3 concurrent tasks.
import asyncio
from time import time
tasks_to_schedule = []
task_queue = []
class Test():
def __init__(self, task_name, repeat_every):
self.name = task_name
self.repeat_every = repeat_every
self.scheduled = 0
def schedule(self, t_now):
self.scheduled = t_now
async def run(self):
print(f'It is {self.name}')
print(f'{self.name} running...')
await asyncio.sleep(2)
print(f'{self.name} finished')
def check_result(self):
pass
async def report(self):
print(f'{self.name} report DONE')
await asyncio.sleep(1)
def prepare_tasks():
task_a = Test('Task A', 2)
task_b = Test('Task B', 4)
tasks_to_schedule.append(task_a)
tasks_to_schedule.append(task_b)
async def scheduler():
turn = 0
while turn < 5:
if tasks_to_schedule:
print(f'***\t Turn {turn} \t***')
task = tasks_to_schedule.pop(0)
if task.scheduled < time():
task_queue.append(task)
print(f'adding task {task.name} to queue,\n queue size = {len(task_queue)}')
turn += 1
else:
tasks_to_schedule.append(task)
await asyncio.sleep(1)
async def worker(name):
while True:
if task_queue:
task = task_queue.pop(0)
print(f'Worker {name} - took task {task.name}')
await task.run()
await task.report()
print(f'Worker {name} - task {task.name} completed, reschedule it')
task.schedule(time())
tasks_to_schedule.append(task)
# await asyncio.sleep(1) #Process stuck without this line
async def main():
task_scheduler = asyncio.create_task(scheduler())
worker1 = asyncio.create_task(worker(1))
worker2 = asyncio.create_task(worker(2))
await asyncio.gather(task_scheduler, worker1, worker2)
if __name__ == '__main__':
prepare_tasks()
asyncio.run(main())
The problem in process stuck after "Task A running...", the only output is:
*** Turn 0 ***
adding task Task A to queue,
queue size = 1
Worker 1 - took task Task A
It is Task A
Task A running...
After several tries, I've noticed, that with the additional "await asyncio.sleep(1)" line in the end of the loop inside "worker" func the process run correctly without any stuck.
I wonder, what is the reason?
Could someone explain me, please, why this additional line change everything?
Platform: Python 3.9.4, Windows 10 x64, inside venv.
I've added an additional line after:
async def worker(name):
while True:
print(f'{strftime("%X")}: worker loop') #this line
and I can see an endless worker loop in the output...
Now I see, the worker can't find task...
Solved :)
I am just getting started with asyncio in python. What I am trying to do is below :
A Websocket connects to a data provider and keeps listening for new data (Run forever )
Parallelly work on this data and maybe save the data to a file. Make buy /sell decisions(Any parellel operation is okay)
Trying to use asyncio to achieve this (Will threads be better? Although threads seem more complicated that asyncio )
I am Using jupyter notebook( So Event loop is already created, maybe that's the problem ?)
Code 1 works , but this blocks my event loop and only keeps printing the data. All other code is blocked. Since its always busy in the web socket I guess.
import ssl
import websocket
import json
from IPython.display import display, clear_output
from threading import Thread
def on_message(ws, message):
resp1 = json.loads(message)
clear_output(wait=True)
print(resp1['events'][0]['price'])
def run():
ws = websocket.WebSocketApp("wss://api.gemini.com/v1/marketdata/BTCUSD",on_message=on_message)
ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})
ws_run = Thread(target=run)
ws_run.start()
print("ok") # this prints once, but then it gets blocked by the websocket code.
I tried code 2 :
But this hangs forever and doesn't do anything.
import asyncio
import ssl
import websocket
import json
from IPython.display import display, clear_output
async def on_message(ws, message):
resp1 = json.loads(message)
clear_output(wait=True)
#print(resp1,sort_keys=True, indent=4)
#print(resp1['events'][0]['side'])
print(resp1['events'][0]['price'])
async def count2(x):
x=0
for i in range(5):
await asyncio.sleep(0.01)
print('second' , x)
y=x+10
return y
async def main():
await asyncio.gather(count(x), on_message(ws, message))
if __name__ == "__main__":
import time
ws = websocket.WebSocketApp("wss://api.gemini.com/v1/marketdata/BTCUSD",on_message=on_message)
asyncio.get_event_loop().run_forever(ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE}))
s = time.perf_counter()
await main()
elapsed = time.perf_counter() - s
print(f" executed in {elapsed:0.2f} seconds.")
Tried this variant in main(), still no response. :
if __name__ == "__main__":
import time
ws = websocket.WebSocketApp("wss://api.gemini.com/v1/marketdata/BTCUSD",on_message=on_message)
ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})
s = time.perf_counter()
await main()
elapsed = time.perf_counter() - s
print(f" executed in {elapsed:0.2f} seconds.")
Update: I got this to work ,but I dont know if this is the right way, without using run_forever():
import ssl
import websocket
import asyncio
import time
async def MySock():
while True:
print(ws.recv())
await asyncio.sleep(0.5)
async def MyPrint():
while True:
print("-------------------------------------------")
await asyncio.sleep(0.5)
async def main():
await asyncio.gather(MySock(),MyPrint())
if __name__ == "__main__":
ws = websocket.WebSocket()
ws.connect("wss://api.gemini.com/v1/marketdata/btcusd?top_of_book=true&offers=true")
s = time.perf_counter()
await main()
elapsed = time.perf_counter() - s
print(f" executed in {elapsed:0.2f} seconds.")
I want to achieve same effect as
# Code 1
from multiprocessing.pool import ThreadPool as Pool
from time import sleep, time
def square(a):
print('start', a)
sleep(a)
print('end', a)
return a * a
def main():
p = Pool(2)
queue = list(range(4))
start = time()
results = p.map(square, queue)
print(results)
print(time() - start)
if __name__ == "__main__":
main()
with async functions like
# Code 2
from multiprocessing.pool import ThreadPool as Pool
from time import sleep, time
import asyncio
async def square(a):
print('start', a)
sleep(a) # await asyncio.sleep same effect
print('end', a)
return a * a
async def main():
p = Pool(2)
queue = list(range(4))
start = time()
results = p.map_async(square, queue)
results = results.get()
results = [await result for result in results]
print(results)
print(time() - start)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
Currently Code 1 takes 4 seconds and Code 2 takes 6 seconds which means it is not running in parallel. What is the correct and cleanest way to run multiple async functions in parallel?
Better to be python 3.6 compatible. Thank you!
map_async() is not the same "async" as in async def - if it is fed with an async def method, it won't actually run it but return a coroutine instance immediately (try calling such a method without await). Then you awaited on the 4 coroutines one by one, that equals to sequential execution, and ended up with 6 seconds.
Please see following example:
from time import time
import asyncio
from asyncio.locks import Semaphore
semaphore = Semaphore(2)
async def square(a):
async with semaphore:
print('start', a)
await asyncio.sleep(a)
print('end', a)
return a * a
async def main():
start = time()
tasks = []
for a in range(4):
tasks.append(asyncio.ensure_future(square(a)))
await asyncio.wait(tasks)
print([t.result() for t in tasks])
print(time() - start)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
The Semaphore acts similarly like the ThreadPool - it allows only 2 concurrent coroutines entering the async with semaphore: block.
I have a task make communication between coroutines with help asyncio and python3.
Please tell me how to do it,if one coroutine,in while tru cycle , return value at different intervals, and the other coroutines receives this data
import asyncio
#asyncio.coroutine
def write(future):
i=0
while True:
yield from asyncio.sleep(1)
future.set_result('data: '.format(i))
i+=1
def got_result(future):
print(future.result())
loop = asyncio.get_event_loop()
future = asyncio.Future()
asyncio.ensure_future(write(future))
future.add_done_callback(got_result)
try:
loop.run_forever()
finally:
loop.close()
The solution was found with the help of the asyncio.Queue()
import asyncio
#asyncio.coroutine
def get_work(task, work_queue):
while not work_queue.empty():
print(task)
queue_item = yield from work_queue.get()
print('{0} grabbed item: {1}'.format(task, queue_item))
yield from asyncio.sleep(0.5)
asyncio.async(get_work(task, work_queue))
# #asyncio.coroutine
i = 0
async def do_work(task, work_queue):
global i
print(task)
while work_queue.empty():
work_queue.put_nowait(i)
i += 1
await asyncio.sleep(2)
break
# asyncio.async(do_work())
print("Dfgdfg")
asyncio.ensure_future(do_work(task, work_queue))
if __name__ == "__main__":
queue_obj = asyncio.Queue()
loop = asyncio.get_event_loop()
tasks = [
asyncio.async(do_work('Run do_work', queue_obj)),
asyncio.async(get_work('Run get_work', queue_obj))]
loop.run_until_complete(asyncio.wait(tasks))
loop.run_forever()