I have a problem at work where I have to wait for 10 seconds when InstrInstallSucceeded event comes in, without blocking the main thread, I should wait for InstrInstallFailed to appear, so in other words 'ToolOn', 'ToolOn', 'ToolOn' should appear without any wait.
import asyncio
from threading import Thread
import time
FLAG = True
async def sleep_loop(t, event):
global FLAG
print(event)
if event == 'InstrInstallSucceeded':
# spwan a seperate thread here such that
# toolon events are not blocked by the sleep
await asyncio.sleep(t)
FLAG = True
if event == 'InstrInstallFailed':
# and I want to update the FLAG whenever I see event == 'InstrInstallFailed'
FLAG = False
async def keep_print():
print(f'Beginning FLAG:: {FLAG}')
while FLAG:
pass
print(f'End FLAG:: {FLAG}')
def start_loop(loop, t):
print("in start loop")
asyncio.set_event_loop(loop)
for i in ['InstrInstallSucceeded', 'ToolOn','ToolOn', 'ToolOn', 'InstrInstallFailed']:
loop.run_until_complete(asyncio.sleep(1))
loop.run_until_complete(sleep_loop(t, i))
loop = asyncio.get_event_loop()
new_loop = asyncio.new_event_loop()
t = Thread(target=start_loop, args=(new_loop,10))
t.start()
coro = keep_print()
loop.run_until_complete(coro)
output
in start loop
Beginning FLAG:: True
Executing <Task pending coro=<sleep() running at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/tasks.py:482> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object at 0x1043f2be8>()] created at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:284> cb=[_run_until_complete_cb() at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:185] created at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:452> took 0.118 seconds
InstrInstallSucceeded
ToolOn
ToolOn
ToolOn
InstrInstallFailed
End FLAG:: False
Executing <Task finished coro=<keep_print() done, defined at fut.py:21> result=None created at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:452> took 15.756 seconds
EDIT: using python 3.6.7
import asyncio
async def dispatch_event(event, alert):
print(event)
if event == 'InstrInstallSucceeded':
# spawn a coroutine if you need something done in parallel
#asyncio.create_task(xxx())
await asyncio.sleep(10)
if event == 'InstrInstallFailed':
await asyncio.sleep(.5)
# alert the watcher(s) of the event that was dispatched
alert.last_event = event
alert.set()
async def keep_print(alert):
while True:
print(f'Beginning FLAG:: {alert.last_event}')
await alert.wait()
alert.clear()
print(f'End FLAG:: {alert.last_event}')
async def main():
alert = asyncio.Event()
alert.last_event = None
# spawn keep_print in the "background"
loop = asyncio.get_event_loop()
t = loop.create_task(keep_print(alert))
for i in ['InstrInstallSucceeded', 'ToolOn','ToolOn', 'ToolOn', 'InstrInstallFailed']:
await asyncio.sleep(1)
await dispatch_event(i, alert)
await asyncio.sleep(1)
t.cancel()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
edit as suggested by #user418.....
async def dispatch_event(event,alert):
alert.last_event = event
alert.set()
print(event)
if event == 'InstrInstallSucceeded':
# spawn a coroutine if you need something done in parallel
#asyncio.create_task(xxx())
await asyncio.sleep(10)
if event == 'InstrInstallFailed':
await asyncio.sleep(.5)
# alert the watcher(s) of the event that was dispatched
Threads and asyncio don't go together, except in specific circumstances (e.g. the implementation of run_in_executor). Instead of spawning new threads, spawn new coroutines.
For example:
import asyncio
async def dispatch_event(event, alert):
print(event)
if event == 'InstrInstallSucceeded':
# spawn a coroutine if you need something done in parallel
#asyncio.create_task(xxx())
await asyncio.sleep(1)
if event == 'InstrInstallFailed':
await asyncio.sleep(.5)
# alert the watcher(s) of the event that was dispatched
alert.last_event = event
alert.set()
async def keep_print(alert):
while True:
print(f'Beginning FLAG:: {alert.last_event}')
await alert.wait()
alert.clear()
print(f'End FLAG:: {alert.last_event}')
async def main():
alert = asyncio.Event()
alert.last_event = None
# spawn keep_print in the "background"
t = asyncio.create_task(keep_print(alert))
for i in ['InstrInstallSucceeded', 'ToolOn','ToolOn', 'ToolOn', 'InstrInstallFailed']:
await asyncio.sleep(1)
await dispatch_event(i, alert)
await asyncio.sleep(1)
t.cancel()
asyncio.run(main())
Related
I'm running an async function(download files) in a separate process. The data between processes is
passed using multiprocessing queues.
in main file/module:
to_download_mpq = Queue()
downloaded_mpq = Queue()
...................
pd = Process(target=async_download_items,
args=(to_download_mpq, downloaded_mpq))
pd.start()
pd.join()
print("end_process")
in a separate file:
# transfer data between async queues and multiprocessing queues when is ready
async def download_piping(to_download_q, downloaded_q, to_download_mpq, downloaded_mpq, sentinel):
to_download_sentinel = False
downloaded_sentinel = False
while not (to_download_sentinel and downloaded_sentinel):
if not to_download_sentinel:
to_download_item = to_download_mpq.get()
await to_download_q.put(to_download_item)
await asyncio.sleep(0.1)
if to_download_item == sentinel:
to_download_sentinel = True
if not downloaded_sentinel:
if downloaded_q.empty():
await asyncio.sleep(2)
else:
downloaded_item = await downloaded_q.get()
downloaded_mpq.put(downloaded_item)
await asyncio.sleep(0)
if downloaded_item == sentinel:
downloaded_sentinel = True
async def download_tasks(to_download_mpq, downloaded_mpq, workers, sentinel=END_QUEUE_SENTINEL, queue_size=50):
downloader = download.AsyncDownloader() # async downloader class using aiohttp
to_download_lq = asyncio.Queue(queue_size)
downloaded_lq = asyncio.Queue(queue_size)
task_download = asyncio.create_task(downloader.download_files(to_download_q=to_download_lq,
downloaded_q=downloaded_lq,
download_workers=workers,
sentinel=sentinel))
task_piping = asyncio.create_task(download_piping(to_download_lq, downloaded_lq,
to_download_mpq, downloaded_mpq,
sentinel=sentinel))
await asyncio.gather(task_download, task_piping)
def async_download_items(to_download_mpq, downloaded_mpq, workers=50):
loop = asyncio.get_event_loop()
print(loop)
loop.run_until_complete(download_tasks(to_download_mpq, downloaded_mpq, workers=workers))
loop.close()
print("end async")
The async tasks, loop finishes, "end async" is printed but the process hangs. The data form "downloaded_mpq" is used in another process.
Are the queues keeping doesn't allow the process to close or can be something else?
I want to run a coroutine in a different thread and get the result that the coroutine returns.
class Main:
def __init__(self, result_from_io_task=None):
self._io_task_result = result_from_io_task
async def io_task(self):
await asyncio.sleep(2)
return "slept of 2s"
def non_async_func(self):
#This can't be made async.
if not self._io_task_result:
#run io_task and get its result
#event loop will be running in the main thread so I can fire the task
task = asyncio.create_task(self.io_task)
#can't await task since I am in non-async func and I cannot
#return from non_async_func until and unless I know what
#self.io_task has returned. Tried following but my app hangs forever.
while not task.done():
pass
I also tried, but it doesn't work "
def run_in_thread(coro, loop):
output = []
def run():
fut = asyncio.run_coroutine_threadsafe(coro, loop)
output.append(fut)
thr = Thread(target=run)
thr.start()
return output
async def main():
main_obj = Main(result_from_io_task=None)
v = main_obj.non_async_func()
How can I spawn a new thread and run the given coroutine using event loop running in main thread
Unfortunately, my codebase depends on python < 3.8 and asyncio.to_thread is not available in python 3.7
Based on the example of my answer, I'm introducing another implementation of the asynchronous decorator that does not use asyncio.to_thread() function but uses ThreadPoolExecutor instead.
import asyncio
import requests
import concurrent.futures
def asynchronous(func):
async def wrapper(*args, **kwargs):
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(func, *args, **kwargs)
return future.result()
return wrapper
#asynchronous
def request(url):
with requests.Session() as session:
response = session.get(url)
try:
return response.json()
except requests.JSONDecodeError:
return response.text
async def main():
task = asyncio.create_task(request("https://google.com/"))
print("waiting for response...")
result = await task
print(result)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
My Scenario:- Start, Wait, Start, Stop or Kill
Starting the first event & waiting for some time.
If I reach the waiting time, I need to start the second event & return both event result.
But, if the first event completed before waiting time.
No need to start the second event.
Return the first event result
Ex:-
import asyncio
async def some_task():
print('io start')
await asyncio.sleep(2)
print('io end')
return "hello"
async def callback(loop):
await asyncio.sleep(4)
if loop.is_running():
print('doing other things')
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop2 = asyncio.get_event_loop()
a = loop.create_task(some_task())
b = loop2.create_task(callback(loop))
result = loop.run_until_complete(a)
loop2.run_until_complete(b)
loop.close()
loop2.close()
The variables loop and loop2 will get the same (main) event loop, which is why it will always still be running. Here is my approach:
import asyncio
async def async_main():
result = None
async def some_task():
print('io start')
await asyncio.sleep(6)
print('io end')
result = "hello"
return result
async def callback():
await asyncio.sleep(4)
if result is None:
print('doing other things')
awaited = await asyncio.gather(
asyncio.create_task(some_task()),
asyncio.create_task(callback()),
)
return awaited
asyncio.run(async_main())
Asyncio can be very confusing and I do not recommend it for non-experts, so here is an alternative using threading instead of asyncio:
import threading
import time
def do_tasks():
result = None
def some_task():
nonlocal result
print('io start')
time.sleep(2)
print('io end')
result = "hello"
def callback():
time.sleep(4)
if result is None:
print('doing other things')
threading.Thread(target=some_task).start()
threading.Thread(target=callback).start()
do_tasks()
an asyncio program has two task that produce messages which are put on a queue, another task consume the queue.
one producer produce periodic task.
the other producer has to be synced with the consumer, it has to await till its own message have been consumed
import asyncio
import logging
import sys
logging.basicConfig( stream=sys.stdout,format='%(asctime)-5s: %(funcName)-15s: %(message)s',datefmt='%I:%M:%S',level=logging.INFO)
logger = logging.getLogger()
async def sync_producer(queue):
for x in range(5):
item = f"sync producer{x}"
logger.info(f"{item} ")
await queue.put(item)# <= at this point I want to await that the message have been consumed
logger.info(f"sync producer finish")
async def periodic_producer(queue):
x=0
while True:
await asyncio.sleep(1)
item = f"periodic producer {x}"
logger.info(f"{item} ")
queue.put_nowait(item)
x+=1
async def consumer(queue):
while True:
item = await queue.get()
logger.info(f"{item}")
queue.task_done()
await asyncio.sleep(1)
async def main():
queue = asyncio.Queue()
consumer_task = asyncio.create_task(consumer(queue))
periodic_producer_task = asyncio.create_task(periodic_producer(queue))
producer_task = asyncio.create_task(sync_producer(queue))
await producer_task
periodic_producer_task.cancel()
await queue.join()
consumer_task.cancel()
asyncio.run(main())
The example does not work as i want beacause await queue.put(item) does'not await queue task_done().
A possible workaround could be to put on the queue (event,item) where event = asyncio.Event() and then await event. Is that a "good" workaraound?
async def start(channel):
while True:
m = await client.send_message(channel, "Generating... ")
generator.makeFile()
with open('tmp.png', 'rb') as f:
await client.send_file(channel, f)
await client.delete_message(m)
await asyncio.sleep(2)
I have a discord bot that runs a task every 2 seconds. I tried using an infinite loop for this, but the script crashes with a Task was destroyed but it is still pending! I have read about asyncio's coroutines, but none of the examples that I found use await in them. Is it possible avoid this error, by running a coroutine with await, for example?
Task was destroyed but it is still pending! is warning that you receive when you call loop.close() when some of tasks in your script aren't finished. Usually you should avoid this situation because unfinished task may not release some resources. You need either to await task done or cancel it before event loop closed.
Since you have infinite loop you probably would need to cancel task, example:
import asyncio
from contextlib import suppress
async def start():
# your infinite loop here, for example:
while True:
print('echo')
await asyncio.sleep(1)
async def main():
task = asyncio.Task(start())
# let script some thime to work:
await asyncio.sleep(3)
# cancel task to avoid warning:
task.cancel()
with suppress(asyncio.CancelledError):
await task # await for task cancellation
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_until_complete(main())
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
See also this answer for more information about tasks.