asyncio queue multi producer (sync) single consumer - python-3.x

an asyncio program has two task that produce messages which are put on a queue, another task consume the queue.
one producer produce periodic task.
the other producer has to be synced with the consumer, it has to await till its own message have been consumed
import asyncio
import logging
import sys
logging.basicConfig( stream=sys.stdout,format='%(asctime)-5s: %(funcName)-15s: %(message)s',datefmt='%I:%M:%S',level=logging.INFO)
logger = logging.getLogger()
async def sync_producer(queue):
for x in range(5):
item = f"sync producer{x}"
logger.info(f"{item} ")
await queue.put(item)# <= at this point I want to await that the message have been consumed
logger.info(f"sync producer finish")
async def periodic_producer(queue):
x=0
while True:
await asyncio.sleep(1)
item = f"periodic producer {x}"
logger.info(f"{item} ")
queue.put_nowait(item)
x+=1
async def consumer(queue):
while True:
item = await queue.get()
logger.info(f"{item}")
queue.task_done()
await asyncio.sleep(1)
async def main():
queue = asyncio.Queue()
consumer_task = asyncio.create_task(consumer(queue))
periodic_producer_task = asyncio.create_task(periodic_producer(queue))
producer_task = asyncio.create_task(sync_producer(queue))
await producer_task
periodic_producer_task.cancel()
await queue.join()
consumer_task.cancel()
asyncio.run(main())
The example does not work as i want beacause await queue.put(item) does'not await queue task_done().
A possible workaround could be to put on the queue (event,item) where event = asyncio.Event() and then await event. Is that a "good" workaraound?

Related

Async loop running in another process; tasks and loop are closed but the process is hanging

I'm running an async function(download files) in a separate process. The data between processes is
passed using multiprocessing queues.
in main file/module:
to_download_mpq = Queue()
downloaded_mpq = Queue()
...................
pd = Process(target=async_download_items,
args=(to_download_mpq, downloaded_mpq))
pd.start()
pd.join()
print("end_process")
in a separate file:
# transfer data between async queues and multiprocessing queues when is ready
async def download_piping(to_download_q, downloaded_q, to_download_mpq, downloaded_mpq, sentinel):
to_download_sentinel = False
downloaded_sentinel = False
while not (to_download_sentinel and downloaded_sentinel):
if not to_download_sentinel:
to_download_item = to_download_mpq.get()
await to_download_q.put(to_download_item)
await asyncio.sleep(0.1)
if to_download_item == sentinel:
to_download_sentinel = True
if not downloaded_sentinel:
if downloaded_q.empty():
await asyncio.sleep(2)
else:
downloaded_item = await downloaded_q.get()
downloaded_mpq.put(downloaded_item)
await asyncio.sleep(0)
if downloaded_item == sentinel:
downloaded_sentinel = True
async def download_tasks(to_download_mpq, downloaded_mpq, workers, sentinel=END_QUEUE_SENTINEL, queue_size=50):
downloader = download.AsyncDownloader() # async downloader class using aiohttp
to_download_lq = asyncio.Queue(queue_size)
downloaded_lq = asyncio.Queue(queue_size)
task_download = asyncio.create_task(downloader.download_files(to_download_q=to_download_lq,
downloaded_q=downloaded_lq,
download_workers=workers,
sentinel=sentinel))
task_piping = asyncio.create_task(download_piping(to_download_lq, downloaded_lq,
to_download_mpq, downloaded_mpq,
sentinel=sentinel))
await asyncio.gather(task_download, task_piping)
def async_download_items(to_download_mpq, downloaded_mpq, workers=50):
loop = asyncio.get_event_loop()
print(loop)
loop.run_until_complete(download_tasks(to_download_mpq, downloaded_mpq, workers=workers))
loop.close()
print("end async")
The async tasks, loop finishes, "end async" is printed but the process hangs. The data form "downloaded_mpq" is used in another process.
Are the queues keeping doesn't allow the process to close or can be something else?

Need to parse two sessions at the same time with telethon on Python

i have some troubles with parsing two or more sessions at the same time with telethon. I have tried this:
class NewSession:
def __init__(self, session_name):
self.client = TelegramClient(session_name, api_id, api_hash)
self.session_name = session_name
async def pool(self):
print("working with:", self.session_name)
#self.client.on(events.NewMessage(outgoing=True))
async def main(event):
message = event.message.to_dict()
msg_text = message['message']
print(msg_text)
try:
await self.client.start()
await self.client.run_until_disconnected()
finally:
await self.client.disconnect()
async def main():
user = NewSession("321")
user2 = NewSession("123")
await user.pool()
await user2.pool()
if __name__ == '__main__':
asyncio.run(main())
But only one is working. Need help :)
The problem is inside your main function. When you await for a coroutine to return it doesn't mean that the execution continues to the next expression. So, in your code the line await user2.pool() is going to be executed only when the user.poll() coroutines returns a value, this is when the session '321' is disconnected.
You need to run the tasks concurrently; you can use the function asyncio.gather. Reworking your main:
async def main():
user = NewSession("321")
user2 = NewSession("123")
await asyncio.gather(user.pool(), user2.pool())

Concurrency Loop in python

My Scenario:- Start, Wait, Start, Stop or Kill
Starting the first event & waiting for some time.
If I reach the waiting time, I need to start the second event & return both event result.
But, if the first event completed before waiting time.
No need to start the second event.
Return the first event result
Ex:-
import asyncio
async def some_task():
print('io start')
await asyncio.sleep(2)
print('io end')
return "hello"
async def callback(loop):
await asyncio.sleep(4)
if loop.is_running():
print('doing other things')
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop2 = asyncio.get_event_loop()
a = loop.create_task(some_task())
b = loop2.create_task(callback(loop))
result = loop.run_until_complete(a)
loop2.run_until_complete(b)
loop.close()
loop2.close()
The variables loop and loop2 will get the same (main) event loop, which is why it will always still be running. Here is my approach:
import asyncio
async def async_main():
result = None
async def some_task():
print('io start')
await asyncio.sleep(6)
print('io end')
result = "hello"
return result
async def callback():
await asyncio.sleep(4)
if result is None:
print('doing other things')
awaited = await asyncio.gather(
asyncio.create_task(some_task()),
asyncio.create_task(callback()),
)
return awaited
asyncio.run(async_main())
Asyncio can be very confusing and I do not recommend it for non-experts, so here is an alternative using threading instead of asyncio:
import threading
import time
def do_tasks():
result = None
def some_task():
nonlocal result
print('io start')
time.sleep(2)
print('io end')
result = "hello"
def callback():
time.sleep(4)
if result is None:
print('doing other things')
threading.Thread(target=some_task).start()
threading.Thread(target=callback).start()
do_tasks()

wait in separate thread without blocking main eventloop, async python

I have a problem at work where I have to wait for 10 seconds when InstrInstallSucceeded event comes in, without blocking the main thread, I should wait for InstrInstallFailed to appear, so in other words 'ToolOn', 'ToolOn', 'ToolOn' should appear without any wait.
import asyncio
from threading import Thread
import time
FLAG = True
async def sleep_loop(t, event):
global FLAG
print(event)
if event == 'InstrInstallSucceeded':
# spwan a seperate thread here such that
# toolon events are not blocked by the sleep
await asyncio.sleep(t)
FLAG = True
if event == 'InstrInstallFailed':
# and I want to update the FLAG whenever I see event == 'InstrInstallFailed'
FLAG = False
async def keep_print():
print(f'Beginning FLAG:: {FLAG}')
while FLAG:
pass
print(f'End FLAG:: {FLAG}')
def start_loop(loop, t):
print("in start loop")
asyncio.set_event_loop(loop)
for i in ['InstrInstallSucceeded', 'ToolOn','ToolOn', 'ToolOn', 'InstrInstallFailed']:
loop.run_until_complete(asyncio.sleep(1))
loop.run_until_complete(sleep_loop(t, i))
loop = asyncio.get_event_loop()
new_loop = asyncio.new_event_loop()
t = Thread(target=start_loop, args=(new_loop,10))
t.start()
coro = keep_print()
loop.run_until_complete(coro)
output
in start loop
Beginning FLAG:: True
Executing <Task pending coro=<sleep() running at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/tasks.py:482> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object at 0x1043f2be8>()] created at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:284> cb=[_run_until_complete_cb() at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:185] created at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:452> took 0.118 seconds
InstrInstallSucceeded
ToolOn
ToolOn
ToolOn
InstrInstallFailed
End FLAG:: False
Executing <Task finished coro=<keep_print() done, defined at fut.py:21> result=None created at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:452> took 15.756 seconds
EDIT: using python 3.6.7
import asyncio
async def dispatch_event(event, alert):
print(event)
if event == 'InstrInstallSucceeded':
# spawn a coroutine if you need something done in parallel
#asyncio.create_task(xxx())
await asyncio.sleep(10)
if event == 'InstrInstallFailed':
await asyncio.sleep(.5)
# alert the watcher(s) of the event that was dispatched
alert.last_event = event
alert.set()
async def keep_print(alert):
while True:
print(f'Beginning FLAG:: {alert.last_event}')
await alert.wait()
alert.clear()
print(f'End FLAG:: {alert.last_event}')
async def main():
alert = asyncio.Event()
alert.last_event = None
# spawn keep_print in the "background"
loop = asyncio.get_event_loop()
t = loop.create_task(keep_print(alert))
for i in ['InstrInstallSucceeded', 'ToolOn','ToolOn', 'ToolOn', 'InstrInstallFailed']:
await asyncio.sleep(1)
await dispatch_event(i, alert)
await asyncio.sleep(1)
t.cancel()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
edit as suggested by #user418.....
async def dispatch_event(event,alert):
alert.last_event = event
alert.set()
print(event)
if event == 'InstrInstallSucceeded':
# spawn a coroutine if you need something done in parallel
#asyncio.create_task(xxx())
await asyncio.sleep(10)
if event == 'InstrInstallFailed':
await asyncio.sleep(.5)
# alert the watcher(s) of the event that was dispatched
Threads and asyncio don't go together, except in specific circumstances (e.g. the implementation of run_in_executor). Instead of spawning new threads, spawn new coroutines.
For example:
import asyncio
async def dispatch_event(event, alert):
print(event)
if event == 'InstrInstallSucceeded':
# spawn a coroutine if you need something done in parallel
#asyncio.create_task(xxx())
await asyncio.sleep(1)
if event == 'InstrInstallFailed':
await asyncio.sleep(.5)
# alert the watcher(s) of the event that was dispatched
alert.last_event = event
alert.set()
async def keep_print(alert):
while True:
print(f'Beginning FLAG:: {alert.last_event}')
await alert.wait()
alert.clear()
print(f'End FLAG:: {alert.last_event}')
async def main():
alert = asyncio.Event()
alert.last_event = None
# spawn keep_print in the "background"
t = asyncio.create_task(keep_print(alert))
for i in ['InstrInstallSucceeded', 'ToolOn','ToolOn', 'ToolOn', 'InstrInstallFailed']:
await asyncio.sleep(1)
await dispatch_event(i, alert)
await asyncio.sleep(1)
t.cancel()
asyncio.run(main())

run async db read task in background

I have function called get_active_allowed_systems_by_poll where I want to call it in the background 10 times/minute and refresh new systems that I have gotten in last 10 seconds.
import asyncio
from threading import Thread
async def create_state_machine_at_init(app):
worker_loop = asyncio.new_event_loop()
worker = Thread(target=start_db_worker, args=(worker_loop,))
worker.start()
worker_loop.call_soon_threadsafe(get_active_allowed_systems_by_poll, app, 30)
async def get_active_allowed_systems_by_poll(app, interval=10):
params = {
param: key
for param, key
in app.config.get_active_allowed_systems_by_poll_params.items()
}
params['interval'] = interval
operation = prepare_exec(
app.config.get_active_allowed_systems_by_poll,
**params
)
ACTIVE_ALLOWED_SYSTEMS
ACTIVE_ALLOWED_SYSTEMS = (await app['database'].execute(operation)).all()
return ACTIVE_ALLOWED_SYSTEMS
def start_db_worker(loop):
"""Switch to new event loop and run forever"""
asyncio.set_event_loop(loop)
loop.run_forever()
/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/events.py:145: RuntimeWarning: coroutine 'get_active_allowed_systems_by_poll' was never awaited

Resources