Concurrency Loop in python - python-3.x

My Scenario:- Start, Wait, Start, Stop or Kill
Starting the first event & waiting for some time.
If I reach the waiting time, I need to start the second event & return both event result.
But, if the first event completed before waiting time.
No need to start the second event.
Return the first event result
Ex:-
import asyncio
async def some_task():
print('io start')
await asyncio.sleep(2)
print('io end')
return "hello"
async def callback(loop):
await asyncio.sleep(4)
if loop.is_running():
print('doing other things')
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop2 = asyncio.get_event_loop()
a = loop.create_task(some_task())
b = loop2.create_task(callback(loop))
result = loop.run_until_complete(a)
loop2.run_until_complete(b)
loop.close()
loop2.close()

The variables loop and loop2 will get the same (main) event loop, which is why it will always still be running. Here is my approach:
import asyncio
async def async_main():
result = None
async def some_task():
print('io start')
await asyncio.sleep(6)
print('io end')
result = "hello"
return result
async def callback():
await asyncio.sleep(4)
if result is None:
print('doing other things')
awaited = await asyncio.gather(
asyncio.create_task(some_task()),
asyncio.create_task(callback()),
)
return awaited
asyncio.run(async_main())
Asyncio can be very confusing and I do not recommend it for non-experts, so here is an alternative using threading instead of asyncio:
import threading
import time
def do_tasks():
result = None
def some_task():
nonlocal result
print('io start')
time.sleep(2)
print('io end')
result = "hello"
def callback():
time.sleep(4)
if result is None:
print('doing other things')
threading.Thread(target=some_task).start()
threading.Thread(target=callback).start()
do_tasks()

Related

Running coroutines in different thread with same event loop

I want to run a coroutine in a different thread and get the result that the coroutine returns.
class Main:
def __init__(self, result_from_io_task=None):
self._io_task_result = result_from_io_task
async def io_task(self):
await asyncio.sleep(2)
return "slept of 2s"
def non_async_func(self):
#This can't be made async.
if not self._io_task_result:
#run io_task and get its result
#event loop will be running in the main thread so I can fire the task
task = asyncio.create_task(self.io_task)
#can't await task since I am in non-async func and I cannot
#return from non_async_func until and unless I know what
#self.io_task has returned. Tried following but my app hangs forever.
while not task.done():
pass
I also tried, but it doesn't work "
def run_in_thread(coro, loop):
output = []
def run():
fut = asyncio.run_coroutine_threadsafe(coro, loop)
output.append(fut)
thr = Thread(target=run)
thr.start()
return output
async def main():
main_obj = Main(result_from_io_task=None)
v = main_obj.non_async_func()
How can I spawn a new thread and run the given coroutine using event loop running in main thread
Unfortunately, my codebase depends on python < 3.8 and asyncio.to_thread is not available in python 3.7
Based on the example of my answer, I'm introducing another implementation of the asynchronous decorator that does not use asyncio.to_thread() function but uses ThreadPoolExecutor instead.
import asyncio
import requests
import concurrent.futures
def asynchronous(func):
async def wrapper(*args, **kwargs):
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(func, *args, **kwargs)
return future.result()
return wrapper
#asynchronous
def request(url):
with requests.Session() as session:
response = session.get(url)
try:
return response.json()
except requests.JSONDecodeError:
return response.text
async def main():
task = asyncio.create_task(request("https://google.com/"))
print("waiting for response...")
result = await task
print(result)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()

Need to parse two sessions at the same time with telethon on Python

i have some troubles with parsing two or more sessions at the same time with telethon. I have tried this:
class NewSession:
def __init__(self, session_name):
self.client = TelegramClient(session_name, api_id, api_hash)
self.session_name = session_name
async def pool(self):
print("working with:", self.session_name)
#self.client.on(events.NewMessage(outgoing=True))
async def main(event):
message = event.message.to_dict()
msg_text = message['message']
print(msg_text)
try:
await self.client.start()
await self.client.run_until_disconnected()
finally:
await self.client.disconnect()
async def main():
user = NewSession("321")
user2 = NewSession("123")
await user.pool()
await user2.pool()
if __name__ == '__main__':
asyncio.run(main())
But only one is working. Need help :)
The problem is inside your main function. When you await for a coroutine to return it doesn't mean that the execution continues to the next expression. So, in your code the line await user2.pool() is going to be executed only when the user.poll() coroutines returns a value, this is when the session '321' is disconnected.
You need to run the tasks concurrently; you can use the function asyncio.gather. Reworking your main:
async def main():
user = NewSession("321")
user2 = NewSession("123")
await asyncio.gather(user.pool(), user2.pool())

How can I change the number of coroutines in the middle of the process?

I want to make async processing in python3.8,
How can I change the number of coroutines in the middle of the process?
Is it possible to change Semaphore by changing the number of limits along the way?
async def exec(id1,id2):
print(f'start: {id1} end: {id2}')
loop = asyncio.get_event_loop()
func = functools.partial(run_insert,id1,id2)
await loop.run_in_executor(None, func)
async def parallel_call(s2elist,limit):
sem = asyncio.Semaphore(limit)
async def call(start,end):
async with sem:
return await exec(start,end)
return await asyncio.gather(*[call(s,e) for s,e in s2elist])
if __name__ == '__main__':
s2elist = [[1000,2000],
[2000,3000],
[3000,4000],
[4000,5000],
[5000,6000],
[6000,7000],
[7000,8000],
[8000,9000],
[9000,10000]
]
loop = asyncio.get_event_loop()
loop.run_until_complete(parallel_call(s2elist,10))

wait in separate thread without blocking main eventloop, async python

I have a problem at work where I have to wait for 10 seconds when InstrInstallSucceeded event comes in, without blocking the main thread, I should wait for InstrInstallFailed to appear, so in other words 'ToolOn', 'ToolOn', 'ToolOn' should appear without any wait.
import asyncio
from threading import Thread
import time
FLAG = True
async def sleep_loop(t, event):
global FLAG
print(event)
if event == 'InstrInstallSucceeded':
# spwan a seperate thread here such that
# toolon events are not blocked by the sleep
await asyncio.sleep(t)
FLAG = True
if event == 'InstrInstallFailed':
# and I want to update the FLAG whenever I see event == 'InstrInstallFailed'
FLAG = False
async def keep_print():
print(f'Beginning FLAG:: {FLAG}')
while FLAG:
pass
print(f'End FLAG:: {FLAG}')
def start_loop(loop, t):
print("in start loop")
asyncio.set_event_loop(loop)
for i in ['InstrInstallSucceeded', 'ToolOn','ToolOn', 'ToolOn', 'InstrInstallFailed']:
loop.run_until_complete(asyncio.sleep(1))
loop.run_until_complete(sleep_loop(t, i))
loop = asyncio.get_event_loop()
new_loop = asyncio.new_event_loop()
t = Thread(target=start_loop, args=(new_loop,10))
t.start()
coro = keep_print()
loop.run_until_complete(coro)
output
in start loop
Beginning FLAG:: True
Executing <Task pending coro=<sleep() running at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/tasks.py:482> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object at 0x1043f2be8>()] created at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:284> cb=[_run_until_complete_cb() at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:185] created at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:452> took 0.118 seconds
InstrInstallSucceeded
ToolOn
ToolOn
ToolOn
InstrInstallFailed
End FLAG:: False
Executing <Task finished coro=<keep_print() done, defined at fut.py:21> result=None created at /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py:452> took 15.756 seconds
EDIT: using python 3.6.7
import asyncio
async def dispatch_event(event, alert):
print(event)
if event == 'InstrInstallSucceeded':
# spawn a coroutine if you need something done in parallel
#asyncio.create_task(xxx())
await asyncio.sleep(10)
if event == 'InstrInstallFailed':
await asyncio.sleep(.5)
# alert the watcher(s) of the event that was dispatched
alert.last_event = event
alert.set()
async def keep_print(alert):
while True:
print(f'Beginning FLAG:: {alert.last_event}')
await alert.wait()
alert.clear()
print(f'End FLAG:: {alert.last_event}')
async def main():
alert = asyncio.Event()
alert.last_event = None
# spawn keep_print in the "background"
loop = asyncio.get_event_loop()
t = loop.create_task(keep_print(alert))
for i in ['InstrInstallSucceeded', 'ToolOn','ToolOn', 'ToolOn', 'InstrInstallFailed']:
await asyncio.sleep(1)
await dispatch_event(i, alert)
await asyncio.sleep(1)
t.cancel()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
edit as suggested by #user418.....
async def dispatch_event(event,alert):
alert.last_event = event
alert.set()
print(event)
if event == 'InstrInstallSucceeded':
# spawn a coroutine if you need something done in parallel
#asyncio.create_task(xxx())
await asyncio.sleep(10)
if event == 'InstrInstallFailed':
await asyncio.sleep(.5)
# alert the watcher(s) of the event that was dispatched
Threads and asyncio don't go together, except in specific circumstances (e.g. the implementation of run_in_executor). Instead of spawning new threads, spawn new coroutines.
For example:
import asyncio
async def dispatch_event(event, alert):
print(event)
if event == 'InstrInstallSucceeded':
# spawn a coroutine if you need something done in parallel
#asyncio.create_task(xxx())
await asyncio.sleep(1)
if event == 'InstrInstallFailed':
await asyncio.sleep(.5)
# alert the watcher(s) of the event that was dispatched
alert.last_event = event
alert.set()
async def keep_print(alert):
while True:
print(f'Beginning FLAG:: {alert.last_event}')
await alert.wait()
alert.clear()
print(f'End FLAG:: {alert.last_event}')
async def main():
alert = asyncio.Event()
alert.last_event = None
# spawn keep_print in the "background"
t = asyncio.create_task(keep_print(alert))
for i in ['InstrInstallSucceeded', 'ToolOn','ToolOn', 'ToolOn', 'InstrInstallFailed']:
await asyncio.sleep(1)
await dispatch_event(i, alert)
await asyncio.sleep(1)
t.cancel()
asyncio.run(main())

Why does asyncio.create_task not run the method?

Code example:
async def download_page(session, url):
print(True)
async def downloader_init(session):
while True:
url = await download_queue.get()
task = asyncio.create_task(download_page(session, url))
print(task)
print(f"url: {url}")
async def get_urls(url):
while True:
try:
url = find_somewhere_url
await download_queue.put(url)
except NoSuchElementException:
break
return True
async def main():
async with aiohttp.ClientSession(headers=headers) as session:
get_urls_task = asyncio.create_task(get_urls(url))
downloader_init_task = asyncio.create_task(downloader_init(session))
asyncio.gather(get_urls_task, downloader_init_task)
if __name__ == "__main__":
asyncio.get_event_loop().run_until_complete(main())
Output:
<Task pending coro=<download_page() running at main.py:69>>
url: https://someurl.com/example
<Task pending coro=<download_page() running at main.py:69>>
url: https://someurl.com/example
<Task pending coro=<download_page() running at main.py:69>>
url: https://someurl.com/example
Why is the method download_page not executed?
The strange thing is that the script just ends its work, there are no errors anywhere.
downloader_init should work endlessly, but it does not.
In download_queue, method get_urls adds links as it finds them, after which it stops working.
downloader_init should immediately execute as soon as a new link appears in the queue, but it starts its work only when get_urls has completed its work.
Try this instead:
Note: Your problem wasn't with the task creation, it was because
there wasn't an await at the asyncio.gather part.
import asyncio
import aiohttp
async def download_page(session, url):
# Dummy function.
print(f"session={session}, url={url}")
async def downloader_init(session):
while True:
url = await download_queue.get()
task = asyncio.create_task(download_page(session, url))
print(f"task={task}, url={url}")
async def get_urls(url):
while True:
try:
url = find_somewhere_url()
await download_queue.put(url)
except NoSuchElementException:
break
async def main():
async with aiohttp.ClientSession(headers=headers) as session:
get_urls_task = asyncio.create_task(get_urls(url))
downloader_init_task = asyncio.create_task(downloader_init(session))
# Use await here to make it finish the tasks.
await asyncio.gather(get_urls_task, downloader_init_task)
if __name__ == "__main__":
# Use this as it deals with the loop creation, shutdown,
# and other stuff for you.
asyncio.run(main()) # This is new in Python 3.7

Resources