If I have the following code sample
async def coro():
# Cancelled error could be raised here
await asyncio.sleep(1)
# Or here
await asyncio.shield(
another_coro()
)
# Or here
async def wait_on_it(loop):
f = loop.create_task(coro())
# Pretend f may or may not happen, I just sleep in this example
await asyncio.sleep(1)
if not f.done():
f.cancel() # Will raise CancelledError when some await finishes in coro()
How can I determine whether or not the shielded task actually ran? I have important logic that must be run iff the shielded task did run. Maybe shielding that function is not the correct method?
coro() can transfer the information to the caller by modifying a mutable object it receives from the caller:
class Ref:
def __init__(self, **kwargs):
self.__dict__.update(**kwargs)
async def coro(run_ref):
await asyncio.sleep(1)
run_ref.ran_another_coro = True
await asyncio.shield(another_coro())
async def wait_on_it(loop):
run_ref = Ref(ran_another_coro=False)
f = loop.create_task(coro(run_ref))
await asyncio.sleep(1)
if not f.done():
f.cancel()
if run_ref.ran_another_coro:
# ... another_coro() was started
Since asyncio.shield can't suspend, if wait_on_it observes a true value of run_ref.ran_another_coro, then another_coro() is guaranteed to have been started.
Related
In the following, the coroutinerunIt() is created and provided as a parameter to delegate(...) - which is turned into a Task that is canceled before runIt executes:
import asyncio
async def cancelTaskTest():
async def runIt():
print("RunIt ran")
async def delegate(coro):
await coro
task = asyncio.create_task(delegate(runIt()))
task.cancel()
if __name__=='__main__':
asyncio.run(cancelTaskTest())
Produces the unwanted warning:
/usr/lib/python3.10/asyncio/base_events.py:1881: RuntimeWarning: coroutine 'cancelTaskTest.<locals>.runIt' was never awaited
handle = self._ready.popleft()
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
I'm aware that runIt did not run. I don't want a warning about it - what's the best way to avoid this.
Simplest method would be to remove () in runIt and call it in runAfterTimeout():
import asyncio
async def cancelTaskTest():
async def runIt():
print("RunIt ran")
async def delegate(asyncFunc):
coro = asyncFunc() # <-- put () here
await coro
task = asyncio.create_task(delegate(runIt)) # <-- removed () in runIt
task.cancel()
if __name__ == "__main__":
asyncio.run(cancelTaskTest())
EDIT: To add parameters to RunIt, just create plain lambda::
import asyncio
async def cancelTaskTest():
async def runIt(p1, p2):
print(f"RunIt({p1}, {p2})")
async def delegate(coro):
await coro()
task = asyncio.create_task(delegate(lambda: runIt(1, 2)))
task.cancel()
if __name__ == "__main__":
asyncio.run(cancelTaskTest())
i have some troubles with parsing two or more sessions at the same time with telethon. I have tried this:
class NewSession:
def __init__(self, session_name):
self.client = TelegramClient(session_name, api_id, api_hash)
self.session_name = session_name
async def pool(self):
print("working with:", self.session_name)
#self.client.on(events.NewMessage(outgoing=True))
async def main(event):
message = event.message.to_dict()
msg_text = message['message']
print(msg_text)
try:
await self.client.start()
await self.client.run_until_disconnected()
finally:
await self.client.disconnect()
async def main():
user = NewSession("321")
user2 = NewSession("123")
await user.pool()
await user2.pool()
if __name__ == '__main__':
asyncio.run(main())
But only one is working. Need help :)
The problem is inside your main function. When you await for a coroutine to return it doesn't mean that the execution continues to the next expression. So, in your code the line await user2.pool() is going to be executed only when the user.poll() coroutines returns a value, this is when the session '321' is disconnected.
You need to run the tasks concurrently; you can use the function asyncio.gather. Reworking your main:
async def main():
user = NewSession("321")
user2 = NewSession("123")
await asyncio.gather(user.pool(), user2.pool())
When I use aiologger, I have to write await logger many times.
For example,
import asyncio
from aiologger import Logger
async def main():
logger = Logger.with_default_handlers(name='my-logger')
await logger.debug("debug at stdout")
await logger.info("info at stdout")
await logger.warning("warning at stderr")
await logger.error("error at stderr")
await logger.critical("critical at stderr")
await logger.shutdown()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
It would be great if I could write something like al instead of await logger.
Disclaimer: I've written about this -- https://coxley.org/logging/#logging-over-the-network
Please don't accept a logging interface like this.
You can't avoid using await to yield the event loop. You just can't. But you can leverage existing features to do I/O outside of the main thread and still use asyncio. You just start a second event loop in that thread.
Example
I don't like to recommend third-party libs in answers, but janus.Queue is important here. Makes it easier to bridge between non-asyncio writers (eg: Log Handler) and asyncio readers (the flusher).
Note 1: If you don't actually need asyncio-compatible I/O from the flusher, use stdlib queue.Queue, remove the async-closure, and get rid of the second loop.
Note 2: This example has both an unbounded queue and does I/O for every message. Add an interval and/or message threshold for flushing to be production-ready. Depending on your system, decide whether you accept memory growth for log bursts, drop logs, or block the main code-path.
import asyncio
import logging
import time
import threading
import typing as t
# pip install --user janus
import janus
LOG = logging.getLogger(__name__)
# Queue must be created within the event loop it will be used from. Start as
# None since this will not be the main thread.
_QUEUE: t.Optional[janus.Queue] = None
class IOHandler(logging.Handler):
def __init__(self, *args, **kwargs):
# This is set from the flusher thread
global _QUEUE
while _QUEUE is None:
time.sleep(0.01)
self.q = _QUEUE.sync_q
super().__init__(*args, **kwargs)
def emit(self, record: logging.LogRecord):
self.q.put(record)
def flusher():
async def run():
global _QUEUE
if _QUEUE is None:
_QUEUE = janus.Queue()
# Upload record instead of print
# Perhaps flush every n-seconds w/ buffer for upper-bound on inserts.
q = _QUEUE.async_q
while True:
record = await q.get()
print("woohoo, doing i/o:", record.msg)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run())
def foo():
print("foo")
def bar():
print("bar")
async def baz():
await asyncio.sleep(1)
print("baz")
async def main():
threading.Thread(target=flusher, daemon=True).start()
LOG.setLevel(logging.INFO)
LOG.addHandler(IOHandler())
foo()
LOG.info("starting program")
LOG.info("doing some stuff")
LOG.info("mighty cool")
bar()
await baz()
if __name__ == "__main__":
asyncio.run(main())
Here's a sample code.
class Foo:
def __init__(self):
self._run_coro()
def _run_coro(self):
async def init():
bar = #some I/O op
self.bar = bar
loop = asyncio.get_event_loop()
loop.run_until_complete(init())
async def spam(self):
return await #I/O op
async def main():
foo = Foo()
await foo.spam()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
When I run this code, I get following exception:
RuntimeError: This event loop is already running
If I initialize Foo outside main, the code runs without any exception. I want to initialize Foo such that during initialization it runs a coroutine which creates a class attribute bar.
I am unable to figure how to do it correctly. How can I run a coroutine from __init__.
Any help would be highly appreciated.
class Foo:
def __init__(self):
self.session = requests.Session()
self.async_session = None
#I guess this can be done to initialize it.
s = self.init_async_session()
try:
s.send(None)
except StopIteration:
pass
finally:
s.close()
async def init_async_session(self):
#ClientSession should be created inside a coroutine.
self.async_session = aiohttp.ClientSession()
What would be the right way to initialize self.async_session
If some method uses something asynchronous it should be explicitly defined as asynchronous either. This is a core idea behind asyncio: make you write code a way you always know if some arbitrary method may do something asynchronous.
In your snippet you want to do async thing (bar I/O) inside sync method __init__ and asyncio prohibits it. You should make _run_coro async and initialize Foo asynchronously, for example, using __await__ method:
import asyncio
class Foo:
def __await__(self):
return self._run_coro().__await__()
async def _run_coro(self): # real async initializer
async def init():
await asyncio.sleep(1) # bar I/O
self.bar = 123
await init()
return self
async def spam(self):
return await asyncio.sleep(1) # I/O op
async def main():
foo = await Foo()
await foo.spam()
asyncio.run(main()) # instead of two lines in Python 3.7+
You may be interested in reading this answer to understand better how asyncio works and how to handle it.
Upd:
s = self.init_async_session()
try:
s.send(None)
Don't do such things: generator's method are only details of implementation in regard of coroutines. You can predict how coroutine will react on calling .send() method and you can rely on this behavior.
If you want to execute coroutine use await, if you want to start it "in background" use task or other functions from asyncio doc.
What would be the right way to initialize self.async_session
When it comes to aiohttp.ClientSession it should not only be created, but properly closed also. Best way to do it is to use async context manager as shown in aiohttp doc.
If you want to hide this operation inside Foo you can make it async manager either. Complete example:
import aiohttp
class Foo:
async def __aenter__(self):
self._session = aiohttp.ClientSession()
await self._session.__aenter__()
return self
async def __aexit__(self, *args):
await self._session.__aexit__(*args)
async def spam(self):
url = 'http://httpbin.org/delay/1'
resp = await self._session.get(url)
text = await resp.text()
print(text)
async def main():
async with Foo() as foo:
await foo.spam()
asyncio.run(main())
Upd2:
You can combine ways to init/close object from above to achive result you like. As long as you keep in mind both operations are asynchronous and thus should be awaited, everything should be fine.
One more possible way:
import asyncio
import aiohttp
class Foo:
def __await__(self):
return self._init().__await__()
async def _init(self):
self._session = aiohttp.ClientSession()
await self._session.__aenter__()
return self
async def close(self):
await self._session.__aexit__(None, None, None)
async def spam(self):
url = 'http://httpbin.org/delay/1'
resp = await self._session.get(url)
text = await resp.text()
print(text)
async def main():
foo = await Foo()
try:
await foo.spam()
finally:
await foo.close()
asyncio.run(main())
Here's my solution.
class Session:
def __init__(self, headers):
self._headers = headers
self._session = requests.Session()
self._async_session = None
async def _init(self):
self._session = aiohttp.ClientSession(headers=headers)
async def async_request(self, url):
while True:
try:
async with self._async_session.get(url) as resp:
resp.raise_for_status()
return await resp.text()
except aiohttp.client_exceptions.ClientError:
#retry or raise
except AttributeError:
if isinstance(self._async_session, aiohttp.ClientSession):
raise
await self._init()
def request(self, url):
return self._session.get(url).text
async def close(self):
if isinstance(self._async_session, aiohttp.ClientSession):
await self._session.close()
async def main():
session = Session({})
print(await session.async_request('https://httpstat.us/200')
await session.close()
asyncio.run(main())
I can initialize the Session class and make synchronous as well as asynchronous requests. I do not have to explicitly call await session._init() to initialize self._async_session as when session._async_request is called and self._async_session is None, then await session._init() will be called and the request will be retried.
async def start(channel):
while True:
m = await client.send_message(channel, "Generating... ")
generator.makeFile()
with open('tmp.png', 'rb') as f:
await client.send_file(channel, f)
await client.delete_message(m)
await asyncio.sleep(2)
I have a discord bot that runs a task every 2 seconds. I tried using an infinite loop for this, but the script crashes with a Task was destroyed but it is still pending! I have read about asyncio's coroutines, but none of the examples that I found use await in them. Is it possible avoid this error, by running a coroutine with await, for example?
Task was destroyed but it is still pending! is warning that you receive when you call loop.close() when some of tasks in your script aren't finished. Usually you should avoid this situation because unfinished task may not release some resources. You need either to await task done or cancel it before event loop closed.
Since you have infinite loop you probably would need to cancel task, example:
import asyncio
from contextlib import suppress
async def start():
# your infinite loop here, for example:
while True:
print('echo')
await asyncio.sleep(1)
async def main():
task = asyncio.Task(start())
# let script some thime to work:
await asyncio.sleep(3)
# cancel task to avoid warning:
task.cancel()
with suppress(asyncio.CancelledError):
await task # await for task cancellation
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_until_complete(main())
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
See also this answer for more information about tasks.