How to avoid writing `await` every time - python-3.x

When I use aiologger, I have to write await logger many times.
For example,
import asyncio
from aiologger import Logger
async def main():
logger = Logger.with_default_handlers(name='my-logger')
await logger.debug("debug at stdout")
await logger.info("info at stdout")
await logger.warning("warning at stderr")
await logger.error("error at stderr")
await logger.critical("critical at stderr")
await logger.shutdown()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
It would be great if I could write something like al instead of await logger.

Disclaimer: I've written about this -- https://coxley.org/logging/#logging-over-the-network
Please don't accept a logging interface like this.
You can't avoid using await to yield the event loop. You just can't. But you can leverage existing features to do I/O outside of the main thread and still use asyncio. You just start a second event loop in that thread.
Example
I don't like to recommend third-party libs in answers, but janus.Queue is important here. Makes it easier to bridge between non-asyncio writers (eg: Log Handler) and asyncio readers (the flusher).
Note 1: If you don't actually need asyncio-compatible I/O from the flusher, use stdlib queue.Queue, remove the async-closure, and get rid of the second loop.
Note 2: This example has both an unbounded queue and does I/O for every message. Add an interval and/or message threshold for flushing to be production-ready. Depending on your system, decide whether you accept memory growth for log bursts, drop logs, or block the main code-path.
import asyncio
import logging
import time
import threading
import typing as t
# pip install --user janus
import janus
LOG = logging.getLogger(__name__)
# Queue must be created within the event loop it will be used from. Start as
# None since this will not be the main thread.
_QUEUE: t.Optional[janus.Queue] = None
class IOHandler(logging.Handler):
def __init__(self, *args, **kwargs):
# This is set from the flusher thread
global _QUEUE
while _QUEUE is None:
time.sleep(0.01)
self.q = _QUEUE.sync_q
super().__init__(*args, **kwargs)
def emit(self, record: logging.LogRecord):
self.q.put(record)
def flusher():
async def run():
global _QUEUE
if _QUEUE is None:
_QUEUE = janus.Queue()
# Upload record instead of print
# Perhaps flush every n-seconds w/ buffer for upper-bound on inserts.
q = _QUEUE.async_q
while True:
record = await q.get()
print("woohoo, doing i/o:", record.msg)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run())
def foo():
print("foo")
def bar():
print("bar")
async def baz():
await asyncio.sleep(1)
print("baz")
async def main():
threading.Thread(target=flusher, daemon=True).start()
LOG.setLevel(logging.INFO)
LOG.addHandler(IOHandler())
foo()
LOG.info("starting program")
LOG.info("doing some stuff")
LOG.info("mighty cool")
bar()
await baz()
if __name__ == "__main__":
asyncio.run(main())

Related

Running coroutines in different thread with same event loop

I want to run a coroutine in a different thread and get the result that the coroutine returns.
class Main:
def __init__(self, result_from_io_task=None):
self._io_task_result = result_from_io_task
async def io_task(self):
await asyncio.sleep(2)
return "slept of 2s"
def non_async_func(self):
#This can't be made async.
if not self._io_task_result:
#run io_task and get its result
#event loop will be running in the main thread so I can fire the task
task = asyncio.create_task(self.io_task)
#can't await task since I am in non-async func and I cannot
#return from non_async_func until and unless I know what
#self.io_task has returned. Tried following but my app hangs forever.
while not task.done():
pass
I also tried, but it doesn't work "
def run_in_thread(coro, loop):
output = []
def run():
fut = asyncio.run_coroutine_threadsafe(coro, loop)
output.append(fut)
thr = Thread(target=run)
thr.start()
return output
async def main():
main_obj = Main(result_from_io_task=None)
v = main_obj.non_async_func()
How can I spawn a new thread and run the given coroutine using event loop running in main thread
Unfortunately, my codebase depends on python < 3.8 and asyncio.to_thread is not available in python 3.7
Based on the example of my answer, I'm introducing another implementation of the asynchronous decorator that does not use asyncio.to_thread() function but uses ThreadPoolExecutor instead.
import asyncio
import requests
import concurrent.futures
def asynchronous(func):
async def wrapper(*args, **kwargs):
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(func, *args, **kwargs)
return future.result()
return wrapper
#asynchronous
def request(url):
with requests.Session() as session:
response = session.get(url)
try:
return response.json()
except requests.JSONDecodeError:
return response.text
async def main():
task = asyncio.create_task(request("https://google.com/"))
print("waiting for response...")
result = await task
print(result)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()

How to return values ​from an asynchronous function that is executed in a thread in python?

I am new to asynchronous functions and threads, and I am trying to return a series of values ​​obtained from a Web socket to pass to another thread where synchronous code is executing. In the code, I also use a multi Web socket approach. Below I show you the code:
"""
This code is designed to run an asynchronous loop
with asyncio in a separate thread. This allows mixing
a synchronous code with an asynchronous one.
"""
import asyncio
from datetime import datetime
from threading import Thread
import websockets
from typing import Tuple, List, Iterable
import json
import time
URLS = [
"wss://stream.binance.com:9443/ws/xrpusdt#kline_1m",
"wss://stream.binance.com:9443/ws/btcusdt#kline_1m",
]
def start_background_loop(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
loop.run_forever()
async def IndividualSubscription(url: str):
"""An individual subscription to each WebSocket is created"""
async with websockets.connect(url) as websocket:
data = await websocket.recv()
data = json.loads(data)
print('\n', data)
return data
async def Subscriptions(URLS: Iterable[str]):
"""All concurrent tickets are subscribed and all are combined
in a single coroutine."""
while True:
task = [asyncio.create_task(SuscripcionIndividual(url)) for url in URLS]
# All tasks are run in parallel
await asyncio.gather(*tareas)
#return tareas
def main():
loop = asyncio.new_event_loop()
t = Thread(target=start_background_loop, args=(loop,), daemon=True)
t.start()
task = asyncio.run_coroutine_threadsafe(Suscripciones(URLS), loop)
for i in task.result():
print(f"{i}")
#return tareas
def function():
for i in range(100):
print("This is out of asynchronous ", i)
time.sleep(1)
if __name__ == "__main__":
main()
T2 = Thread(target=function,)
T2.start()
I tried to just put return to the async code, but by doing this the async loop only runs once and not continuously as I would expect. Also, I've tried the method .result() over .create_task() . Is it possible to return values ​​from an asynchronous function?
If you want interoperability between synchronous and asynchronous code you need to design some communication mechanism that won't block the thread running async code. Queues are commonly used for communication between threads, janus library implements queues compatible with threads running async code, it does so by exposing sync queue interface to sync code and async queue interface to async code.
Your code is a little chaotic, so I cleaned it up just to show communication between sync thread (main thread) and async thread (background thread running asyncio loop)
import asyncio
from datetime import datetime
from threading import Thread
import websockets
from typing import Tuple, List, Iterable
import json
import time
import janus # pip install janus
URLS = [
"wss://stream.binance.com:9443/ws/xrpusdt#kline_1m",
"wss://stream.binance.com:9443/ws/btcusdt#kline_1m",
]
def start_background_loop(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
loop.run_forever()
async def IndividualSubscription(url: str):
"""An individual subscription to each WebSocket is created"""
async with websockets.connect(url) as websocket:
return json.loads(await websocket.recv())
async def Subscriptions(URLS: Iterable[str], results: asyncio.Queue):
"""All concurrent tickets are subscribed and all are combined
in a single coroutine."""
while True:
tasks = [asyncio.create_task(SuscripcionIndividual(url)) for url in URLS]
for task in await asyncio.gather(*tasks):
await results.put(task.result())
def async_main(results: asyncio.Queue):
asyncio.run(Subscriptions(URLS, results))
if __name__ == "__main__":
results = janus.Queue(100) # max size of 100
async_thread = Thread(target=async_main, args=(results.async_q,))
async_thread.daemon = True # exit if main thread exits
async_thread.start()
while True:
print(f"[sync thread] got result from async thread: {results.sync_q.get()}")

How to get Task created by asyncio.start_server that runs client_connected_cb

Here are the docs:
https://docs.python.org/3/library/asyncio-stream.html
asyncio.start_server takes in a client_connected_cb, which is run in a Task as soon as a client connects to the server. My goal is to write a teardown function that waits until everything is finished: all readers are done, all writers are done, and all tasks created by the server are done. In order to do that, I need to know the Tasks created by the server that are running the client_connected_cb, but I cannot figure out how to do that?
Is there some way to get this Task? Otherwise I have to resort to hacks where every time client_connected_cb is created, I store it in a dictionary of some sort and then wait for that dictionary to be cleared by polling it or something.
If you want fine grained control you could create a custom task_factory to catch all new tasks and do something with them, like adding a callback to all new tasks of a type that you want to do something after it finishes.
Task.add_done_callback(callback)
def end(task: asyncio.Task):
print(f'Task: {task.get_name()} Finished')
tasks = []
def my_task_factory(loop, coro):
task = asyncio.Task(coro, loop=loop)
if task.get_coro().__name__ == 'handle_client':
task.add_done_callback(end)
tasks.append(task)
return task # type: asyncio.Task
Loop = asyncio.get_event_loop()
Loop.set_task_factory(my_task_factory)
There is also 2 high level methods for accessing tasks.
current_task() all_tasks()
asyncio.current_task(loop=None) # Returns _asyncio.Task
asyncio.all_tasks(loop=None) # Returns a set of _asyncio.Task
import _asyncio
import asyncio
from asyncio import StreamReader, StreamWriter
class Server:
def __init__(self, bind_ip='localhost', bind_port=15555):
self.Loop = asyncio.get_event_loop()
self.bind_port = bind_port
self.bind_ip = bind_ip
def startup(self):
self.Server_Task = self.Loop.create_task(self._run_server(), name='Server')
self.Loop.run_forever()
async def _run_server(self):
self.Server = await asyncio.start_server(self.handle_client, host=self.bind_ip, port=self.bind_port)
async with self.Server:
print(f'Listening on {self.bind_ip}:{self.bind_port}')
await self.Server.serve_forever()
async def handle_client(self, reader: StreamReader, writer: StreamWriter):
peer = writer.get_extra_info('peername') # type: Tuple[str, int]
try:
task = asyncio.current_task(self.Loop) # type: _asyncio.Task
tasks = asyncio.all_tasks(self.Loop) # type: Set # of _asyncio.Task
print(f'Tasks Type:{type(tasks)}\n------')
for i in tasks:
print(f'Task: Type:{type(i)}\nTask: {i.get_name()}\nCoro: {i.get_coro().__name__}')
print(f'------\nhandle_client: Type:{type(task)}\nTask: {task.get_name()}\nCoro: {task.get_coro().__name__}')
except Exception as err:
print(err)
finally:
writer.close()
await writer.wait_closed()
print(f'Closed: {peer}')
if __name__ == '__main__':
TCPServer = Server()
TCPServer.startup()
print('End')

Attach data to running event loop

I would like to attach some data to a currently running loop. This can be very useful in sharing secret keys, or reusing sessions. For example, something like
import asyncio
import aiohttp
async def func1():
loop=asyncio.get_running_loop()
loop["client_session"] = aiohttp.ClientSession()
async def func2():
loop=asyncio.get_running_loop()
return await loop["client_session"].get("https://google.com")
async def main():
await func1()
return await func2()
if __name__ == "__main__":
asyncio.run(main())
My question is, would the above code work, is it reliable, or is there a better way?
I know that data can be attached to an aiohttp.Application, and so I'm wondering if the same can be done for general asyncio loops.

Python async: Waiting for stdin input while doing other stuff

I'm trying to create a WebSocket command line client that waits for messages from a WebSocket server but waits for user input at the same time.
Regularly polling multiple online sources every second works fine on the server, (the one running at localhost:6789 in this example), but instead of using Python's normal sleep() method, it uses asyncio.sleep(), which makes sense because sleeping and asynchronously sleeping aren't the same thing, at least not under the hood.
Similarly, waiting for user input and asynchronously waiting for user input aren't the same thing, but I can't figure out how to asynchronously wait for user input in the same way that I can asynchronously wait for an arbitrary amount of seconds, so that the client can deal with incoming messages from the WebSocket server while simultaneously waiting for user input.
The comment below in the else-clause of monitor_cmd() hopefully explains what I'm getting at:
import asyncio
import json
import websockets
async def monitor_ws():
uri = 'ws://localhost:6789'
async with websockets.connect(uri) as websocket:
async for message in websocket:
print(json.dumps(json.loads(message), indent=2, sort_keys=True))
async def monitor_cmd():
while True:
sleep_instead = False
if sleep_instead:
await asyncio.sleep(1)
print('Sleeping works fine.')
else:
# Seems like I need the equivalent of:
# line = await asyncio.input('Is this your line? ')
line = input('Is this your line? ')
print(line)
try:
asyncio.get_event_loop().run_until_complete(asyncio.wait([
monitor_ws(),
monitor_cmd()
]))
except KeyboardInterrupt:
quit()
This code just waits for input indefinitely and does nothing else in the meantime, and I understand why. What I don't understand, is how to fix it. :)
Of course, if I'm thinking about this problem in the wrong way, I'd be very happy to learn how to remedy that as well.
You can use the aioconsole third-party package to interact with stdin in an asyncio-friendly manner:
line = await aioconsole.ainput('Is this your line? ')
Borrowing heavily from aioconsole, if you would rather avoid using an external library you could define your own async input function:
async def ainput(string: str) -> str:
await asyncio.get_event_loop().run_in_executor(
None, lambda s=string: sys.stdout.write(s+' '))
return await asyncio.get_event_loop().run_in_executor(
None, sys.stdin.readline)
Borrowing heavily from aioconsole, there are 2 ways to handle.
start a new daemon thread:
import sys
import asyncio
import threading
from concurrent.futures import Future
async def run_as_daemon(func, *args):
future = Future()
future.set_running_or_notify_cancel()
def daemon():
try:
result = func(*args)
except Exception as e:
future.set_exception(e)
else:
future.set_result(result)
threading.Thread(target=daemon, daemon=True).start()
return await asyncio.wrap_future(future)
async def main():
data = await run_as_daemon(sys.stdin.readline)
print(data)
if __name__ == "__main__":
asyncio.run(main())
use stream reader:
import sys
import asyncio
async def get_steam_reader(pipe) -> asyncio.StreamReader:
loop = asyncio.get_event_loop()
reader = asyncio.StreamReader(loop=loop)
protocol = asyncio.StreamReaderProtocol(reader)
await loop.connect_read_pipe(lambda: protocol, pipe)
return reader
async def main():
reader = await get_steam_reader(sys.stdin)
data = await reader.readline()
print(data)
if __name__ == "__main__":
asyncio.run(main())

Resources