I would like to attach some data to a currently running loop. This can be very useful in sharing secret keys, or reusing sessions. For example, something like
import asyncio
import aiohttp
async def func1():
loop=asyncio.get_running_loop()
loop["client_session"] = aiohttp.ClientSession()
async def func2():
loop=asyncio.get_running_loop()
return await loop["client_session"].get("https://google.com")
async def main():
await func1()
return await func2()
if __name__ == "__main__":
asyncio.run(main())
My question is, would the above code work, is it reliable, or is there a better way?
I know that data can be attached to an aiohttp.Application, and so I'm wondering if the same can be done for general asyncio loops.
Related
When I use aiologger, I have to write await logger many times.
For example,
import asyncio
from aiologger import Logger
async def main():
logger = Logger.with_default_handlers(name='my-logger')
await logger.debug("debug at stdout")
await logger.info("info at stdout")
await logger.warning("warning at stderr")
await logger.error("error at stderr")
await logger.critical("critical at stderr")
await logger.shutdown()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
It would be great if I could write something like al instead of await logger.
Disclaimer: I've written about this -- https://coxley.org/logging/#logging-over-the-network
Please don't accept a logging interface like this.
You can't avoid using await to yield the event loop. You just can't. But you can leverage existing features to do I/O outside of the main thread and still use asyncio. You just start a second event loop in that thread.
Example
I don't like to recommend third-party libs in answers, but janus.Queue is important here. Makes it easier to bridge between non-asyncio writers (eg: Log Handler) and asyncio readers (the flusher).
Note 1: If you don't actually need asyncio-compatible I/O from the flusher, use stdlib queue.Queue, remove the async-closure, and get rid of the second loop.
Note 2: This example has both an unbounded queue and does I/O for every message. Add an interval and/or message threshold for flushing to be production-ready. Depending on your system, decide whether you accept memory growth for log bursts, drop logs, or block the main code-path.
import asyncio
import logging
import time
import threading
import typing as t
# pip install --user janus
import janus
LOG = logging.getLogger(__name__)
# Queue must be created within the event loop it will be used from. Start as
# None since this will not be the main thread.
_QUEUE: t.Optional[janus.Queue] = None
class IOHandler(logging.Handler):
def __init__(self, *args, **kwargs):
# This is set from the flusher thread
global _QUEUE
while _QUEUE is None:
time.sleep(0.01)
self.q = _QUEUE.sync_q
super().__init__(*args, **kwargs)
def emit(self, record: logging.LogRecord):
self.q.put(record)
def flusher():
async def run():
global _QUEUE
if _QUEUE is None:
_QUEUE = janus.Queue()
# Upload record instead of print
# Perhaps flush every n-seconds w/ buffer for upper-bound on inserts.
q = _QUEUE.async_q
while True:
record = await q.get()
print("woohoo, doing i/o:", record.msg)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run())
def foo():
print("foo")
def bar():
print("bar")
async def baz():
await asyncio.sleep(1)
print("baz")
async def main():
threading.Thread(target=flusher, daemon=True).start()
LOG.setLevel(logging.INFO)
LOG.addHandler(IOHandler())
foo()
LOG.info("starting program")
LOG.info("doing some stuff")
LOG.info("mighty cool")
bar()
await baz()
if __name__ == "__main__":
asyncio.run(main())
I am new to asynchronous functions and threads, and I am trying to return a series of values obtained from a Web socket to pass to another thread where synchronous code is executing. In the code, I also use a multi Web socket approach. Below I show you the code:
"""
This code is designed to run an asynchronous loop
with asyncio in a separate thread. This allows mixing
a synchronous code with an asynchronous one.
"""
import asyncio
from datetime import datetime
from threading import Thread
import websockets
from typing import Tuple, List, Iterable
import json
import time
URLS = [
"wss://stream.binance.com:9443/ws/xrpusdt#kline_1m",
"wss://stream.binance.com:9443/ws/btcusdt#kline_1m",
]
def start_background_loop(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
loop.run_forever()
async def IndividualSubscription(url: str):
"""An individual subscription to each WebSocket is created"""
async with websockets.connect(url) as websocket:
data = await websocket.recv()
data = json.loads(data)
print('\n', data)
return data
async def Subscriptions(URLS: Iterable[str]):
"""All concurrent tickets are subscribed and all are combined
in a single coroutine."""
while True:
task = [asyncio.create_task(SuscripcionIndividual(url)) for url in URLS]
# All tasks are run in parallel
await asyncio.gather(*tareas)
#return tareas
def main():
loop = asyncio.new_event_loop()
t = Thread(target=start_background_loop, args=(loop,), daemon=True)
t.start()
task = asyncio.run_coroutine_threadsafe(Suscripciones(URLS), loop)
for i in task.result():
print(f"{i}")
#return tareas
def function():
for i in range(100):
print("This is out of asynchronous ", i)
time.sleep(1)
if __name__ == "__main__":
main()
T2 = Thread(target=function,)
T2.start()
I tried to just put return to the async code, but by doing this the async loop only runs once and not continuously as I would expect. Also, I've tried the method .result() over .create_task() . Is it possible to return values from an asynchronous function?
If you want interoperability between synchronous and asynchronous code you need to design some communication mechanism that won't block the thread running async code. Queues are commonly used for communication between threads, janus library implements queues compatible with threads running async code, it does so by exposing sync queue interface to sync code and async queue interface to async code.
Your code is a little chaotic, so I cleaned it up just to show communication between sync thread (main thread) and async thread (background thread running asyncio loop)
import asyncio
from datetime import datetime
from threading import Thread
import websockets
from typing import Tuple, List, Iterable
import json
import time
import janus # pip install janus
URLS = [
"wss://stream.binance.com:9443/ws/xrpusdt#kline_1m",
"wss://stream.binance.com:9443/ws/btcusdt#kline_1m",
]
def start_background_loop(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
loop.run_forever()
async def IndividualSubscription(url: str):
"""An individual subscription to each WebSocket is created"""
async with websockets.connect(url) as websocket:
return json.loads(await websocket.recv())
async def Subscriptions(URLS: Iterable[str], results: asyncio.Queue):
"""All concurrent tickets are subscribed and all are combined
in a single coroutine."""
while True:
tasks = [asyncio.create_task(SuscripcionIndividual(url)) for url in URLS]
for task in await asyncio.gather(*tasks):
await results.put(task.result())
def async_main(results: asyncio.Queue):
asyncio.run(Subscriptions(URLS, results))
if __name__ == "__main__":
results = janus.Queue(100) # max size of 100
async_thread = Thread(target=async_main, args=(results.async_q,))
async_thread.daemon = True # exit if main thread exits
async_thread.start()
while True:
print(f"[sync thread] got result from async thread: {results.sync_q.get()}")
It's clear for my how I use asyncio in python3.
import asyncio
import time
async def do_something(number):
print(f"do something no.{number} at timestamp : {time.perf_counter():6.2f}")
await asyncio.sleep(1)
print(f"do something no.{number} ended at timestamp: {time.perf_counter():6.2f}")
async def main():
await asyncio.gather(
do_something(1),
do_something(2)
)
asyncio.run(main() )
However, I have no idea how I could create an own "await"-able object like asyncio.sleep. In this "await"-able I could encapsulate urllib.request, isn't it?
Can someone post an example?
Many thanks.
Please, take a look at this answer it uses old yield for-based syntax, but the idea stays the same: using asyncio.Future and loop.call_later() you can cast callback-based code into coroutine-based code:
import asyncio
async def my_sleep(delay):
fut = asyncio.Future()
loop = asyncio.get_event_loop()
loop.call_later(
delay,
lambda *_: fut.set_result(True)
)
return await fut
async def main():
await asyncio.gather(
my_sleep(1),
my_sleep(2),
my_sleep(3)
)
print('ok')
asyncio.run(main())
I believe, urllib.request is blocking and doesn't provide callbacks so it can't be cast into coroutine-based form directly. A common way to handle the situation is to run it in async thread (see links in this answer).
But if you want just to make async http reqeust, forget all above and use aiohttp: it's created for the purpose.
You can await all coroutines, so any function with the prefix async is awaitable
For example:
import asyncio
async def some_function():
await asyncio.sleep(1)
print("Hello")
async def main():
await some_function()
asyncio.run(main())
You can find some more information about coroutines at the python docs:
https://docs.python.org/3/library/asyncio-task.html
Because urllib is blocking (does not allow other code to be running before it finishes) by default, it is not that easy to just "wrap" it in an awaitable function.
It is probably possible to offload it to something like another thread and then have an awaiable wait for that thread to finish, but it is probably easier to use an async web request library like aiohttp.
I'd like to do a non-blocking http request in Python 3.7. What I'm trying to do is described well in this SO post, but it doesn't yet have an accepted answer.
Here's my code so far:
import asyncio
from aiohttp import ClientSession
[.....]
async def call_endpoint_async(endpoint, data):
async with ClientSession() as session, session.post(url=endpoint, data=data) as result:
response = await result.read()
print(response)
return response
class CreateTestScores(APIView):
permission_classes = (IsAuthenticated,)
def post(self, request):
[.....]
asyncio.run(call_endpoint_async(url, data))
print('cp #1') # <== `async.io` BLOCKS -- PRINT STATEMENT DOESN'T RUN UNTIL `asyncio.run` RETURNS
What is the correct way to do an Ajax-style non-blocking http request in Python?
Asyncio makes it easy to make a non-blocking request if your program runs in asyncio. For example:
async def doit():
task = asyncio.create_task(call_endpoint_async(url, data))
print('cp #1')
await asyncio.sleep(1)
print('is it done?', task.done())
await task
print('now it is done')
But this requires that the "caller" be async as well. In your case you want the whole asyncio event loop to run in the background, so that. This can be achieved by running it in a separate thread, e.g.:
pool = concurrent.futures.ThreadPoolExecutor()
# ...
def post(self, request):
fut = pool.submit(asyncio.run, call_endpoint_async(url, data))
print('cp #1')
However, in that case you're not getting anything by using asyncio. Since you're using threads anyway, you could as well call a sync function such as requests.get() to begin with.
I'm trying to create a WebSocket command line client that waits for messages from a WebSocket server but waits for user input at the same time.
Regularly polling multiple online sources every second works fine on the server, (the one running at localhost:6789 in this example), but instead of using Python's normal sleep() method, it uses asyncio.sleep(), which makes sense because sleeping and asynchronously sleeping aren't the same thing, at least not under the hood.
Similarly, waiting for user input and asynchronously waiting for user input aren't the same thing, but I can't figure out how to asynchronously wait for user input in the same way that I can asynchronously wait for an arbitrary amount of seconds, so that the client can deal with incoming messages from the WebSocket server while simultaneously waiting for user input.
The comment below in the else-clause of monitor_cmd() hopefully explains what I'm getting at:
import asyncio
import json
import websockets
async def monitor_ws():
uri = 'ws://localhost:6789'
async with websockets.connect(uri) as websocket:
async for message in websocket:
print(json.dumps(json.loads(message), indent=2, sort_keys=True))
async def monitor_cmd():
while True:
sleep_instead = False
if sleep_instead:
await asyncio.sleep(1)
print('Sleeping works fine.')
else:
# Seems like I need the equivalent of:
# line = await asyncio.input('Is this your line? ')
line = input('Is this your line? ')
print(line)
try:
asyncio.get_event_loop().run_until_complete(asyncio.wait([
monitor_ws(),
monitor_cmd()
]))
except KeyboardInterrupt:
quit()
This code just waits for input indefinitely and does nothing else in the meantime, and I understand why. What I don't understand, is how to fix it. :)
Of course, if I'm thinking about this problem in the wrong way, I'd be very happy to learn how to remedy that as well.
You can use the aioconsole third-party package to interact with stdin in an asyncio-friendly manner:
line = await aioconsole.ainput('Is this your line? ')
Borrowing heavily from aioconsole, if you would rather avoid using an external library you could define your own async input function:
async def ainput(string: str) -> str:
await asyncio.get_event_loop().run_in_executor(
None, lambda s=string: sys.stdout.write(s+' '))
return await asyncio.get_event_loop().run_in_executor(
None, sys.stdin.readline)
Borrowing heavily from aioconsole, there are 2 ways to handle.
start a new daemon thread:
import sys
import asyncio
import threading
from concurrent.futures import Future
async def run_as_daemon(func, *args):
future = Future()
future.set_running_or_notify_cancel()
def daemon():
try:
result = func(*args)
except Exception as e:
future.set_exception(e)
else:
future.set_result(result)
threading.Thread(target=daemon, daemon=True).start()
return await asyncio.wrap_future(future)
async def main():
data = await run_as_daemon(sys.stdin.readline)
print(data)
if __name__ == "__main__":
asyncio.run(main())
use stream reader:
import sys
import asyncio
async def get_steam_reader(pipe) -> asyncio.StreamReader:
loop = asyncio.get_event_loop()
reader = asyncio.StreamReader(loop=loop)
protocol = asyncio.StreamReaderProtocol(reader)
await loop.connect_read_pipe(lambda: protocol, pipe)
return reader
async def main():
reader = await get_steam_reader(sys.stdin)
data = await reader.readline()
print(data)
if __name__ == "__main__":
asyncio.run(main())