websocket messages appears at once rather than individual messages - python-3.x

I want my backend send progress message to UI via websocket.
My problem is all messages,which produced by calling
automate_algorithm()
function appear together at once at the end of process, instead of appear one by one. Is there any wrong with my code.
This class create a dictionary which key is project id, and value is the opened websocket
class ConnectionManager:
def __init__(
self
):
self.connections: dict[str, WebSocket] = {}
async def connect(
self,
id: str,
websocket: WebSocket
):
"""To add new open socket to memory storage
Args:
id:(str)the
"""
await websocket.accept()
self.connections[id] = websocket
async def disconnect(self, id: str):
if id in self.connections:
await self.connections[id].close(code=100,reason=None)
del self.connections[id]
async def send_response(
self,
id: str,
data: str,
status:str='running'
):
print(
f"tries to send response for client with id :{id}. Response is {data}")
try:
await self.connections[id].send_json(data=dict(
timestamp=time.strftime("%H:%M:%S", time.localtime()),
message=data,
id=id,
status=status
)
)
if status=="completed":
await self.disconnect(id)
except Exception as e:
print(str(e))
self.disconnect(id)
manager = ConnectionManager()#create a context for web socket manager
This method get user HTTP request, and start process
#router.websocket("/auto_algo/{client_id}")
async def auto_algo(
websocket: WebSocket,
client_id: str,
):
await manager.connect(client_id, websocket)
# HANDLE FUNCTION*****
await automate_algorithm(idt=client_id)
This is the main method which produce the messages,that should write in websocket.
async def send_message_to_socket(
client_id: str,
what: str,
status:str='running'
):
global manager
await manager.send_response(client_id, what,status)
# automate to algorithm ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
async def automate_algorithm(idt,language='en'):
from controllers.dispatcher_controller import send_message_to_socket
await send_message_to_socket(client_id=idt,what="process starting")#This message appear at start correctly
mds2 = create_mds(idt,mapper=False)
await send_message_to_socket(client_id=idt,what="main_data_structure 2 created...")#the rest of message appear together at the end of process
sample_data = create_sample_data(idt,mapper=False)
await send_message_to_socket(client_id=idt,what="sample data created...")
corr = correlation_matrix(idt,mapper=False)
await send_message_to_socket(client_id=idt,what="correlation created...")
mds3 = accomplish_mds(idt,mapper=False)
await send_message_to_socket(client_id=idt,what="main_data_structure 3 created...")

When the code is executed, the control is never returned to the event loop. There is an easy fix for this; add some await asyncio.sleep(0) to semi-manually return control to the event loop, so it has a chance to execute the send() method of other tasks.So I add await asyncio.sleep(0) right after each await send_message_to_socket(...) line, and the code works correctly

Related

Need to parse two sessions at the same time with telethon on Python

i have some troubles with parsing two or more sessions at the same time with telethon. I have tried this:
class NewSession:
def __init__(self, session_name):
self.client = TelegramClient(session_name, api_id, api_hash)
self.session_name = session_name
async def pool(self):
print("working with:", self.session_name)
#self.client.on(events.NewMessage(outgoing=True))
async def main(event):
message = event.message.to_dict()
msg_text = message['message']
print(msg_text)
try:
await self.client.start()
await self.client.run_until_disconnected()
finally:
await self.client.disconnect()
async def main():
user = NewSession("321")
user2 = NewSession("123")
await user.pool()
await user2.pool()
if __name__ == '__main__':
asyncio.run(main())
But only one is working. Need help :)
The problem is inside your main function. When you await for a coroutine to return it doesn't mean that the execution continues to the next expression. So, in your code the line await user2.pool() is going to be executed only when the user.poll() coroutines returns a value, this is when the session '321' is disconnected.
You need to run the tasks concurrently; you can use the function asyncio.gather. Reworking your main:
async def main():
user = NewSession("321")
user2 = NewSession("123")
await asyncio.gather(user.pool(), user2.pool())

aiomysql makes random errors in events appear

so far I've been using mysql.connector to manage mysql in my discord bot, however, since that's not async, I'm trying to change to aiomysql. It works, kinda... I'm having a problem where sometimes I get something like
ERROR:asyncio:Task was destroyed but it is pending!
task: <ClientEventTask state=pending event=on_raw_message_delete coro=<bound method LoggingSystem.on_raw_message_delete of <cogs.loggingsystem.LoggingSystem object at 0x059C7A48>>>
In this case it was in on_raw_message_delete of a cog, but the event where this happens isn't always the same and it's only on events, never in commands.
So the question is, am I doing something wrong that may cause that?
# mysqlconnection.py
pool: aiomysql.Pool = None
async def getpool():
global pool
pool = await aiomysql.create_pool(host="", user="", password="", db="", pool_recycle=0)
async def execute_search(search: str, vals: tuple = None): # This function is what I use to get data from the DB across the whole bot
async with pool.acquire() as connection:
async with connection.cursor() as cursor:
if vals is None:
await cursor.execute(search)
else:
await cursor.execute(search, vals)
result = await cursor.fetchall()
return result
async def execute_alter(query: str, vals: tuple): # This function is what I use to alter data in the DB across the whole bot
async with pool.acquire() as connection:
async with connection.cursor() as cursor:
await cursor.execute(query, vals)
await connection.commit()
# bot.py
from mysqlconnection import execute_search, execute_alter, getpool
#bot.listen()
async def on_ready():
await getpool()
...
So, anything wrong with my approach?

Tornado websocket client: how to async on_message? (coroutine was never awaited)

How can I make the on_message function work asynchronously in my Tornado WebSocketClient?
I guess I need to await the on_message function, but I don't know how.
Or is there even a fundamental misconception in the way how I try to implement an asynchronous WebSocketClient?
import tornado.websocket
from tornado.queues import Queue
from tornado import gen
import json
q = Queue()
class WebsocketClient():
def __init__(self, url, connections):
self.url = url
self.connections = connections
print("CLIENT started")
print("CLIENT initial connections: ", len(self.connections))
async def send_message(self):
async for message in q:
try:
msg = json.loads(message)
print(message)
await gen.sleep(0.001)
finally:
q.task_done()
async def update_connections(self, connections):
self.connections = connections
print("CLIENT updated connections: ", len(self.connections))
async def on_message(self, message):
await q.put(message)
await gen.sleep(0.001)
async def connect(self):
client = await tornado.websocket.websocket_connect(url=self.url, on_message_callback=self.on_message)
RuntimeWarning: coroutine 'WebsocketClient.on_message' was never awaited
self._on_message_callback(message)
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
on_message_callback is supposed to be a regular function, not a coroutine. And it is meant to be used in old-style code when people used callbacks instead of coroutines.
For the newer async-style code, you don't need this callback. You can just do this:
async def connect(self):
client = await tornado.websocket.websocket_connect(url=self.url)
while True:
message = await client.read_message()
if message is None:
# None message means the connection was closed
break
print("Message received:", message)
await q.put(message)
await gen.sleep(0.001)

How to gather task results in Trio?

I wrote a script that uses a nursery and the asks module to loop through and call an API based upon the loop variables. I get responses but don't know how to return the data like you would with asyncio.
I also have a question on limiting the APIs to 5 per second.
from datetime import datetime
import asks
import time
import trio
asks.init("trio")
s = asks.Session(connections=4)
async def main():
start_time = time.time()
api_key = 'API-KEY'
org_id = 'ORG-ID'
networkIds = ['id1','id2','idn']
url = 'https://api.meraki.com/api/v0/networks/{0}/airMarshal?timespan=3600'
headers = {'X-Cisco-Meraki-API-Key': api_key, 'Content-Type': 'application/json'}
async with trio.open_nursery() as nursery:
for i in networkIds:
nursery.start_soon(fetch, url.format(i), headers)
print("Total time:", time.time() - start_time)
async def fetch(url, headers):
print("Start: ", url)
response = await s.get(url, headers=headers)
print("Finished: ", url, len(response.content), response.status_code)
if __name__ == "__main__":
trio.run(main)
When I run nursery.start_soon(fetch...) , I am printing data within fetch, but how do I return the data? I didn't see anything similar to asyncio.gather(*tasks) function.
Also, I can limit the number of sessions to 1-4, which helps get down below the 5 API per second limit, but was wondering if there was a built in way to ensure that no more than 5 APIs get called in any given second?
Returning data: pass the networkID and a dict to the fetch tasks:
async def main():
…
results = {}
async with trio.open_nursery() as nursery:
for i in networkIds:
nursery.start_soon(fetch, url.format(i), headers, results, i)
## results are available here
async def fetch(url, headers, results, i):
print("Start: ", url)
response = await s.get(url, headers=headers)
print("Finished: ", url, len(response.content), response.status_code)
results[i] = response
Alternately, create a trio.Queue to which you put the results; your main task can then read the results from the queue.
API limit: create a trio.Queue(10) and start a task along these lines:
async def limiter(queue):
while True:
await trio.sleep(0.2)
await queue.put(None)
Pass that queue to fetch, as another argument, and call await limit_queue.get() before each API call.
Based on this answers, you can define the following function:
async def gather(*tasks):
async def collect(index, task, results):
task_func, *task_args = task
results[index] = await task_func(*task_args)
results = {}
async with trio.open_nursery() as nursery:
for index, task in enumerate(tasks):
nursery.start_soon(collect, index, task, results)
return [results[i] for i in range(len(tasks))]
You can then use trio in the exact same way as asyncio by simply patching trio (adding the gather function):
import trio
trio.gather = gather
Here is a practical example:
async def child(x):
print(f"Child sleeping {x}")
await trio.sleep(x)
return 2*x
async def parent():
tasks = [(child, t) for t in range(3)]
return await trio.gather(*tasks)
print("results:", trio.run(parent))
Technically, trio.Queue has been deprecated in trio 0.9. It has been replaced by trio.open_memory_channel.
Short example:
sender, receiver = trio.open_memory_channel(len(networkIds)
async with trio.open_nursery() as nursery:
for i in networkIds:
nursery.start_soon(fetch, sender, url.format(i), headers)
async for value in receiver:
# Do your job here
pass
And in your fetch function you should call async sender.send(value) somewhere.
When I run nursery.start_soon(fetch...) , I am printing data within fetch, but how do I return the data? I didn't see anything similar to asyncio.gather(*tasks) function.
You're asking two different questions, so I'll just answer this one. Matthias already answered your other question.
When you call start_soon(), you are asking Trio to run the task in the background, and then keep going. This is why Trio is able to run fetch() several times concurrently. But because Trio keeps going, there is no way to "return" the result the way a Python function normally would. where would it even return to?
You can use a queue to let fetch() tasks send results to another task for additional processing.
To create a queue:
response_queue = trio.Queue()
When you start your fetch tasks, pass the queue as an argument and send a sentintel to the queue when you're done:
async with trio.open_nursery() as nursery:
for i in networkIds:
nursery.start_soon(fetch, url.format(i), headers)
await response_queue.put(None)
After you download a URL, put the response into the queue:
async def fetch(url, headers, response_queue):
print("Start: ", url)
response = await s.get(url, headers=headers)
# Add responses to queue
await response_queue.put(response)
print("Finished: ", url, len(response.content), response.status_code)
With the changes above, your fetch tasks will put responses into the queue. Now you need to read responses from the queue so you can process them. You might add a new function to do this:
async def process(response_queue):
async for response in response_queue:
if response is None:
break
# Do whatever processing you want here.
You should start this process function as a background task before you start any fetch tasks so that it will process responses as soon as they are received.
Read more in the Synchronizing and Communicating Between Tasks section of the Trio documentation.

Timeout WebSocket connections in aiohttp

My WebSocket server implementation is open to the world, but the client is required to send an authenticate message after the connection was established or the server should close the connection.
How can I implement this in aiohttp? It seems, I need to do the following things:
Create an on_open method for every socket connection: I can't find a way (similarly to on_open in Tornado) to create such event.
Create a timer: asyncio's sleep or call_back methods of the main event loop may be used. But I can't find a way to send the WebSocketResponse to the callback function:
await asyncio.sleep(10, timer, loop=request.app.loop)
Closing the connection if not authenticated
This is what I had before with Tornado:
def open(self, *args, **kwargs):
self.timeout = ioloop.IOLoop.instance().add_timeout(
datetime.timedelta(seconds=60),
self._close_on_timeout
)
def remove_timeout_timer(self):
ioloop.IOLoop.instance().remove_timeout(self.timeout)
self.timeout = None
def on_message(self, message):
if message = 'AUTHENTICATE':
self.authenticated = True
self.remove_timeout_timer
def _close_on_timeout(self):
if not self.authenticated:
if self.ws_connection:
self.close()
Here is what I have using aiohttp for setting up a timer:
async def ensure_client_logged(ws):
await asyncio.sleep(3) # wait 3 seconds
await ws.send_str('hello')
async def ws_handler(request):
ws = web.WebSocketResponse()
asyncio.ensure_future(ensure_client_logged(ws), loop=request.app.loop)
But the code is running in a blocking way, meaning the server becomes unresponsive while it is sleeping.
Can someone please point me in the right direction?
You need to establish a deadline for the authentication procedure. asyncio.wait_for is a convenient way to do that:
async def ws_handler(request):
loop = asyncio.get_event_loop()
ws = web.WebSocketResponse()
loop.create_task(handle_client(ws))
async def handle_client(ws):
try:
authenticated = await asyncio.wait_for(_authenticate(ws), 10)
except asyncio.TimeoutError:
authenticated = False
if not authenticated:
ws.close()
return
# continue talking to the client
async def _authenticate(ws):
# implement authentication here, without worrying about
# timeout - the coroutine will be automatically canceled
# once the timeout elapses
...
return True # if successfully authenticated
This is a full working example for the benefits of the future users:
from aiohttp import web
import asyncio
async def wait_for_authentication(ws, app):
async for msg in ws:
if msg.type == web.WSMsgType.TEXT and msg.data == 'AUTHENTICATE': # Implement your own authentication
await ws.send_str('WELCOME')
return True
else:
await ws.send_str('NOT AUTHENTICATED')
async def authenticate(ws, app) -> bool:
try:
authenticated = await asyncio.wait_for(wait_for_authentication(ws, app), 5)
except asyncio.TimeoutError:
authenticated = False
if not authenticated:
await ws.send_str('The AUTHENTICATE command was not received. Closing the connection...')
await ws.close()
return False
async def ws_handler(request):
ws = web.WebSocketResponse()
await ws.prepare(request)
await request.app.loop.create_task(authenticate(ws, request.app))
async for msg in ws:
if msg.type != web.WSMsgType.TEXT:
continue
await ws.send_str(msg.data)
def init():
app = web.Application()
app.router.add_get('/', ws_handler)
return app
web.run_app(init())

Resources