I wanted to use aiomysql in mutithreading mode. Then it failed. Here is my code.
import asyncio
import aiomysql
from threading import Thread
def start_loop(loop):
asyncio.set_event_loop(loop)
loop.run_forever()
async def create_pool(loop, **kw):
global __pool
__pool = await aiomysql.create_pool(
host=kw.get('host', 'localhost'),
port=kw.get('port', 3306),
user=kw['user'],
password=kw['password'],
db=kw['db'],
charset=kw.get('charset', 'utf8'),
autocommit=kw.get('autocommit', True),
maxsize=kw.get('maxsize', 10),
minsize=kw.get('minsize', 1),
loop=loop
)
async def select():
global __pool
async with __pool.acquire() as conn:
async with conn.cursor(aiomysql.DictCursor) as cur:
print('ok here')
await cur.execute('select * from test')
print('failed here')
rs = await cur.fetchall()
print('rows returned: %s' % len(rs))
loop = asyncio.get_event_loop()
new_loop = asyncio.new_event_loop()
t = Thread(target=start_loop, args=(new_loop,))
loop.run_until_complete(create_pool(loop, user='user', password='password', db='test'))
t.start()
asyncio.run_coroutine_threadsafe(select(), new_loop)
'ok here' can be printed while 'failed here' can't. So it fails in "await cur.execute('select * from test')". It'll work if I use create_pool and select in one thread. So it confuses me why it fails in multithreading mode.
Thanks.
Related
i have some troubles with parsing two or more sessions at the same time with telethon. I have tried this:
class NewSession:
def __init__(self, session_name):
self.client = TelegramClient(session_name, api_id, api_hash)
self.session_name = session_name
async def pool(self):
print("working with:", self.session_name)
#self.client.on(events.NewMessage(outgoing=True))
async def main(event):
message = event.message.to_dict()
msg_text = message['message']
print(msg_text)
try:
await self.client.start()
await self.client.run_until_disconnected()
finally:
await self.client.disconnect()
async def main():
user = NewSession("321")
user2 = NewSession("123")
await user.pool()
await user2.pool()
if __name__ == '__main__':
asyncio.run(main())
But only one is working. Need help :)
The problem is inside your main function. When you await for a coroutine to return it doesn't mean that the execution continues to the next expression. So, in your code the line await user2.pool() is going to be executed only when the user.poll() coroutines returns a value, this is when the session '321' is disconnected.
You need to run the tasks concurrently; you can use the function asyncio.gather. Reworking your main:
async def main():
user = NewSession("321")
user2 = NewSession("123")
await asyncio.gather(user.pool(), user2.pool())
I'm trying simple publish subscribe with nats.aio.client module but callback is not working below is my code
from nats.aio.client import Client as NATS
class NAT:
def init(self):
self.nc = NATS()
async def run(self):
# nc = NATS()
print("connection starts")
await self.nc.connect("demo.nats.io:4222", connect_timeout=10, verbose=True)
print("connection success")
async def publish_msg(self):
# nc = NATS()
print("msg to publish")
await self.nc.publish("Hello", b'Hellowelcome')
async def subscribe_msg(self):
async def message_handler(msg):
print("Hello")
subject = msg.subject
reply = msg.reply
print("Received a message on '{subject} {reply}'".format(
subject=subject, reply=reply))
await self.nc.subscribe("Hello", cb=message_handler)
main file
import asyncio
from nats_client import NAT
nat = NAT()
nats_connection = asyncio.get_event_loop()
nats_connection.run_until_complete(nat.run())
nats_connection.run_until_complete(nat.subscribe_msg())
nats_connection.run_until_complete(nat.publish_msg())
#nats_connection.close()
Let me know if I'm missing anything any help would be appreciated
I think your program might be exiting too early so can neither publish nor receive the message, here is a full example on how to start a service in NATS:
import asyncio
from nats.aio.client import Client as NATS
async def run(loop):
nc = NATS()
async def disconnected_cb():
print("Got disconnected...")
async def reconnected_cb():
print("Got reconnected...")
await nc.connect("127.0.0.1",
reconnected_cb=reconnected_cb,
disconnected_cb=disconnected_cb,
max_reconnect_attempts=-1,
loop=loop)
async def help_request(msg):
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
print("Received a message on '{subject} {reply}': {data}".format(
subject=subject, reply=reply, data=data))
await nc.publish(reply, b'I can help')
# Use queue named 'workers' for distributing requests
# among subscribers.
await nc.subscribe("help", "workers", help_request)
print("Listening for requests on 'help' subject...")
for i in range(1, 1000000):
await asyncio.sleep(1)
try:
response = await nc.request("help", b'hi')
print(response)
except Exception as e:
print("Error:", e)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(run(loop))
loop.run_forever()
loop.close()
My Scenario:- Start, Wait, Start, Stop or Kill
Starting the first event & waiting for some time.
If I reach the waiting time, I need to start the second event & return both event result.
But, if the first event completed before waiting time.
No need to start the second event.
Return the first event result
Ex:-
import asyncio
async def some_task():
print('io start')
await asyncio.sleep(2)
print('io end')
return "hello"
async def callback(loop):
await asyncio.sleep(4)
if loop.is_running():
print('doing other things')
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop2 = asyncio.get_event_loop()
a = loop.create_task(some_task())
b = loop2.create_task(callback(loop))
result = loop.run_until_complete(a)
loop2.run_until_complete(b)
loop.close()
loop2.close()
The variables loop and loop2 will get the same (main) event loop, which is why it will always still be running. Here is my approach:
import asyncio
async def async_main():
result = None
async def some_task():
print('io start')
await asyncio.sleep(6)
print('io end')
result = "hello"
return result
async def callback():
await asyncio.sleep(4)
if result is None:
print('doing other things')
awaited = await asyncio.gather(
asyncio.create_task(some_task()),
asyncio.create_task(callback()),
)
return awaited
asyncio.run(async_main())
Asyncio can be very confusing and I do not recommend it for non-experts, so here is an alternative using threading instead of asyncio:
import threading
import time
def do_tasks():
result = None
def some_task():
nonlocal result
print('io start')
time.sleep(2)
print('io end')
result = "hello"
def callback():
time.sleep(4)
if result is None:
print('doing other things')
threading.Thread(target=some_task).start()
threading.Thread(target=callback).start()
do_tasks()
How can I make the on_message function work asynchronously in my Tornado WebSocketClient?
I guess I need to await the on_message function, but I don't know how.
Or is there even a fundamental misconception in the way how I try to implement an asynchronous WebSocketClient?
import tornado.websocket
from tornado.queues import Queue
from tornado import gen
import json
q = Queue()
class WebsocketClient():
def __init__(self, url, connections):
self.url = url
self.connections = connections
print("CLIENT started")
print("CLIENT initial connections: ", len(self.connections))
async def send_message(self):
async for message in q:
try:
msg = json.loads(message)
print(message)
await gen.sleep(0.001)
finally:
q.task_done()
async def update_connections(self, connections):
self.connections = connections
print("CLIENT updated connections: ", len(self.connections))
async def on_message(self, message):
await q.put(message)
await gen.sleep(0.001)
async def connect(self):
client = await tornado.websocket.websocket_connect(url=self.url, on_message_callback=self.on_message)
RuntimeWarning: coroutine 'WebsocketClient.on_message' was never awaited
self._on_message_callback(message)
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
on_message_callback is supposed to be a regular function, not a coroutine. And it is meant to be used in old-style code when people used callbacks instead of coroutines.
For the newer async-style code, you don't need this callback. You can just do this:
async def connect(self):
client = await tornado.websocket.websocket_connect(url=self.url)
while True:
message = await client.read_message()
if message is None:
# None message means the connection was closed
break
print("Message received:", message)
await q.put(message)
await gen.sleep(0.001)
Hi I created async websocket client which I can receive and send messages asynchronously.
Here is the clint class that I use.
import websockets
import asyncio
from models import Heartbeat
from model_helper import ModelHelper
from json_helper import JSONHelper
from os.path import dirname, abspath
import logging
import time
waiting_time = 0.5
class WebSocketClient():
def __init__(self, websocket_queue, configs):
self.Heartbeat = Heartbeat().message
self.websocket_queue = websocket_queue
self.configs = configs
def get_connection_string(self):
server_url = self.configs.URL + ":" + str(self.configs.Port) + self.configs.Root
return server_url
async def connect(self):
try:
server_url = self.get_connection_string()
self.connection = await websockets.client.connect(server_url)
if self.connection.open:
print("Connection stablished. Client correcly connected")
# Send greeting
await self.connection.send(self.Heartbeat)
return self.connection
else:
print("Can not connect")
except ConnectionRefusedError as err:
print("Connection Error: {}".format(err))
async def send_message_to_socket(self, connection):
while True:
message = self.websocket_queue.get_send_queue()
try:
if message is not None:
message_ = ModelHelper.to_outgoing_request_model(message)
await connection.send(message_)
else:
await asyncio.sleep(waiting_time)
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
self.connect()
async def receive_message_from_socket(self, connection):
while True:
try:
message = await connection.recv()
obj = JSONHelper.toObject(message)
print("Received object from websocket: {}".format(obj))
#If a websocket entry has SendMessage in its Action property
#Consider it as an sms content to be sent.
if(obj.Action == "SendMessage"):
self.websocket_queue.add_received_queue(obj)
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed1')
async def send_heartbeat_to_socket(self, connection):
while True:
#print("Heartbeat loop\n")
try:
await connection.send(self.Heartbeat)
await asyncio.sleep(3)
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed2')
And this is the code where I listen for websocket messages:
def listen_websocket_routine(sms_queue, websocket_queue):
while True:
time.sleep(waiting_time)
#Check Websocket queue for new messages
message = websocket_queue.get_received_queue()
# If Incoming websocket JSON has SendMessage string in its Action attribute transform it to request_model
if message is not None and send_sms.Action == "SendMessage":
# Transform it to outgoing_sms_model
reset = ModelHelper.to_outgoing_sms_model(message.Data)
# Add To send_queue of sms
sms_queue.add_send_queue(reset)
finally How I initiate them with asyncio and threading.
client = WebSocketClient(websocket_queue, configs)
loop = asyncio.get_event_loop()
connection = loop.run_until_complete(client.connect())
task1 = asyncio.ensure_future(client.receive_message_from_socket(connection))
task2 = asyncio.ensure_future(client.send_heartbeat_to_socket(connection))
task3 = asyncio.ensure_future(client.send_message_to_socket(connection))
listen_websocket_thread = threading.Thread(target=listen_websocket_routine, args=(sms_queue, websocket_queue))
listen_websocket_thread.start()
loop.run_forever()
So my question is, whenever connetion breaks, I need to re establish the connection. But I am not sure where should I do that. Should it be before everytime I try to send message or receive or should I do that in more general approach?
Since update 10.0 you could do it with an async iterator, like this(example from the official doc):
async for websocket in websockets.connect(...):
try:
...
except websockets.ConnectionClosed:
continue