I try to implement a REQ/REP pattern, with python3 asyncio and ZeroMQ
My client async function:
import zmq
import os
from time import time
import asyncio
import zmq.asyncio
print ('Client %i'%os.getpid())
context = zmq.asyncio.Context(1)
loop = zmq.asyncio.ZMQEventLoop()
asyncio.set_event_loop(loop)
async def client():
socket = context.socket(zmq.REQ)
socket.connect('tcp://11.111.11.245:5555')
while True:
data = zmq.Message(str(os.getpid()).encode('utf8'))
start = time()
print('send')
await socket.send(data)
print('wait...')
data = await socket.recv()
print('recv')
print(time() - start, data)
loop.run_until_complete(client())
As I understand, the call to a socket.connect( "tcp://11.111.11.245:5555" ) method is a blocking method.
How to make a non-blocking connection call, in my case?
As far as I understand the ZeroMQ API, the call to .connect() method is not synchronous with building the real connection ( if not introduced by the wrapper, the underlying API is non-blocking - ref. below ).
The connection will not be performed immediately but as needed by ØMQ. Thus a successful invocation of zmq_connect() does not indicate that a physical connection was or can actually be established.
Ref.: ZeroMQ API - zmq_connect(3)
Related
I am new to asynchronous functions and threads, and I am trying to return a series of values obtained from a Web socket to pass to another thread where synchronous code is executing. In the code, I also use a multi Web socket approach. Below I show you the code:
"""
This code is designed to run an asynchronous loop
with asyncio in a separate thread. This allows mixing
a synchronous code with an asynchronous one.
"""
import asyncio
from datetime import datetime
from threading import Thread
import websockets
from typing import Tuple, List, Iterable
import json
import time
URLS = [
"wss://stream.binance.com:9443/ws/xrpusdt#kline_1m",
"wss://stream.binance.com:9443/ws/btcusdt#kline_1m",
]
def start_background_loop(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
loop.run_forever()
async def IndividualSubscription(url: str):
"""An individual subscription to each WebSocket is created"""
async with websockets.connect(url) as websocket:
data = await websocket.recv()
data = json.loads(data)
print('\n', data)
return data
async def Subscriptions(URLS: Iterable[str]):
"""All concurrent tickets are subscribed and all are combined
in a single coroutine."""
while True:
task = [asyncio.create_task(SuscripcionIndividual(url)) for url in URLS]
# All tasks are run in parallel
await asyncio.gather(*tareas)
#return tareas
def main():
loop = asyncio.new_event_loop()
t = Thread(target=start_background_loop, args=(loop,), daemon=True)
t.start()
task = asyncio.run_coroutine_threadsafe(Suscripciones(URLS), loop)
for i in task.result():
print(f"{i}")
#return tareas
def function():
for i in range(100):
print("This is out of asynchronous ", i)
time.sleep(1)
if __name__ == "__main__":
main()
T2 = Thread(target=function,)
T2.start()
I tried to just put return to the async code, but by doing this the async loop only runs once and not continuously as I would expect. Also, I've tried the method .result() over .create_task() . Is it possible to return values from an asynchronous function?
If you want interoperability between synchronous and asynchronous code you need to design some communication mechanism that won't block the thread running async code. Queues are commonly used for communication between threads, janus library implements queues compatible with threads running async code, it does so by exposing sync queue interface to sync code and async queue interface to async code.
Your code is a little chaotic, so I cleaned it up just to show communication between sync thread (main thread) and async thread (background thread running asyncio loop)
import asyncio
from datetime import datetime
from threading import Thread
import websockets
from typing import Tuple, List, Iterable
import json
import time
import janus # pip install janus
URLS = [
"wss://stream.binance.com:9443/ws/xrpusdt#kline_1m",
"wss://stream.binance.com:9443/ws/btcusdt#kline_1m",
]
def start_background_loop(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
loop.run_forever()
async def IndividualSubscription(url: str):
"""An individual subscription to each WebSocket is created"""
async with websockets.connect(url) as websocket:
return json.loads(await websocket.recv())
async def Subscriptions(URLS: Iterable[str], results: asyncio.Queue):
"""All concurrent tickets are subscribed and all are combined
in a single coroutine."""
while True:
tasks = [asyncio.create_task(SuscripcionIndividual(url)) for url in URLS]
for task in await asyncio.gather(*tasks):
await results.put(task.result())
def async_main(results: asyncio.Queue):
asyncio.run(Subscriptions(URLS, results))
if __name__ == "__main__":
results = janus.Queue(100) # max size of 100
async_thread = Thread(target=async_main, args=(results.async_q,))
async_thread.daemon = True # exit if main thread exits
async_thread.start()
while True:
print(f"[sync thread] got result from async thread: {results.sync_q.get()}")
I have an async method that I am trying to use to return data from a NodeJs application websocket interface.
When the method runs though I am just receiving a <Future pending> message within a websockets object.
What do I need to change in order for the method to receive and print the actual message from the server?
import asyncio
import json
import asyncws
import websockets
import requests
...
...
...
async def inject():
ws = await asyncws.connect(url + get_sid())
print("Sending...")
payload = base_query + f"(SELECT substring((SELECT token FROM AuthTokens WHERE UserId = 1),1,1))=3 #"
payload = string_to_json(payload, token)
payload = f'42["checkEmail", {payload}]'
ws.send(payload)
print("Sent...")
print("Receiving...")
reply = ws.recv()
if reply is None:
print("NoneType reply")
else:
print(reply)
for r in reply:
print(r)
asyncio.get_event_loop().run_until_complete(inject())
asyncio.get_event_loop().close()
Output:
Sending...
Sent...
Receiving...
<generator object Websocket.recv at 0x7fbb1813eb30>
<Future pending>
Websockets uses Python's asyncio to manage concurrency and I/O.
The websocket.send and websocket.recv are async functions. That means you need to await these to get the value.
In your code, you need to change ws.send(payload) to await ws.send(payload) and ws.recv() to await ws.recv().
You can read the documentation for Websockets here: https://websockets.readthedocs.io/en/stable/api.html#websockets.protocol.WebSocketCommonProtocol.recv
I am trying to make a discord bot using discord.py that uses the library quarry to interact with a minecraft server.
Unfortunately once I start the discord bot using bot.run() I can't start the reactor for quarry with reactor.run().
I've looked around including aiostream and asyncio but I can't find a solution. I also looked into twisted as quarry uses that.
EDIT: Including current code.
import asyncio
from twisted.internet import asyncioreactor
asyncioreactor.install(asyncio.get_event_loop())
from quarry.net.client import SpawningClientProtocol, ClientFactory
from quarry.net.auth import Profile
from discord.ext import commands
profile = Profile.from_credentials("email", "password")
bot = commands.Bot(command_prefix=">")
async def start():
loop = asyncio.get_event_loop()
loop.create_task(bot.start("token"))
client = MinecraftBotFactory(profile)
client.connect("creative.starlegacy.net", 25565)
class MinecraftBotProtocol(SpawningClientProtocol):
pass
class MinecraftBotFactory(ClientFactory):
protocol = MinecraftBotProtocol
asyncio.get_event_loop().create_task(start())
asyncio.get_event_loop().run_forever()
quarry uses twisted and in order to use it with an asyncio you will need to setup the AsyncioReactor at the start of your application.
import asyncio
from twisted.internet import asyncioreactor
asyncioreactor.install(asyncio.get_event_loop())
After that, you should only need to execute bot.start() since quarry relies on the reactor running, and just start the event loop. You cannot await reactor.run() because it's not a coroutine, so just get rid of that function.
Update: Added example and explanation
I had a chance to go over discord.py code a bit and found that discord.Client._connect() is basically "blocking code" if you await it, but only if you have another task waits for bot.start() to complete with in the same async def. In other words, it's an infinite poller loop that never returns anything, therefore cannot proceed past the await point. To avoid this you could create a Task. This example "should" work (I haven't tried it on a live server since I don't have a Minecraft client on hand). It starts a server and then when clients try to connect, they get a message saying the Minecraft server is down and then get booted. When a user tries to connect, a Discord message is also sent to the channel.
import asyncio
from twisted.internet import asyncioreactor
asyncioreactor.install(asyncio.get_event_loop())
import discord
from quarry.net.server import ServerFactory, ServerProtocol
from twisted.internet import endpoints, reactor
async def start():
loop = asyncio.get_event_loop()
minecraft_server = DowntimeFactory()
minecraft_server_host = "0.0.0.0"
minecraft_server_port = 25565
discord_bot_token = "APP_TOKEN"
discord_channel_id = 0 # CHANNEL ID
# Start a quarry server
quarry_server = endpoints.TCP4ServerEndpoint(
reactor,
port=minecraft_server_port,
interface=minecraft_server_host,
)
try:
await quarry_server.listen(minecraft_server).asFuture(loop)
except Exception as err:
print(err)
loop.stop()
# Start Discord poller
discord_client = discord.Client(loop=loop)
loop.create_task(discord_client.start(discord_bot_token))
#discord_client.event
async def on_ready():
"""
After the Discord client connects, set the discord_channel attribute in the
quarry factory so that server can access Discord functionality.
"""
discord_channel = discord_client.get_channel(discord_channel_id)
if discord_channel:
minecraft_server.discord_channel = discord_channel
await discord_channel.send("hello world")
else:
print("[!] channel no found")
loop.stop()
class DowntimeProtocol(ServerProtocol):
def packet_login_start(self, buff):
buff.discard()
self.close(self.factory.motd)
# The Discord channel might not have been set up yet
if self.factory.discord_channel:
self.factory.loop.create_task(
self.factory.discord_channel.send(
"Someone is trying to access the Minecraft server!"
)
)
class DowntimeFactory(ServerFactory):
motd = "Down for maintenance"
loop = asyncio.get_event_loop()
discord_channel = None
protocol = DowntimeProtocol
asyncio.get_event_loop().create_task(start())
try:
asyncio.get_event_loop().run_forever()
except KeyboardInterrupt:
pass
bot.run() is a blocking call, meaning that it stops the execution of the program until it is done. Try an alternative like await bot.start():
import asyncio
from discord.ext import tasks
async def start_bot():
await bot.start('my_token_goes_here')
#tasks.loop(count=1)
async def login_quarry():
# login to quarry here #
login_quarry.start()
# put any code you need BEFORE this:
asyncio.get_event_loop().run_until_complete(start_bot())
In python 3.7 and up, there is a simpler alternative:
import asyncio
from discord.ext import tasks
async def start_bot():
await bot.start('my_token_goes_here')
#tasks.loop(count=1)
async def login_quarry():
# login to quarry here #
login_quarry.start()
# put any code you need BEFORE this:
asyncio.run(start_bot())
In my project I try to start a REST API (built with FastAPI and run with Hypercorn), additional I want on startup also to start a RabbitMQ Consumer (with aio_pika):
Aio Pika offers a robust connection which automatically reconnects on failure. If I run the code below with hypercorn app:app the consumer and the rest interface starts correctly, but the reconnect from aio_pika does not work anymore. How can I archive a production stable RabbitMQ Consumer and RestAPI in two different processes (or threads?). My python version is 3.7, please note I am actually a Java and Go developer in case my approach is not the Python way :-)
#app.on_event("startup")
def startup():
loop = asyncio.new_event_loop()
asyncio.ensure_future(main(loop))
#app.get("/")
def read_root():
return {"Hello": "World"}
async def main(loop):
connection = await aio_pika.connect_robust(
"amqp://guest:guest#127.0.0.1/", loop=loop
)
async with connection:
queue_name = "test_queue"
# Creating channel
channel = await connection.channel() # type: aio_pika.Channel
# Declaring queue
queue = await channel.declare_queue(
queue_name,
auto_delete=True
) # type: aio_pika.Queue
async with queue.iterator() as queue_iter:
# Cancel consuming after __aexit__
async for message in queue_iter:
async with message.process():
print(message.body)
if queue.name in message.body.decode():
break
With the help of #pgjones I managed changed the consuming start to:
#app.on_event("startup")
def startup():
loop = asyncio.get_event_loop()
asyncio.ensure_future(main(loop))
And start the job with asyncio.ensure_future and pass the current event loop as an argument, which solved the issue.
Would be interesting if somebody has a different/better approach
Thanks!
I am playing with aiohttp+aiomysql. I want to share same connection pool instance between request calls.
So i create a global var and preinit it once in corouting call.
My code:
import asyncio
from aiohttp import web
from aiohttp_session import get_session, session_middleware
from aiohttp_session.cookie_storage import EncryptedCookieStorage
from aiohttp_session import SimpleCookieStorage
#from mysql_pool import POOL
from aiomysql import create_pool
M_POOL = None
async def get_pool(loop):
global M_POOL
if M_POOL: return M_POOL
M_POOL = await create_pool(host='127.0.0.1', port=3306, user='user', password='user', db='test', loop=loop)
return M_POOL
async def query(request):
loop = asyncio.get_event_loop()
pool = await get_pool(loop)
print(id(pool))
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute("SELECT 42;")
value = await cur.fetchone()
print(value)
return web.Response(body=str.encode(str(value)))
app = web.Application(middlewares=[session_middleware(SimpleCookieStorage())])
app.router.add_route('GET', '/query', query)
web.run_app(app)
Is it convinient way of doing this, or may be something better?
I highly discourage global variable usage.
Please take a look on aiohttp demo for canonical approach.
SiteHandler is a class that implements website views.
But you've given demos for a case where request object in access.
I have the same problem using aiohttp. In my application I've
done to parts of modules:
one is for server functionality, and one for client (crawler).
So in server part it's ok, I can user request.app['dbpool']
But in crawler part I want use db connections to, and
I can't see reasons for another one pool connection creation.