asyncpg + aiogram. cannot perform operation: another operation is in progress - python-3.x

How I can fix it? I played with it a lot and for a long time, but nothing came of it.
sql_db.py:
import asyncio
import asyncpg
LOG_PG = {"database": 'test_bot',
"user": 'misha',
"password": '1234',
"host": 'localhost'}
class Database:
SELECT_USER_LANG = "SELECT lang FROM lang_set WHERE user_id = $1 AND bot_token = $2"
def __init__(self, loop: asyncio.AbstractEventLoop):
self.pool = loop.run_until_complete(
asyncpg.create_pool(**LOG_PG)
)
async def get_lang(self, user_id, token):
search_d = [user_id, token]
res = await self.pool.fetchval(self.SELECT_USER_LANG, *search_d)
if res is None:
return "ru"
return res
I tried to insert this loop everywhere, run without it, multiple combinations in the code itself. But nothing changed. I do not know how to describe the problem in more detail
main.py:
from aiogram import Bot, Dispatcher
from aiogram.types import Message
import asyncio
from sql_db import Database
loop = asyncio.get_event_loop()
token = "TOKEN"
dp = Dispatcher()
bot = Bot(token=token, parse_mode="HTML")
db = Database(loop)
async def echo_msg(message: Message):
user_id = message.from_user.id
await message.send_copy(user_id)
await db.get_lang(user_id, token)
dp.message.register(callback=echo_msg)
if __name__ == '__main__':
dp.run_polling(bot, skip_updates=True)
error:
...
File "/home/mickey/Desktop/chat_admin/venv/lib/python3.8/site-packages/asyncpg/pool.py", line 867, in release
return await asyncio.shield(ch.release(timeout))
File "/home/mickey/Desktop/chat_admin/venv/lib/python3.8/site-packages/asyncpg/pool.py", line 224, in release
raise ex
File "/home/mickey/Desktop/chat_admin/venv/lib/python3.8/site-packages/asyncpg/pool.py", line 214, in release
await self._con.reset(timeout=budget)
File "/home/mickey/Desktop/chat_admin/venv/lib/python3.8/site-packages/asyncpg/connection.py", line 1367, in reset
await self.execute(reset_query, timeout=timeout)
File "/home/mickey/Desktop/chat_admin/venv/lib/python3.8/site-packages/asyncpg/connection.py", line 318, in execute
return await self._protocol.query(query, timeout)
File "asyncpg/protocol/protocol.pyx", line 323, in query
File "asyncpg/protocol/protocol.pyx", line 707, in asyncpg.protocol.protocol.BaseProtocol._check_state
asyncpg.exceptions._base.InterfaceError: cannot perform operation: another operation is in progress

Works through such a launch. It must be turned on through the aiogram. I do not know how to formulate, but I was lucky to understand the problem
...
data_ = {}
class Database:
def __init__(self, pool: asyncpg.create_pool):
self.pool = pool
async def get_lang(self, user_id, token):
search_d = [user_id, token]
async with self.pool.acquire() as conn:
res = await conn.fetchval(SELECT_USER_LANG, *search_d)
if res is None:
return "ru"
return res
async def create_pool():
pool = await asyncpg.create_pool(**LOG_PG)
data_["db"] = Database(pool)
async def echo_msg(message: Message):
user_id = message.from_user.id
await message.send_copy(user_id)
await data_["db"].get_lang(user_id, token)
dp.message.register(callback=echo_msg)
if __name__ == '__main__':
dp.startup.register(create_pool) # ANSWER
dp.run_polling(bot, skip_updates=True)

Related

Python await is not finished but skipped

I am trying to implement an async database connection library to my FastAPI program, but it seems like python skips my await keyword.
My expected behaviour is that the login func will call get_user_info, which will print "before calling query" and then call _query and print "about to execute" and then print the result.
The actual outcome:
before calling query
user: None
INFO: 127.0.0.1:60548 - "POST /api/auth/login HTTP/1.1" 401 Unauthorized
2021-05-24 12:31:44.007 | INFO | authentication.routes:login:31 - 127.0.0.1 tried to log in with user x
my code:
route.py
#auth_app.post("/auth/login", response_model=Token, tags=["Authentication"])
async def login(request: Request, form_data: OAuth2PasswordRequestForm = Depends()):
credentials_exceptions = HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect credentials.",
headers={"Authenticate": "Bearer"}
)
username = form_data.username
password = form_data.password
async with MysqlConnector() as conn:
user = await conn.get_user_info(username)
print("user:", user)
logger.info(f"{request.client.host} tried to log in with user {username}")
if not user:
raise credentials_exceptions
My MysqlConnector() class:
class MysqlConnector:
async def __aenter__(self):
await self._create_connection()
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.connection.disconnect()
def __await__(self):
return self.__aenter__().__await__()
#staticmethod
def _create_conn_string():
return f"mysql://{MYSQL_SETTINGS['user']}:{MYSQL_SETTINGS['password']}#{MYSQL_SETTINGS['host']}:" \
f"{MYSQL_SETTINGS['port']}/{MYSQL_SETTINGS['database']}"
async def disconnect(self):
await self.connection.disconnect()
async def _create_connection(self):
MYSQL_SETTINGS['use_unicode'] = True
MYSQL_SETTINGS['charset'] = 'utf8'
self.connection = Database(self._create_conn_string())
await self.connection.connect()
async def _query(self, sql_query, dictionary=False, **kwargs):
print("about to execute")
await self.connection.execute(query=sql_query, values=kwargs)
self.res = await self.connection.fetch_all(query=sql_query)
print("result: ", self.res)
async def get_user_info(self, username: str) -> str:
sql_query = """
CALL `sp_get_user`(:username);
"""
try:
print("before calling query")
await self._query(sql_query, username=username)
result = self.res
print("result:")
print(result)
except IndexError:
return
user = User( # User is a pydantic model
user_id=result[0],
username=result[1],
first_name=result[2],
last_name=result[3],
role=result[4],
hashed_password=result[5],
email_addr=result[6],
phone=result[7],
desc=result[8],
last_login=datetime.strptime(result[9], '%Y-%m-%d %H:%M:%S.%f'),
active=result[10],
api_access=result[11]
)
print("user:", user)
return user
Thank you for any help!

Send Discord Message from a looped function

So I've run into a wall trying to get this code to work.
I'm writing a fairly simple Discord bot that does two things.
The main part of the bot is to run an API request when the bot initializes to get a base value of a return. I then have a function that I want to run every X seconds, that checks the value again, and if it changes, send a message to a Discord Channel and then update the variable to the new value, so that the loop can report when it changes again.
The second part, which is not as critical but I still want to include it, is that a user can send a bot command to report the current value of the return.
Problem #1: Arises when I start the loop without the code to send the result via discord message. If i don't include the line to send the discord message in the loop code, the loop runs and prints out the results to the console (for test purposes). However, when the loop is running, I can no longer get the bot to respond to the command.
Problem #2: If I include the code to send the discord message, the whole thing immediately fails and gives me the following error:
File "bot.py", line 67, in <module>
asyncio.run(update_status())
File "/usr/lib/python3.7/asyncio/runners.py", line 43, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.7/asyncio/base_events.py", line 584, in run_until_complete
return future.result()
File "bot.py", line 63, in update_status
await channel.send(f'Server status is still: {updatestatus}.')
File "/usr/local/lib/python3.7/dist-packages/discord/abc.py", line 905, in send
nonce=nonce, allowed_mentions=allowed_mentions)
File "/usr/local/lib/python3.7/dist-packages/discord/http.py", line 185, in request
async with self.__session.request(method, url, **kwargs) as r:
File "/usr/local/lib/python3.7/dist-packages/aiohttp/client.py", line 1012, in __aenter__
self._resp = await self._coro
File "/usr/local/lib/python3.7/dist-packages/aiohttp/client.py", line 357, in _request
raise RuntimeError('Session is closed')
RuntimeError: Session is closed
Problem #3: More of a minor nuisance, but if I set my code up so that it works in Problem #1, the loop does not start automatically, but only after I press Ctrl+C (note: I am writing the code on a Raspi 4 via PuTTY.
Any thoughts?
Full Code:
# bot.py
import os
import requests
import discord
import json
import asyncio
from dotenv import load_dotenv
from discord.ext import commands
from discord.ext import tasks
from discord.utils import get
load_dotenv()
TOKEN = os.getenv('DISCORD_TOKEN')
CLIENTID = os.getenv('CLIENT_ID')
CLIENTSECRET = os.getenv('CLIENT_SECRET')
bot = commands.Bot(command_prefix='+')
#bot.event
async def on_ready():
print(f'{bot.user.name} has connected to Discord!')
channel = bot.get_channel(795162312112865280)
await channel.send('MurkyBot has connected')
def create_access_token(client_id, client_secret, region = 'us'):
data = { 'grant_type': 'client_credentials' }
response = requests.post('https://%s.battle.net/oauth/token' % region, data=data, auth=(client_id, client_secret))
return response.json()
tokenresponse = create_access_token(CLIENTID, CLIENTSECRET)
accesstoken = tokenresponse["access_token"]
print(accesstoken)
initialrequest = requests.get(f'https://us.api.blizzard.com/data/wow/connected-realm/154?namespace=dynamic-us&locale=en_US&access_token={accesstoken}')
initialrequest = initialrequest.json()
initialstatus = initialrequest['status']['type']
print(f'Initial status: {initialstatus}')
#bot.command(name='status', help='Gets the current server status')
async def manual_status(ctx):
manualrequest = requests.get(f'https://us.api.blizzard.com/data/wow/connected-realm/154?namespace=dynamic-us&locale=en_US&access_token={accesstoken}')
manualrequest = manualrequest.json()
manualstatus = manualrequest['status']['type']
channel = bot.get_channel(795162312112865280)
await ctx.send(f'Current world server status is: {manualstatus}')
bot.run(TOKEN)
async def update_status():
while True:
global initialstatus
channel = bot.get_channel(795162312112865280)
updaterequest = requests.get(f'https://us.api.blizzard.com/data/wow/connected-realm/154?namespace=dynamic-us&locale=en_US&access_token={accesstoken}')
updaterequest = updaterequest.json()
updatestatus = updaterequest['status']['type']
if updatestatus != initialstatus:
await channel.send(f'Server status has changed to: {updatestatus}!')
initialstatus = updatestatus
print('Status Change')
await asyncio.sleep(5)
else:
await channel.send(f'Server status is still: {updatestatus}.')
print('No Change')
await asyncio.sleep(5)
asyncio.run(update_status())
.run is a blocking call, meaning that everything stops and waits for it to complete. Therefore, the second part can only run after you ctrl+c, ending the script however.
You may want to check out discord.ext.tasks, it will have the intended functionality.
For what you're trying to do, here would be the updated code using discord.ext.tasks:
from dotenv import load_dotenv
from discord.ext import commands
from discord.ext import tasks
from discord.utils import get
load_dotenv()
TOKEN = os.getenv('DISCORD_TOKEN')
CLIENTID = os.getenv('CLIENT_ID')
CLIENTSECRET = os.getenv('CLIENT_SECRET')
bot = commands.Bot(command_prefix='+')
#tasks.loop(seconds=60.0)
async def update_status():
while True:
global initialstatus
channel = bot.get_channel(795162312112865280)
updaterequest = requests.get(f'https://us.api.blizzard.com/data/wow/connected-realm/154?namespace=dynamic-us&locale=en_US&access_token={accesstoken}')
updaterequest = updaterequest.json()
updatestatus = updaterequest['status']['type']
if updatestatus != initialstatus:
await channel.send(f'Server status has changed to: {updatestatus}!')
initialstatus = updatestatus
print('Status Change')
await asyncio.sleep(5)
else:
await channel.send(f'Server status is still: {updatestatus}.')
print('No Change')
await asyncio.sleep(5)
#bot.event
async def on_ready():
update_status.start()
print(f'{bot.user.name} has connected to Discord!')
channel = bot.get_channel(795162312112865280)
await channel.send('MurkyBot has connected')
def create_access_token(client_id, client_secret, region = 'us'):
data = { 'grant_type': 'client_credentials' }
response = requests.post('https://%s.battle.net/oauth/token' % region, data=data, auth=(client_id, client_secret))
return response.json()
tokenresponse = create_access_token(CLIENTID, CLIENTSECRET)
accesstoken = tokenresponse["access_token"]
print(accesstoken)
initialrequest = requests.get(f'https://us.api.blizzard.com/data/wow/connected-realm/154?namespace=dynamic-us&locale=en_US&access_token={accesstoken}')
initialrequest = initialrequest.json()
initialstatus = initialrequest['status']['type']
print(f'Initial status: {initialstatus}')
#bot.command(name='status', help='Gets the current server status')
async def manual_status(ctx):
manualrequest = requests.get(f'https://us.api.blizzard.com/data/wow/connected-realm/154?namespace=dynamic-us&locale=en_US&access_token={accesstoken}')
manualrequest = manualrequest.json()
manualstatus = manualrequest['status']['type']
channel = bot.get_channel(795162312112865280)
await ctx.send(f'Current world server status is: {manualstatus}')
bot.run(TOKEN)

Python continously receive message through websocket

I am trying to connect through websocket and keeping it alive to retrieve continously message from the server. I wrote the clientHelper and find the socketManager and ReconnectingWebsocket on the internet and I have not much idea on what going wrong with it as I do not receive anything thought the clientHelper.process_user_message function.
Can someone point me out the error please?
import websockets as ws
import asyncio
from client import Client
class ClientHelper(Client):
def __init__(self, api_key, api_secret):
super().__init__(api_key, api_secret)
self.loop = asyncio.get_event_loop()
def _request(self, method, uri, signed, force_params=False, **kwargs):
kwargs = self._get_request_kwargs(method, signed, force_params, **kwargs)
response = getattr(self.session, method)(uri, **kwargs)
return self._handle_response(response, method)
def _request(self, method, path, signed=False, version=API_VERSION, **kwargs):
uri = self._create_api_uri(path, signed, version)
return self._request(method, uri, signed, **kwargs)
def get_listen_key(self):
res = self._request('post', 'userDataStream', signed=True, data={})
return res['listenKey']
async def start_websockets(self):
self.sm = SocketManager(self, self.loop)
await self.sm.start_socket(self.process_user_message)
async def process_user_message(self, msg):
self.msg = msg
print(msg)
async def main(self)
await self.start_websockets()
while True:
await client.getInfo()
if 'data' in self.msg:
print(self.msg['data'])
def start(self):
self.loop.run_until_complete(self.main())
class SocketManager:
STREAM_URL = url
def __init__(self, client, loop, user_timeout=DEFAULT_USER_TIMEOUT):
self._client = client
self._loop = loop
self._conns = None
async def _start_user_socket(self, path, coro, prefix='ws/'):
if path in self._conns:
return False
self._conns[path] = ReconnectingWebsocket(self._loop, path, coro, prefix)
return path
async def start_user_socket(self, coro):
user_listen_key = await self._client.stream_get_listen_key() # manage to get the key from serveur
conn_key = await self._start_user_socket('user', user_listen_key, coro)
return conn_key
class ReconnectingWebsocket:
STREAM_URL = url
MAX_RECONNECTS = 5
MAX_RECONNECT_SECONDS = 60
MIN_RECONNECT_WAIT = 0.1
TIMEOUT = 10
def __init__(self, loop, path, coro, prefix='ws/'):
self._loop = loop
self._log = logging.getLogger(__name__)
self._path = path
self._coro = coro
self._prefix = prefix
self._reconnects = 0
self._conn = None
self._socket = None
self._connect()
def _connect(self):
self._conn = asyncio.ensure_future(self._run(), loop=self._loop)
async def _run(self):
keep_waiting = True
async with ws.connect(self.STREAM_URL) as socket:
self._socket = socket
self._reconnects = 0
try:
while keep_waiting:
try:
#evt = await self._coro(evt_obj)
evt = await asyncio.wait_for(self._socket.recv(), timeout=self.TIMEOUT)
except asyncio.TimeoutError:
#self._log.debug("no message in {} seconds".format(self.TIMEOUT))
print("no message in {} seconds".format(self.TIMEOUT))
await self.send_ping()
except asyncio.CancelledError:
#self._log.debug("cancelled error")
print("cancelled error")
await self.send_ping()
else:
try:
evt_obj = json.loads(evt)
except ValueError:
#self._log.debug('error parsing evt json:{}'.format(evt))
print('error parsing evt json:{}'.format(evt))
else:
await self._coro(evt_obj)
except ws.ConnectionClosed as e:
#self._log.debug('ws connection closed:{}'.format(e))
print('ws connection closed:{}'.format(e))
await self._reconnect()
except Exception as e:
#self._log.debug('ws exception:{}'.format(e))
print('ws exception:{}'.format(e))
await self._reconnect()
def _get_reconnect_wait(self, attempts: int) -> int:
expo = 2 ** attempts
return round(random() * min(self.MAX_RECONNECT_SECONDS, expo - 1) + 1)
async def _reconnect(self):
await self.cancel()
self._reconnects += 1
if self._reconnects < self.MAX_RECONNECTS:
reconnect_wait = self._get_reconnect_wait(self._reconnects)
await asyncio.sleep(reconnect_wait)
self._connect()
else:
self._log.error('Max reconnections {} reached:'.format(self.MAX_RECONNECTS))
async def send_ping(self):
if self._socket:
await self._socket.ping()
async def cancel(self):
self._conn.cancel()
self._socket = None

netdev lib co-routine exception

I've been trying netdev lib for some time now and the program below never worked it throughs the following exception:
Traceback (most recent call last):
File "D:/Code/async_npa/async_npa.py", line 93, in
r = asyncio.run(main(dev_data()))
File "C:\Users\omera\AppData\Local\Programs\Python\Python38-32\lib\asyncio\runners.py",
line 43, in run
return loop.run_until_complete(main)
File "C:\Users\omera\AppData\Local\Programs\Python\Python38-32\lib\asyncio\base_events.py",
line 612, in run_until_complete
return future.result()
File "D:/Code/async_npa/async_npa.py", line 88, in main
result = await asyncio.gather(task for task in tasks)
File "D:/Code/async_npa/async_npa.py", line 88, in
result = await asyncio.gather(task for task in tasks)
RuntimeError: Task got bad yield:
sys:1: RuntimeWarning: coroutine 'device_connection' was never awaited
I've also tried using the old syntax of asyncio creating an event loop and tasks but still no luck
code block:
from jinja2 import Environment, FileSystemLoader
import yaml
import asyncio
import netdev
def j2_command(file_name: dict = 'script.j2', directory: str = '.') -> dict:
env = Environment(loader=FileSystemLoader(directory))
temp = env.get_template(file_name)
temp_1 = temp.render()
temp_1 = temp_1.split('\n')
return temp_1
def get_host_name(open_connection) -> str:
hostname = open_connection.base_prompt()
hostname = hostname.split('#')[0]
return hostname
def write_to_file(data, dev_conn):
with open(f'./output/config_{get_host_name(dev_conn)}.txt', 'w') as conf:
conf.write(data)
def load_yml(yaml_file='inventory.yml'):
with open(yaml_file) as f:
host_obj = yaml.safe_load(f)
return host_obj
async def device_connection(connect_param):
dev_connect = netdev.create(**connect_param)
await dev_connect.connect()
commands = j2_command()
output = [f'\n\n\n\n\n########################## 1'
f' ##########################\n\n\n\n\n']
for command in commands:
breaker = f'\n\n\n\n\n########################## {command} ##########################\n\n\n\n\n'
command_result = await dev_connect.send_command(command)
output.append(breaker + command_result)
await dev_connect.disconnect()
output_result_string = "\n\n".join(output)
return output_result_string
def dev_data():
device_data = []
# devices_names = []
host_obj = load_yml()
generic_data = host_obj[0]['generic_data']
generic_username = generic_data['username']
generic_password = generic_data['password']
devices = host_obj[0]['devices']
device_type = generic_data['device_type']
device_secret = generic_data['secret']
for device in devices:
device_ip = device['ip_address']
try:
if device["username"]: generic_username = device['username']
if device['password']: generic_password = device['password']
if device["device_type"]: device_type = device['device_type']
if device['secret']: device_secret = device['secret']
except:
pass
dev = {
'device_type': device_type,
'host': device_ip,
'username': generic_username,
'password': generic_password,
'secret': device_secret
}
print(dev)
device_data.append(dev)
return device_data
async def main(device_data):
tasks = [device_connection(dev) for dev in device_data]
result = await asyncio.gather(task for task in tasks)
return result
if __name__ == '__main__':
r = asyncio.run(main(dev_data()))
print(r)
any help would be appreciated
Sorry for my late response but i hope it will help you. it seems like you have problem on running tasks.
instead of returning results in device_connection() you can define a global output_result_stringvariable and append it in every task. By this way you, don't have to collect anything in main()
Then change the main() with run() like below:
async def run(device_data):
tasks = [device_connection(dev) for dev in device_data]
await asyncio.wait(tasks)
and start it in your main block:
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(run())
Here is the docs link: netdev example link

Python async AttributeError aexit

I keep getting error AttributeError: __aexit__ on the code below, but I don't really understand why this happens.
My Python version is: 3.6.4 (v3.6.4:d48eceb, Dec 19 2017, 06:04:45) [MSC v.1900 32 bit (Intel)]
import aiohttp
import asyncio
import tqdm
async def fetch_url(session_, url_, timeout_=10):
with aiohttp.Timeout(timeout_):
async with session_.get(url_) as response:
text = await response.text()
print("URL: {} - TEXT: {}".format(url_, len(text)))
return text
async def parse_url(session, url, timeout=10):
# get doc from url
async with await fetch_url(session, url, timeout) as doc:
print("DOC: {}".format(doc, len(doc)))
return doc
async def parse_urls(session, urls, loop):
tasks = [parse_url(session, url) for url in urls]
responses = [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total = len(tasks))]
return responses
if __name__ == '__main__':
tickers = ['CTXS', 'MSFT', 'AAPL', 'GPRO', 'G', 'INTC', 'SYNC', 'SYNA']
urls = ["https://finance.yahoo.com/quote/{}".format(ticker) for ticker in tickers]
loop = asyncio.get_event_loop()
with aiohttp.ClientSession(loop=loop) as session:
parsed_data = loop.run_until_complete(parse_urls(session, urls, loop))
print(parsed_data)
Error callstack:
C:\Python\Python36\python.exe C:/Users/me/.PyCharmCE2017.3/config/scratches/scratch_4.py
0%| | 0/8 [00:00<?, ?it/s]Traceback (most recent call last):
URL: https://finance.yahoo.com/quote/CTXS - TEXT: 462138
File "C:/Users/me/.PyCharmCE2017.3/config/scratches/scratch_4.py", line 34, in <module>
parsed_data = loop.run_until_complete(parse_urls(session, urls, loop))
File "C:\Python\Python36\lib\asyncio\base_events.py", line 467, in run_until_complete
return future.result()
File "C:/Users/me/.PyCharmCE2017.3/config/scratches/scratch_4.py", line 23, in parse_urls
responses = [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total = len(tasks))]
File "C:/Users/me/.PyCharmCE2017.3/config/scratches/scratch_4.py", line 23, in <listcomp>
responses = [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total = len(tasks))]
File "C:\Python\Python36\lib\asyncio\tasks.py", line 458, in _wait_for_one
return f.result() # May raise f.exception().
File "C:/Users/me/.PyCharmCE2017.3/config/scratches/scratch_4.py", line 16, in parse_url
async with await fetch_url(session, url, timeout) as doc:
AttributeError: __aexit__
Process finished with exit code 1
You are trying to use fetch_url as a context manager, but it isn't one. You can either make it one
class fetch_url:
def __init__(self, session, url, timeout=10):
self.session = session
self.url = url
self.timeout = timeout
async def __aenter__(self):
with aiohttp.Timeout(self.timeout):
async with self.session.get(self.url) as response:
text = await response.text()
print("URL: {} - TEXT: {}".format(self.url, len(text)))
return text
async def __aexit__(self, exc_type, exc, tb):
# clean up anything you need to clean up
or change your code to
async def parse_url(session, url, timeout=10):
# get doc from url
doc = await fetch_url(session, url, timeout)
print("DOC: {}".format(doc, len(doc)))
return doc

Resources