I have switched my SQLAlchemy database code to use an async engine and am having trouble establishing basic functionality.
I have a class that starts the database like this:
class PostgresDb:
def __init__(self):
self._session = None
self._engine = None
def __getattr__(self, name):
return getattr(self._session, name)
def init(self):
self._engine = create_async_engine(
ENGINE,
echo=True,
future=True)
self._session = sessionmaker(
self._engine, expire_on_commit=False, class_=AsyncSession
)()
async def create_all(self):
async with self._engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
# Other methods...
Example of how create_all gets called:
async def init_db_tables():
self.init()
await self.create_all()
asyncio.run(init_db_tables())
When I want to achieve basic functionality, like getting all the tables, I can do something like:
def get_tables(self):
with create_engine(SYNCHRONOUS_ENGINE).connect() as conn:
meta = MetaData(conn, schema=SCHEMA)
meta.reflect(views=True)
table_list = meta.tables
return table_list
This is not ideal as I need to actually pass in a synchronous engine connection instead of the actual async engine I am using in the Class. It is also very verbose and shouldn't need to be initiated like this for every query.
I have tried doing something like this to select the table 'appuser' from the database:
async def get_tables(self):
self.init()
async with self._session() as session:
q = select('appuser')
result = await session.execute(q)
curr = result.scalars()
for i in curr:
print(i)
Which I've tried calling like this
db = PostgresDb()
asyncio.run(db.get_tables())
asyncio.get_event_loop().run_until_complete(db.get_tables())
These both give error:
async with self._session() as session:
TypeError: 'AsyncSession' object is not callable
Calling it with db.get_tables() errors RuntimeWarning: coroutine 'PostgresDb.get_tables' was never awaited db.get_tables() RuntimeWarning: Enable tracemalloc to get the object allocation traceback
Trying to use inspector with run_sync like this:
async def get_tables(self):
await self.init()
async with self._engine.begin() as conn:
inspector = conn.run_sync(inspect(conn))
table_names = await conn.run_sync(inspector.get_table_names())
print(table_names)
Returns error
sqlalchemy.exc.NoInspectionAvailable: Inspection on an AsyncConnection is currently not supported. Please use ``run_sync`` to pass a callable where it's possible to call ``inspect`` on the passed connection.
I have read the documentation at https://docs.sqlalchemy.org/en/14/orm/extensions/asyncio.html#sqlalchemy.ext.asyncio.AsyncConnection.run_sync but I am still unclear about how to work cleanly with async engines.
Thanks for any and all insight you're able to offer on how to execute a simple query get all tables in SQLAlchmey using the async engine!
Related
I've met some problem with running tests using FastAPI+SQLAlchemy and PostgreSQL, which leads to lots of errors (however, it works well on SQLite). I've created a repo with MVP app and Pytest on Docker Compose testing.
The basic error is sqlalchemy.exc.InterfaceError('cannot perform operation: another operation is in progress'). This may be related to the app/DB initialization, though I checked that all the operations get performed sequentially. Also I tried to use single instance of TestClient for the all tests, but got no better results. I hope to find a solution, a correct way for testing such apps 🙏
Here are the most important parts of the code:
app.py:
app = FastAPI()
some_items = dict()
#app.on_event("startup")
async def startup():
await create_database()
# Extract some data from env, local files, or S3
some_items["pi"] = 3.1415926535
some_items["eu"] = 2.7182818284
#app.post("/{name}")
async def create_obj(name: str, request: Request):
data = await request.json()
if data.get("code") in some_items:
data["value"] = some_items[data["code"]]
async with async_session() as session:
async with session.begin():
await create_object(session, name, data)
return JSONResponse(status_code=200, content=data)
else:
return JSONResponse(status_code=404, content={})
#app.get("/{name}")
async def get_connected_register(name: str):
async with async_session() as session:
async with session.begin():
objects = await get_objects(session, name)
result = []
for obj in objects:
result.append({
"id": obj.id, "name": obj.name, **obj.data,
})
return result
tests.py:
#pytest.fixture(scope="module")
def event_loop():
loop = asyncio.get_event_loop()
yield loop
loop.close()
#pytest_asyncio.fixture(scope="module")
#pytest.mark.asyncio
async def get_db():
await delete_database()
await create_database()
#pytest.mark.parametrize("test_case", test_cases_post)
def test_post(get_db, test_case):
with TestClient(app)() as client:
response = client.post(f"/{test_case['name']}", json=test_case["data"])
assert response.status_code == test_case["res"]
#pytest.mark.parametrize("test_case", test_cases_get)
def test_get(get_db, test_case):
with TestClient(app)() as client:
response = client.get(f"/{test_case['name']}")
assert len(response.json()) == test_case["count"]
db.py:
DATABASE_URL = environ.get("DATABASE_URL", "sqlite+aiosqlite:///./test.db")
engine = create_async_engine(DATABASE_URL, future=True, echo=True)
async_session = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
Base = declarative_base()
async def delete_database():
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.drop_all)
async def create_database():
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
class Model(Base):
__tablename__ = "smth"
id = Column(Integer, primary_key=True)
name = Column(String, nullable=False)
data = Column(JSON, nullable=False)
idx_main = Index("name", "id")
async def create_object(db: Session, name: str, data: dict):
connection = Model(name=name, data=data)
db.add(connection)
await db.flush()
async def get_objects(db: Session, name: str):
raw_q = select(Model) \
.where(Model.name == name) \
.order_by(Model.id)
q = await db.execute(raw_q)
return q.scalars().all()
At the moment the testing code is quite coupled, so the test suite seems to work as follows:
the database is created once for all tests
the first set of tests runs and populates the database
the second set of tests runs (and will only succeed if the database is fully populated)
This has value as an end-to-end test, but I think it would work better if the whole thing were placed in a single test function.
As far as unit testing goes, it is a bit problematic. I'm not sure whether pytest-asyncio makes guarantees about test running order (there are pytest plugins that exist solely to make tests run in a deterministic order), and certainly the principle is that unit tests should be independent of each other.
The testing is coupled in another important way too - the database I/O code and the application logic are being tested simultaneously.
A practice that FastAPI encourages is to make use of dependency injection in your routes:
from fastapi import Depends, FastAPI, Request
...
def get_sessionmaker() -> Callable:
# this is a bit baroque, but x = Depends(y) assigns x = y()
# so that's why it's here
return async_session
#app.post("/{name}")
async def create_obj(name: str, request: Request, get_session = Depends(get_sessionmaker)):
data = await request.json()
if data.get("code") in some_items:
data["value"] = some_items[data["code"]]
async with get_session() as session:
async with session.begin():
await create_object(session, name, data)
return JSONResponse(status_code=200, content=data)
else:
return JSONResponse(status_code=404, content={})
When it comes to testing, FastAPI then allows you to swap out your real dependencies so that you can e.g. mock the database and test the application logic in isolation from database I/O:
from app import app, get_sessionmaker
from mocks import mock_sessionmaker
...
client = TestClient(app)
...
async def override_sessionmaker():
return mock_sessionmaker
app.dependency_overrides[get_sessionmaker] = override_sessionmaker
# now we can run some tests
This will mean that when you run your tests, whatever you put in mocks.mock_sessionmaker will give you the get_session function in your tests, rather than get_sessionmaker. We could have our mock_sessionmaker return a function called get_mock_session.
In other words, rather than with async_session() as session:, in the tests we'd have with get_mock_session() as session:.
Unfortunately this get_mock_session has to return something a little complicated (let's call it mock_session), because the application code then does an async with session.begin().
I'd be tempted to refactor the application code for simplicity, but if not then it will have to not throw errors when you call .begin, .add, and .flush on it, in this example, and those methods have to be async. But they don't have to do anything, so it's not too bad...
The FastAPI docs have an alternative example of databases + dependencies that does leave the code a little coupled, but uses SQLite strictly for the purpose of unit tests, leaving you free to do something different for an end-to-end test and in the application itself.
I have a class i created thats a websocket and it connects to my data endpoint with no issues. However, I wanted my socket to run forever. I am using the websockets python library. Heres a sample:
from websockets import connect
class Socket(metaclass=ABCMeta):
def __init__(self, url: str):
self.url = url
async def __aenter__(self):
self._conn = connect(self.url, ping_interval=None)
self.websocket = await self._conn.__aenter__()
return self
async def __aexit__(self, *args, **kwargs):
await self._conn.__aexit__(*args, **kwargs)
Now, i am able to write async with statement with no problems. My issue arises when i want my socket to remain connected.
reading in the library, it seems one suggested way is to do the following:
async for socket in websockets.connect(", ping_interval=None):
try:
your logic
except websockets.closedConnection as e:
continue
This allows me to keep trying to connect if there is an issue. How would i incorporate this into my class as an iterator? I tried the following but getting error:
TypeError: 'async for' received an object from __aiter__ that does not implement __anext__: coroutine
After i added the following code in the above class:
async def __aiter__(self):
return self
async def __anext__(self):
async for websocket in connect(self.url, ping_interval=None):
try:
self.websocket = await websocket
except StopIteration:
raise StopAsyncIteration
I am not posting my entire code here as the goal is to encapsulate a class around this socket class i created with the goal being
async for object in MyCustomClassSocketIterator(url):
try:
await object.send()
await object.receive()
except websockets.closedConnection as e:
etc....
where the encapsulated class has implemented receive() and send() functions. So each time program starts, object is instantiated asynchronously. If anything breaks...then it attemps to connect again if there is a socket.closedConnection. Thanks
so far I've been using mysql.connector to manage mysql in my discord bot, however, since that's not async, I'm trying to change to aiomysql. It works, kinda... I'm having a problem where sometimes I get something like
ERROR:asyncio:Task was destroyed but it is pending!
task: <ClientEventTask state=pending event=on_raw_message_delete coro=<bound method LoggingSystem.on_raw_message_delete of <cogs.loggingsystem.LoggingSystem object at 0x059C7A48>>>
In this case it was in on_raw_message_delete of a cog, but the event where this happens isn't always the same and it's only on events, never in commands.
So the question is, am I doing something wrong that may cause that?
# mysqlconnection.py
pool: aiomysql.Pool = None
async def getpool():
global pool
pool = await aiomysql.create_pool(host="", user="", password="", db="", pool_recycle=0)
async def execute_search(search: str, vals: tuple = None): # This function is what I use to get data from the DB across the whole bot
async with pool.acquire() as connection:
async with connection.cursor() as cursor:
if vals is None:
await cursor.execute(search)
else:
await cursor.execute(search, vals)
result = await cursor.fetchall()
return result
async def execute_alter(query: str, vals: tuple): # This function is what I use to alter data in the DB across the whole bot
async with pool.acquire() as connection:
async with connection.cursor() as cursor:
await cursor.execute(query, vals)
await connection.commit()
# bot.py
from mysqlconnection import execute_search, execute_alter, getpool
#bot.listen()
async def on_ready():
await getpool()
...
So, anything wrong with my approach?
The following code fails and I'm not able to get the actual error, I just get numerous CancelledError messages
import aiobotocore, asyncio
async def replicate_to_region(chunks, region):
session = aiobotocore.get_session()
client = session.create_client('dynamodb', region_name=region)
start = 0
while True:
chunk = chunks[start]
item = {'my_table': chunk}
response = await client.batch_write_item(RequestItems=item)
async def main():
asyncio.gather(*(replicate_to_region(payloads, region) for region in regions))
asyncio.run(main())
I get the following errors;
client_session: <aiohttp.client.ClientSession object at 0x7f6fb65a34a8>
Unclosed client session
client_session: <aiohttp.client.ClientSession object at 0x7f6fb64c82b0>
_GatheringFuture exception was never retrieved
future: <_GatheringFuture finished exception=CancelledError()>
concurrent.futures._base.CancelledError
_GatheringFuture exception was never retrieved
future: <_GatheringFuture finished exception=CancelledError()>
I've tried quite a number of variations of the replicate_to_region function but they all fail with the same error above. It would be useful just to be able to see what the actual error is.
async def main():
asyncio.gather(...)
asyncio.gather() is an awaitable itself:
awaitable asyncio.gather(*aws, loop=None, return_exceptions=False)
It means you should use await when deal with it:
async def main():
await asyncio.gather(*(replicate_to_region(payloads, region) for region in regions))
off-topic:
I didn't work with aiobotocore and not sure if it's important, but it's better to do as documentation says. In particular you should probably use async with when creating a client as example shows.
I'm trying to write tests for a method that uses "async with" statements (in this case, aioredis's connection pool), i want to mock the connection to redis, but i'm having trouble figuring out how.
Here's what i have so far:
from asyncio import Future
from unittest.mock import MagicMock
import pytest
# The thing i'm trying to test
async def set_value(redis, value):
# Do things
async with redis.get() as conn:
await conn.set("key", value)
#My Mock classes
class MockRedis():
def get(self):
return MockAsyncPool()
class MockAsyncPool(MagicMock):
async def __aenter__(self):
conn = MagicMock()
f = Future()
f.set_result(True)
conn.set = MagicMock(return_value=f)
return conn
def __aexit__(self, exc_type, exc_val, exc_tb):
pass
# The actual test
#pytest.mark.asyncio
async def test_get_token():
redis = MockRedis()
token = await set_value(redis, 'something')
assert token is not None
I run it with:
py.test path/to/file.py
And i'm getting this error:
> await conn.set("key", value)
E TypeError: object NoneType can't be used in 'await' expression
__aexit__ needs to also be asyncronous (needs to return an awaitable):
async def __aexit__(self, exc_type, exc_val, exc_tb):
pass
Without being async it is returning None instead of a coroutine so it raises an error, as for the very misleading error message I have created this issue to point out that the error message could be greatly improved.