I wanted to organize a connection pool when initializing the class with the method below
import asyncio
import asyncpg
class DBCommands:
def __init__(self, uri: str) -> None:
loop = asyncio.get_event_loop()
self.pool: asyncpg.pool.Pool = loop.run_until_complete(asyncpg.create_pool(dsn=uri))
async def get_id_admins(self) -> list:
async with self.pool.acquire():
result = await self.pool.fetch("SELECT chat_id FROM users WHERE role_user = 'admin'")
admins_id = [row[0] for row in result]
return admins_id
Since the pool should be one, with the above implementation, this will not work. I decided to use singleton, but I don’t understand how to implement this. Below is the version that I came up with. Tell me how best to solve this problem. In addition, I do not understand how best and where to close connections. I'm new to using patterns and just starting to study OOP.
import asyncio
import asyncpg
class Singleton(type):
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
class DBManager(metaclass=Singleton):
#classmethod
def connect(cls, uri):
loop = asyncio.get_event_loop()
return loop.run_until_complete(asyncpg.create_pool(dsn=uri))
class DBCommands:
def __init__(self, uri) -> None:
self.uri = uri
self.pool = DBManager.connect(uri)
async def get_id_admins(self) -> list:
async with self.pool.acquire():
result = await self.pool.fetch("SELECT chat_id FROM users WHERE role_user = 'admin'")
admins_id = [row[0] for row in result]
return admins_id
I have an assumption that opening and closing a pool can be added to __aenter__ and __aexit__
You can use a class attribute and create the pool the first time it's needed in an async function:
class Database:
self.pool = None
...
async def get_id_admins(self)
if self.pool is None:
self.pool = await asyncpg.create_pool(dsn=...`).
I generally use a regular class and create a single instance attached to global object (like the aiohttp application for web applications) as in:
class Database:
def __init__(self, dsn):
self.dsn = dsn
self.pool = None
async def connect(self):
"""Initialize asyncpg Pool"""
self.pool = await asyncpg.create_pool(dsn=self.dsn, min_size=2, max_size=4)
logging.info("successfully initialized database pool")
async def get_id_admins(self):
...
And use it like:
async def startup(app):
await app.database.connect()
async def shutdown(app):
await app.database.pool.close()
def main():
app = web.Application()
app.database = Database(app.config.DSN)
app.on_startup.append(startup)
app.on_shutdown.append(shutdown)
Related
I have a bunch of online data that I want to download and process efficiently. Downloading already takes some time but cpu-bound processing takes much longer. I struggle to implement a combination of async and ProcessPoolExecutor.
import asyncio
import time
import aiohttp
from aiohttp import ClientSession
from concurrent.futures import ProcessPoolExecutor
class WebData:
def __init__(self, url):
self.url = url
self.binary = b''
async def download(self, client):
time.sleep(0.2)
try:
async with client.get(self.url, timeout=5) as resp:
self.binary = await resp.read()
print(f'Downloaded {self.url}')
except (aiohttp.ClientConnectionError,
asyncio.exceptions.TimeoutError):
pass
return
def process(self):
print(f'Start processing {self.url}')
time.sleep(1)
print(f'Finished processing {self.url}')
async def main():
list_urls = [f'https://www.google.com/search?q={i}'
for i in range(10)]
list_obj = [WebData(url) for url in list_urls]
with ProcessPoolExecutor() as executor:
async with ClientSession() as session:
tasks = [obj.download(session) for obj in list_obj]
await asyncio.gather(*tasks)
list_futures = [
executor.submit(obj.process)
for obj in list_obj]
return list_futures
res = asyncio.run(main())
This works as expected but it fails to accomplish what I am looking for. It first downloads all data and starts processing it only afterwards, which leaves my cores idle during download. Is there any way I can pipe the downloaded objects to the executor while other objects are still downloading?
I found this thread but it isn't what I need.
You should submit the self.process inside after the coroutine ends. For that, you can have a separate asynchronous method that will await the download method and submit the process to ProcessPoolExecutor.
class WebData:
def __init__(self, url):
"""The code has not been changed"""
async def download(self, client):
"""The code has not been changed"""
def process(self):
"""The code has not been changed"""
async def execute(self, session, pool):
await self.download(session)
pool.submit(self.process)
async def main():
list_urls = [f'https://www.google.com/search?q={i}' for i in range(10)]
list_obj = [WebData(url) for url in list_urls]
with ProcessPoolExecutor() as pool:
async with ClientSession() as session:
list_futures = await asyncio.gather(*[obj.execute(session, pool) for obj in list_obj])
return list_futures
This is my 1st fastapi exercise. I've my old model serving code implemented with Flask as following:
class HealthCheck(Resource):
def __init__(self, **kwargs):
super(HealthCheck, self).__init__()
self._model = kwargs['model']
self._logger = kwargs['logger']
def get(self):
if self._model:
return {"status" : "healthy"}, HTTPStatus.OK
return {"status": "unavailable"}, HTTPStatus.BAD_REQUEST
def put(self):
raise MethodNotAllowed('PUT request not supported')
# similarly other methods are disabled
# In a different module, say in App class
class App():
def __init__(self, name, logger, config):
self._logger = logger
self._model = load_model(config['model_path'])
self._flask_app = Flask(name)
api = Api(self._flask_app)
# logger and model is passed to HealthCheck resource
api.add_resource(HealthCheck, "/api/healthcheck",
resource_class_kwargs={'model': self._model, 'logger': self._logger})
How do I achieve same with fastapi APIRouter?
My example fastapi implementation is following:
class HealthResult(BaseModel):
healthy: bool
health_router = fastapi.APIRouter()
#health_router.get("/healthcheck", response_model=HealthResult, name="heathcheck")
async def heartbeat() -> HealthResult:
hb = HealthResult(healthy=True)
return hb
# in the App module
class App():
def __init__(self, name, logger, config):
self._logger = logger
self._model = load_model(config['model_path'])
self._api = fastapi.FastAPI(title=name)
self._api.include_router(health_router, prefix="/api")
# how do I pass model and logger to health_router to use that in heartbeat method?
I want to avoid using any global storage for model and logger and access in health_router from there.
Also, since my fastapi object is inside App class, how can I invoke multiple worker with uvicorn in this case?
Though I don't like it, I modified my App class as following to get around the problem - still looking for cleaner solution.
class App(metaclass=Singleton):
_MODEL: Union[None, Model] = None
_LOGGER: Union[None, CustomLogger] = None
#classmethod
def setLogger(cls, logger: CustomLogger) -> None:
cls._LOGGER = logger
#classmethod
def getLogger(cls) -> CustomLogger:
return cls._LOGGER
#classmethod
def setModel(cls, model: Model) -> None:
assert model
cls._MODEL = model
#classmethod
def getModel(cls) -> Model:
return cls._MODEL
def __init__(self, name: str, logger: CustomLogger, config: YAML) -> None:
App.setLogger(logger)
model: Model = load_model(config['model_path'])
App.setModel(model)
self._api = fastapi.FastAPI(title=name)
self._api.include_router(health_router, prefix="/api")
....
class HealthResult(BaseModel):
healthy: bool
health_router = fastapi.APIRouter()
#health_router.get("/healthcheck", response_model=HealthResult, name="heathcheck")
async def heartbeat() -> HealthResult:
model: Model = App.getModel()
hb: HealthResult = HealthResult(healthy=True) if model else HealthResult(healthy=False)
return hb
from fastapi import Request
app = FastAPI()
app.share_text = 'aaa'
# in Apirouter
#router.get("/share}", request: Request)
async def read( request: Request):
#retrieve from app context
share_text = request.app.share_text
return {}
You could retrive the app context from Request
Ref: https://fastapi.tiangolo.com/advanced/using-request-directly/#use-the-request-object-directly
Ref: https://www.starlette.io/requests/
Application The originating Starlette application can be accessed via
request.app.
Other state If you want to store additional information on the request
you can do so using request.state.
For example:
request.state.time_started = time.time()
I have a database connection class that creates a connection pool. Now as the application grows and I'm adding different types of database writers, I want to move database connections to a separate class and inherit from it. So far I have this:
class ServiceDB:
#classmethod
async def init(cls, settings):
self = ServiceDB()
self.pool = await asyncpg.create_pool(
database=settings["POSTGRES_DB"],
user=settings["POSTGRES_USER"],
password=settings["POSTGRES_PASSWORD"],
host=settings["DB_HOST"],
port=settings["DB_PORT"],
)
return self
class ChildWriter(ServiceDB):
async def write_db(self, query):
# Write to specific table
pass
if __name__ == "__main__":
settings = {'info': "some connection settings"}
query = "SELECT * FROM 'table'"
connection = await ChildWriter().init(settings)
await connection.write_db(msg, query)
When I run this I get AttributeError: 'ServiceDB' object has no attribute 'write_db'. How do I properly extend ServiceDB with the write_db method?
Classmethods receive the "current class" as the first argument. Instantiate this cls, not the fixed baseclass.
class ServiceDB:
#classmethod
async def init(cls, settings):
self = cls() # cls is the *current* class, not just ServiceDB
self.pool = await asyncpg.create_pool(
database=settings["POSTGRES_DB"],
user=settings["POSTGRES_USER"],
password=settings["POSTGRES_PASSWORD"],
host=settings["DB_HOST"],
port=settings["DB_PORT"],
)
return self
Note that ideally, all attributes are set via __init__ instead of a separate classmethod constructor. The separate constructor should just pass on any attributes constructed externally.
class ServiceDB:
def __init__(self, pool):
self.pool = pool
#classmethod
async def init(cls, settings, **kwargs):
pool = await asyncpg.create_pool(
database=settings["POSTGRES_DB"],
user=settings["POSTGRES_USER"],
password=settings["POSTGRES_PASSWORD"],
host=settings["DB_HOST"],
port=settings["DB_PORT"],
)
return cls(pool=pool, **kwargs)
class ChildWriter(ServiceDB):
async def write_db(self, query): ...
if __name__ == "__main__":
settings = {'info': "some connection settings"}
query = "SELECT * FROM 'table'"
# call classmethod on class V
connection = await ChildWriter.init(settings)
await connection.write_db(msg, query)
I want Faust agent to write to PostgreSQL table. I'd like to use asyncpg connection pool but cannot find a clean way to inject it into the app initialization code.
Simply add the below function to your faust App
class KafkaWorker(faust.App):
def __init__(self, *args: List, **kwargs: Dict) -> None:
self.broker : str = kwargs.pop('broker')
self._db_pool = None
super().__init__(*args, broker=KafkaWorker._broker_faust_string(self.broker), **kwargs)
async def db_pool(self) -> Pool:
''' '''
if not self._db_pool:
logging.warning('kafka.db_pool initialization...')
self._db_pool = await db.db_pool()
logging.warning('kafka.db_pool initialization...done ✓')
return self._db_pool
where db.db_pool is
from os import environ
import asyncpg
from asyncpg.pool import Pool
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
session = scoped_session(sessionmaker())
Base = declarative_base()
async def db_pool() -> Pool:
return await asyncpg.create_pool(
dsn=environ.get('DB_CNX_STRING', 'postgresql://postgres:postgres#postgres:5432/actions')
)
and then you access it as
#kafka.agent(actions_topic)
async def store_actions(actions: StreamT):
async for action in actions:
db_pool = await current_agent().app.db_pool()
async with db_pool.acquire() as conn:
try:
yield await save_action(conn, action.to_representation())
except StoreException:
logger.exception(f'Error while inserting action in DB, continuing....')
finally:
yield action.id
Here's a sample code.
class Foo:
def __init__(self):
self._run_coro()
def _run_coro(self):
async def init():
bar = #some I/O op
self.bar = bar
loop = asyncio.get_event_loop()
loop.run_until_complete(init())
async def spam(self):
return await #I/O op
async def main():
foo = Foo()
await foo.spam()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
When I run this code, I get following exception:
RuntimeError: This event loop is already running
If I initialize Foo outside main, the code runs without any exception. I want to initialize Foo such that during initialization it runs a coroutine which creates a class attribute bar.
I am unable to figure how to do it correctly. How can I run a coroutine from __init__.
Any help would be highly appreciated.
class Foo:
def __init__(self):
self.session = requests.Session()
self.async_session = None
#I guess this can be done to initialize it.
s = self.init_async_session()
try:
s.send(None)
except StopIteration:
pass
finally:
s.close()
async def init_async_session(self):
#ClientSession should be created inside a coroutine.
self.async_session = aiohttp.ClientSession()
What would be the right way to initialize self.async_session
If some method uses something asynchronous it should be explicitly defined as asynchronous either. This is a core idea behind asyncio: make you write code a way you always know if some arbitrary method may do something asynchronous.
In your snippet you want to do async thing (bar I/O) inside sync method __init__ and asyncio prohibits it. You should make _run_coro async and initialize Foo asynchronously, for example, using __await__ method:
import asyncio
class Foo:
def __await__(self):
return self._run_coro().__await__()
async def _run_coro(self): # real async initializer
async def init():
await asyncio.sleep(1) # bar I/O
self.bar = 123
await init()
return self
async def spam(self):
return await asyncio.sleep(1) # I/O op
async def main():
foo = await Foo()
await foo.spam()
asyncio.run(main()) # instead of two lines in Python 3.7+
You may be interested in reading this answer to understand better how asyncio works and how to handle it.
Upd:
s = self.init_async_session()
try:
s.send(None)
Don't do such things: generator's method are only details of implementation in regard of coroutines. You can predict how coroutine will react on calling .send() method and you can rely on this behavior.
If you want to execute coroutine use await, if you want to start it "in background" use task or other functions from asyncio doc.
What would be the right way to initialize self.async_session
When it comes to aiohttp.ClientSession it should not only be created, but properly closed also. Best way to do it is to use async context manager as shown in aiohttp doc.
If you want to hide this operation inside Foo you can make it async manager either. Complete example:
import aiohttp
class Foo:
async def __aenter__(self):
self._session = aiohttp.ClientSession()
await self._session.__aenter__()
return self
async def __aexit__(self, *args):
await self._session.__aexit__(*args)
async def spam(self):
url = 'http://httpbin.org/delay/1'
resp = await self._session.get(url)
text = await resp.text()
print(text)
async def main():
async with Foo() as foo:
await foo.spam()
asyncio.run(main())
Upd2:
You can combine ways to init/close object from above to achive result you like. As long as you keep in mind both operations are asynchronous and thus should be awaited, everything should be fine.
One more possible way:
import asyncio
import aiohttp
class Foo:
def __await__(self):
return self._init().__await__()
async def _init(self):
self._session = aiohttp.ClientSession()
await self._session.__aenter__()
return self
async def close(self):
await self._session.__aexit__(None, None, None)
async def spam(self):
url = 'http://httpbin.org/delay/1'
resp = await self._session.get(url)
text = await resp.text()
print(text)
async def main():
foo = await Foo()
try:
await foo.spam()
finally:
await foo.close()
asyncio.run(main())
Here's my solution.
class Session:
def __init__(self, headers):
self._headers = headers
self._session = requests.Session()
self._async_session = None
async def _init(self):
self._session = aiohttp.ClientSession(headers=headers)
async def async_request(self, url):
while True:
try:
async with self._async_session.get(url) as resp:
resp.raise_for_status()
return await resp.text()
except aiohttp.client_exceptions.ClientError:
#retry or raise
except AttributeError:
if isinstance(self._async_session, aiohttp.ClientSession):
raise
await self._init()
def request(self, url):
return self._session.get(url).text
async def close(self):
if isinstance(self._async_session, aiohttp.ClientSession):
await self._session.close()
async def main():
session = Session({})
print(await session.async_request('https://httpstat.us/200')
await session.close()
asyncio.run(main())
I can initialize the Session class and make synchronous as well as asynchronous requests. I do not have to explicitly call await session._init() to initialize self._async_session as when session._async_request is called and self._async_session is None, then await session._init() will be called and the request will be retried.