I have a database connection class that creates a connection pool. Now as the application grows and I'm adding different types of database writers, I want to move database connections to a separate class and inherit from it. So far I have this:
class ServiceDB:
#classmethod
async def init(cls, settings):
self = ServiceDB()
self.pool = await asyncpg.create_pool(
database=settings["POSTGRES_DB"],
user=settings["POSTGRES_USER"],
password=settings["POSTGRES_PASSWORD"],
host=settings["DB_HOST"],
port=settings["DB_PORT"],
)
return self
class ChildWriter(ServiceDB):
async def write_db(self, query):
# Write to specific table
pass
if __name__ == "__main__":
settings = {'info': "some connection settings"}
query = "SELECT * FROM 'table'"
connection = await ChildWriter().init(settings)
await connection.write_db(msg, query)
When I run this I get AttributeError: 'ServiceDB' object has no attribute 'write_db'. How do I properly extend ServiceDB with the write_db method?
Classmethods receive the "current class" as the first argument. Instantiate this cls, not the fixed baseclass.
class ServiceDB:
#classmethod
async def init(cls, settings):
self = cls() # cls is the *current* class, not just ServiceDB
self.pool = await asyncpg.create_pool(
database=settings["POSTGRES_DB"],
user=settings["POSTGRES_USER"],
password=settings["POSTGRES_PASSWORD"],
host=settings["DB_HOST"],
port=settings["DB_PORT"],
)
return self
Note that ideally, all attributes are set via __init__ instead of a separate classmethod constructor. The separate constructor should just pass on any attributes constructed externally.
class ServiceDB:
def __init__(self, pool):
self.pool = pool
#classmethod
async def init(cls, settings, **kwargs):
pool = await asyncpg.create_pool(
database=settings["POSTGRES_DB"],
user=settings["POSTGRES_USER"],
password=settings["POSTGRES_PASSWORD"],
host=settings["DB_HOST"],
port=settings["DB_PORT"],
)
return cls(pool=pool, **kwargs)
class ChildWriter(ServiceDB):
async def write_db(self, query): ...
if __name__ == "__main__":
settings = {'info': "some connection settings"}
query = "SELECT * FROM 'table'"
# call classmethod on class V
connection = await ChildWriter.init(settings)
await connection.write_db(msg, query)
Related
I am fairly new to FastAPI(migrating from Django) and I am trying to create a generic CRUD operations class that I can inherit and use across my CBV endpoints.
Something like this :
class AbstractCrud
model: Base = NotImplemented
session: Session = NotImplemented
def get_items(self, limit, **filters):
""" Read operation """
def get_item(self, pk: int):
def create_item(self, obj: BaseModel):
""" Create operation """
def update_item(self, pk: int, **params):
""" Update operation"""
def delete_item(self, pk: int):
""" Delete operation """
router = InferringRouter()
#cbv(router)
class UserAPI(AbstractCrud):
router.tags = ["User"]
router.prefix = "/users"
model = User
session: Session = Depends(get_db)
# my endpoints
#e.g. #router.get(...)
#cbv(router)
class PostAPI(AbstractCrud):
router.tags = ["Post"]
router.prefix = "/posts"
model = Post
session: Session = Depends(get_db)
# my endpoints
#e.g. #router.get(...)
I get the following error if I try to do the above:
fastapi.exceptions.FastAPIError: Invalid args for response field! Hint: check that <class 'sqlalchemy.orm.decl_api.Base'> is a valid pydantic field type
For now, I am able to achieve this as follows:
class AbstractCrud
model: Base = NotImplemented
session: Session = NotImplemented
def get_items(self, limit, **filters):
""" Read operation """
def get_item(self, pk: int):
def create_item(self, obj: BaseModel):
""" Create operation """
def update_item(self, pk: int, **params):
""" Update operation"""
def delete_item(self, pk: int):
""" Delete operation """
class UserCrud(AbstractCrud):
def __init__(self, session: Session):
self.session = session
self.model = User
class PostCrud(AbstractCrud):
def __init__(self, session: Session):
self.session = session
self.model = Post
router = InferringRouter()
#cbv(router)
class UserAPI:
router.tags = ["User"]
router.prefix = "/users"
def __init__(self, session=Depends(get_db)):
self.session = session
self.crud = UserCrud(self.session)
# my endpoints
#e.g. #router.get(...)
#cbv(router)
class PostAPI:
router.tags = ["Post"]
router.prefix = "/posts"
def __init__(self, session=Depends(get_db)):
self.session = session
self.crud = PostCrud(self.session)
# my endpoints
#e.g. #router.get(...)
Although this is working fine for me now, I can't help but think if there is a better(or correct) way to do this.
Also, Is my use of a single router variable across multiple classes correct?
I'm trying to use the aiobotocore library with context managers, but I'm having a hard time trying to configure my credentials.
I need to create a class that configure my AWS client so I can use the put, read and delete functions in this library.
The following code is being used to this:
from contextlib import AsyncExitStack
from aiobotocore.session import AioSession
from credentials import aws_access_key_id, aws_secret_access_key
class AWSConnectionManager:
def __init__(self, aws_acces_key_id, aws_secret_access_key):
self.aws_acces_key_id = aws_acces_key_id
self.aws_secret_access_key = aws_secret_access_key
self._exit_stack = AsyncExitStack()
self._client = None
print('__init__')
async def __aenter__(self):
session = AioSession
self._client = await self._exit_stack.enter_async_context(session.create_client('s3'))
print('__aenter__')
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self._exit_stack.__aexit__(exc_type, exc_val, exc_tb)
print('__aexit__')
res = AWSConnectionManager(aws_acces_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
But, it doesn't pass through aenter and aexit method.
With the code above I have the following output:
__init__
<__main__.AWSConnectionManager object at 0x7f03921ac640>
Does anyone know what can be wrong with my code?
First: you need to fix 'session = AioSession' => 'session = AioSession()' + add return + pass credentials
async def __aenter__(self):
session = AioSession()
self._client = await self._exit_stack.enter_async_context(
session.create_client(
's3',
aws_secret_access_key=self.aws_secret_access_key,
aws_access_key_id=self.aws_access_key_id,
)
)
return self
Second: you need to write/add proxy calls for put_object/get_object or make _client public by rename _client => client
async def save_file(self, content, s3_filename: str):
await self._client.put_object(Bucket=self.bucket, Body=content, Key=f'{s3_filename}')
async def load_file(self, name):
obj = await self._client.get_object(Bucket=self.bucket, Key=f'{name}')
return obj['Body'].read()
now you can use like
async with SkyFileStorageProxy() as storage:
await storage.load_file(name='test.txt')
class XmlRpcClient(ServerProxy):
"""
XmlRpcClient is a wrapper around the standard library's ServerProxy.
It proxies any function calls and fires the request event when they finish,
so that the calls get recorded in Locust.
"""
def __init__(self, host, request_event):
super().__init__(host)
self._request_event = request_event
def __getattr__(self, name):
func = ServerProxy.__getattr__(self, name)
def wrapper(*args, **kwargs):
request_meta = {
"request_type": "xmlrpc",
"name": name,
"start_time": time.time(),
"response_length": 0, # calculating this for an xmlrpc.client response would be too hard
"response": None,
"context": {}, # see HttpUser if you actually want to implement contexts
"exception": None,
}
start_perf_counter = time.perf_counter()
try:
request_meta["response"] = func(*args, **kwargs)
except Fault as e:
request_meta["exception"] = e
request_meta["response_time"] = (time.perf_counter() - start_perf_counter) * 1000
self._request_event.fire(**request_meta) # This is what makes the request actually get logged in Locust
return request_meta["response"]
return wrapper
class XmlRpcUser(User):
"""
A minimal Locust user class that provides an XmlRpcClient to its subclasses
"""
abstract = True # dont instantiate this as an actual user when running Locust
def __init__(self, environment):
super().__init__(environment)
self.client = XmlRpcClient(self.host, request_event=environment.events.request)
The real user class that will be instantiated and run by Locust
This is the only thing that is actually specific to the service that we are testing.
class MyUser(XmlRpcUser):
host = "http://127.0.0.1:8877/"
#task
def get_time(self):
self.client.get_time()
#task
def get_random_number(self):
self.client.get_random_number(0, 100)
This is my 1st fastapi exercise. I've my old model serving code implemented with Flask as following:
class HealthCheck(Resource):
def __init__(self, **kwargs):
super(HealthCheck, self).__init__()
self._model = kwargs['model']
self._logger = kwargs['logger']
def get(self):
if self._model:
return {"status" : "healthy"}, HTTPStatus.OK
return {"status": "unavailable"}, HTTPStatus.BAD_REQUEST
def put(self):
raise MethodNotAllowed('PUT request not supported')
# similarly other methods are disabled
# In a different module, say in App class
class App():
def __init__(self, name, logger, config):
self._logger = logger
self._model = load_model(config['model_path'])
self._flask_app = Flask(name)
api = Api(self._flask_app)
# logger and model is passed to HealthCheck resource
api.add_resource(HealthCheck, "/api/healthcheck",
resource_class_kwargs={'model': self._model, 'logger': self._logger})
How do I achieve same with fastapi APIRouter?
My example fastapi implementation is following:
class HealthResult(BaseModel):
healthy: bool
health_router = fastapi.APIRouter()
#health_router.get("/healthcheck", response_model=HealthResult, name="heathcheck")
async def heartbeat() -> HealthResult:
hb = HealthResult(healthy=True)
return hb
# in the App module
class App():
def __init__(self, name, logger, config):
self._logger = logger
self._model = load_model(config['model_path'])
self._api = fastapi.FastAPI(title=name)
self._api.include_router(health_router, prefix="/api")
# how do I pass model and logger to health_router to use that in heartbeat method?
I want to avoid using any global storage for model and logger and access in health_router from there.
Also, since my fastapi object is inside App class, how can I invoke multiple worker with uvicorn in this case?
Though I don't like it, I modified my App class as following to get around the problem - still looking for cleaner solution.
class App(metaclass=Singleton):
_MODEL: Union[None, Model] = None
_LOGGER: Union[None, CustomLogger] = None
#classmethod
def setLogger(cls, logger: CustomLogger) -> None:
cls._LOGGER = logger
#classmethod
def getLogger(cls) -> CustomLogger:
return cls._LOGGER
#classmethod
def setModel(cls, model: Model) -> None:
assert model
cls._MODEL = model
#classmethod
def getModel(cls) -> Model:
return cls._MODEL
def __init__(self, name: str, logger: CustomLogger, config: YAML) -> None:
App.setLogger(logger)
model: Model = load_model(config['model_path'])
App.setModel(model)
self._api = fastapi.FastAPI(title=name)
self._api.include_router(health_router, prefix="/api")
....
class HealthResult(BaseModel):
healthy: bool
health_router = fastapi.APIRouter()
#health_router.get("/healthcheck", response_model=HealthResult, name="heathcheck")
async def heartbeat() -> HealthResult:
model: Model = App.getModel()
hb: HealthResult = HealthResult(healthy=True) if model else HealthResult(healthy=False)
return hb
from fastapi import Request
app = FastAPI()
app.share_text = 'aaa'
# in Apirouter
#router.get("/share}", request: Request)
async def read( request: Request):
#retrieve from app context
share_text = request.app.share_text
return {}
You could retrive the app context from Request
Ref: https://fastapi.tiangolo.com/advanced/using-request-directly/#use-the-request-object-directly
Ref: https://www.starlette.io/requests/
Application The originating Starlette application can be accessed via
request.app.
Other state If you want to store additional information on the request
you can do so using request.state.
For example:
request.state.time_started = time.time()
I wanted to organize a connection pool when initializing the class with the method below
import asyncio
import asyncpg
class DBCommands:
def __init__(self, uri: str) -> None:
loop = asyncio.get_event_loop()
self.pool: asyncpg.pool.Pool = loop.run_until_complete(asyncpg.create_pool(dsn=uri))
async def get_id_admins(self) -> list:
async with self.pool.acquire():
result = await self.pool.fetch("SELECT chat_id FROM users WHERE role_user = 'admin'")
admins_id = [row[0] for row in result]
return admins_id
Since the pool should be one, with the above implementation, this will not work. I decided to use singleton, but I don’t understand how to implement this. Below is the version that I came up with. Tell me how best to solve this problem. In addition, I do not understand how best and where to close connections. I'm new to using patterns and just starting to study OOP.
import asyncio
import asyncpg
class Singleton(type):
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
class DBManager(metaclass=Singleton):
#classmethod
def connect(cls, uri):
loop = asyncio.get_event_loop()
return loop.run_until_complete(asyncpg.create_pool(dsn=uri))
class DBCommands:
def __init__(self, uri) -> None:
self.uri = uri
self.pool = DBManager.connect(uri)
async def get_id_admins(self) -> list:
async with self.pool.acquire():
result = await self.pool.fetch("SELECT chat_id FROM users WHERE role_user = 'admin'")
admins_id = [row[0] for row in result]
return admins_id
I have an assumption that opening and closing a pool can be added to __aenter__ and __aexit__
You can use a class attribute and create the pool the first time it's needed in an async function:
class Database:
self.pool = None
...
async def get_id_admins(self)
if self.pool is None:
self.pool = await asyncpg.create_pool(dsn=...`).
I generally use a regular class and create a single instance attached to global object (like the aiohttp application for web applications) as in:
class Database:
def __init__(self, dsn):
self.dsn = dsn
self.pool = None
async def connect(self):
"""Initialize asyncpg Pool"""
self.pool = await asyncpg.create_pool(dsn=self.dsn, min_size=2, max_size=4)
logging.info("successfully initialized database pool")
async def get_id_admins(self):
...
And use it like:
async def startup(app):
await app.database.connect()
async def shutdown(app):
await app.database.pool.close()
def main():
app = web.Application()
app.database = Database(app.config.DSN)
app.on_startup.append(startup)
app.on_shutdown.append(shutdown)