Hi I have a following class/ function
class A:
def __init__(aws_profile_name, aws_region, ec2_id):
self.session = boto3.session.Session(profile_name=aws_profile_name, aws_region)
self.ec2 = EC2(self.session, aws_region, id=ec2_id)
class EC2:
def __init__(self, session, aws_region, id):
self.session = session
self.region = aws_region
self.id = id
self.ec2_resouce = self.session.resource("ec2", region_name=self.region)
self.ec2_client = self.session.client("ec2", region_name=self.region)
self.instance = self.filter_ec2_by_id()
def filter_ec2_by_id(self):
return self.filter(
InstanceIds=[
self.get_instance_id(),
]
)
def filter(self, kwargs):
instances = self.ec2_resouce.instances.filter(**kwargs)
instance_list = [instance for instance in instances]
return instance_list
And here is my test test.py
from unittest import mock
import sys
from moto import mock_ec2
import boto3
#mock_ec2
def test_mock_session():
mock_session_obj = mock.Mock()
ec2 = boto3.resource("ec2", region_name="us-east-1")
reservation = client.run_instances(ImageId="ami-1234", MinCount=2, MaxCount=2)
instance1 = ec2.Instance(reservation["Instances"][0]["InstanceId"])
ec2 = EC2(mock_session_obj, region="us-east-1", id=instance1)
test_mock_session()
With above test code, the test failed with following error
instance_list = [instance for instance in instances]
TypeError: 'Mock' object is not iterable
I think that is because ec2 filter return a collection, but I am not sure how should i mock the result. Any recommendation is welcomed.
TIA
Moto will already mock boto3 for you, and intercept any calls to AWS, so there is no need to use mock_session_obj.
Just use EC2(boto3.session.Session(), region="us-east-1", id=instance1). When calling the filter-function, Moto will intercept it and return any created instances in the correct format.
Related
I am trying to mock secret manager. Here is the code which is written for secret manager using boto3 which I am trying to mock and test.
utils.py
import boto3
secret_id = os.environ.get("SECRETS")
client = boto3.client('secretsmanager')
response = client.get_secret_value(SecretId=secret_id)
secrets = json.loads(response['SecretString'])
S3_BUCKET_NAME = secrets["S3_BUCKET_NAME"]
SQS_QUEUE_NAME = secrets["SQS_Queue_Name"]
these variables are then used in different methods.
conftest.py
#pytest.fixture(scope='session', autouse=True)
def secret_manager_resource(aws_credentials):
"""Secret Manager mock client"""
with mock_secretsmanager():
conn = boto3.client("secretsmanager", region_name="us-east-1")
logger.info(f"Secret manager connection {conn}")
yield conn
test_file.py
#contextmanager
def secret_manager_setup(secret_manager_resource):
secret_manager_resource.create_secret(Name="test", SecretString="""{"S3_BUCKET_NAME": "test","SQS_Queue_Name": "test_queue"}""")
yield
class TestSecretManager:
def test_secret_manager(self, secret_manager_resource):
with secret_manager_setup(secret_manager_resource):
try:
result = secret_manager_resource.get_secret_value(SecretId="test")
json_result = json.loads(result['SecretString'])
assert json_result["S3_BUCKET_NAME"] == "test"
assert json_result["SQS_Queue_Name"] == "test_queue"
except Exception as err:
print("Error ---", err)
class TestClass:
def test_some_class(test_var):
from functions.something.some import something
something = someClass({}, param)
When I run pytest it directly goes inside TestClass and calls secret Manager and throws error as it is trying to connect to actual secret manager. Could someone suggest me what can be done to over come this issue?
TestClass is not mocked - so I wouldn't expect that to work. You could use Moto as a class-decorator to ensure everything inside someClass is mocked.
Note that the class-decorator creates a mock around test-methods only, so the code-under-test would have to be inside a test-method for this to work.
#mock_secretsmanager()
class TestClass:
def test_something():
from functions.something.some import something
something = someClass({}, param)
See http://docs.getmoto.org/en/latest/docs/getting_started.html#class-decorator for the documentation and more examples around this.
This is my 1st fastapi exercise. I've my old model serving code implemented with Flask as following:
class HealthCheck(Resource):
def __init__(self, **kwargs):
super(HealthCheck, self).__init__()
self._model = kwargs['model']
self._logger = kwargs['logger']
def get(self):
if self._model:
return {"status" : "healthy"}, HTTPStatus.OK
return {"status": "unavailable"}, HTTPStatus.BAD_REQUEST
def put(self):
raise MethodNotAllowed('PUT request not supported')
# similarly other methods are disabled
# In a different module, say in App class
class App():
def __init__(self, name, logger, config):
self._logger = logger
self._model = load_model(config['model_path'])
self._flask_app = Flask(name)
api = Api(self._flask_app)
# logger and model is passed to HealthCheck resource
api.add_resource(HealthCheck, "/api/healthcheck",
resource_class_kwargs={'model': self._model, 'logger': self._logger})
How do I achieve same with fastapi APIRouter?
My example fastapi implementation is following:
class HealthResult(BaseModel):
healthy: bool
health_router = fastapi.APIRouter()
#health_router.get("/healthcheck", response_model=HealthResult, name="heathcheck")
async def heartbeat() -> HealthResult:
hb = HealthResult(healthy=True)
return hb
# in the App module
class App():
def __init__(self, name, logger, config):
self._logger = logger
self._model = load_model(config['model_path'])
self._api = fastapi.FastAPI(title=name)
self._api.include_router(health_router, prefix="/api")
# how do I pass model and logger to health_router to use that in heartbeat method?
I want to avoid using any global storage for model and logger and access in health_router from there.
Also, since my fastapi object is inside App class, how can I invoke multiple worker with uvicorn in this case?
Though I don't like it, I modified my App class as following to get around the problem - still looking for cleaner solution.
class App(metaclass=Singleton):
_MODEL: Union[None, Model] = None
_LOGGER: Union[None, CustomLogger] = None
#classmethod
def setLogger(cls, logger: CustomLogger) -> None:
cls._LOGGER = logger
#classmethod
def getLogger(cls) -> CustomLogger:
return cls._LOGGER
#classmethod
def setModel(cls, model: Model) -> None:
assert model
cls._MODEL = model
#classmethod
def getModel(cls) -> Model:
return cls._MODEL
def __init__(self, name: str, logger: CustomLogger, config: YAML) -> None:
App.setLogger(logger)
model: Model = load_model(config['model_path'])
App.setModel(model)
self._api = fastapi.FastAPI(title=name)
self._api.include_router(health_router, prefix="/api")
....
class HealthResult(BaseModel):
healthy: bool
health_router = fastapi.APIRouter()
#health_router.get("/healthcheck", response_model=HealthResult, name="heathcheck")
async def heartbeat() -> HealthResult:
model: Model = App.getModel()
hb: HealthResult = HealthResult(healthy=True) if model else HealthResult(healthy=False)
return hb
from fastapi import Request
app = FastAPI()
app.share_text = 'aaa'
# in Apirouter
#router.get("/share}", request: Request)
async def read( request: Request):
#retrieve from app context
share_text = request.app.share_text
return {}
You could retrive the app context from Request
Ref: https://fastapi.tiangolo.com/advanced/using-request-directly/#use-the-request-object-directly
Ref: https://www.starlette.io/requests/
Application The originating Starlette application can be accessed via
request.app.
Other state If you want to store additional information on the request
you can do so using request.state.
For example:
request.state.time_started = time.time()
First of all, I have tried looking for answers in this website. But no luck...
What I wanna achieve is that starting an independent thread in the request handling thread to do some asynchronous task. The tricky point is that there are some database operations needed in this independent thread.
Here is an example. Five files included.
project
|__manager.py
|__config.py
|__deployer
|__`__init__.py`
|__models.py
|__views.py
|__operators.py
Detail code below...
# deployer/__init__.py
from flask import Flask
from deployer.models import db
def create_app():
app = Flask(__name__)
app.config.from_object(object_name)
db.init_app(app)
# Add route for index
#app.route('/')
def index():
return {'code': 200, 'message': 'OK'}
return app
# manager.py
from os import environ
from flask_script import Manager, Server
from deployer import create_app
from flask_restful import Api
from deployer.views import HostView
env = environ.get('APM_ENV', 'dev')
app = create_app('config.%sConfig' % env.capitalize())
api = Api(app)
api.add_resource(HostView, '/api/v1/hosts')
manager = Manager(app)
manager.add_command("server", Server(host='0.0.0.0', port=9527))
if __name__ == '__main__':
manager.run(default_command='server')
# deployer/views.py
from flask_restful import Resource, reqparse
from flask import jsonify
from deployer.models import db, Host
from deployer.operators import HostInitiator
parser = reqparse.RequestParser()
parser.add_argument('host', type=int, help='Specify an unique host.')
class HostView(Resource):
def get(self):
h = db.session.query(Host).filter(Host.id == 1).one()
return jsonify(
host_id=h.id,
host_code=h.code,
host_ip=h.ip_addr_v4
)
def post(self):
h = Host(
code='Harbor',
ip_addr_v4='10.10.10.199',
state='created'
)
db.session.add(h)
db.session.commit()
initiator = HostInitiator(host=h)
initiator.start()
return {
'code': 'Harbor',
'ip_addr_v4': '10.10.10.199',
'state': 'created'
}
# deployer/models.py
from sqlalchemy import Column, Integer, String
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()
class Host(db.Model):
__tablename__ = 'br_host'
id = Column(Integer, primary_key=True, autoincrement=True)
code = Column(String(128), index=True, nullable=False)
ip_addr_v4 = Column(String(15), nullable=False)
state = Column(String(16), nullable=False)
# deployer/operators.py
from threading import Thread
from deployer.models import db
class HostInitiator(Thread):
def __init__(self, host):
super().__init__()
self.host = host
def run(self):
# Update Host.state [created-->initating]
db.session.query(Host).filter(Host.id == self.host.id).update({'state': 'initating'})
db.session.commit()
# do some initiating things...
# Update Host.state [initating-->ready]
db.session.query(Host).filter(Host.id == self.host.id).update({'state': 'ready'})
db.session.commit()
Always got outside application context error with code above. The error message indicates that no database operation is permitted in the HostInitiator thread.
It suggests me to push a context or move my code into a view function. I'm suffering this quite a while, please help out if you guys have any suggestions. Thanks in advance.
The code works for me
def test_multi_threading_query():
# module which i create Flask app instance
from app.main import app
# module which i create sqlalchemhy instance
from app.model.db import db, Post
with app.app_context():
posts = Post.query.all()
p = posts[0]
p.foo = 1
db.session.add(p)
db.session.commit()
print(p)
#api.route('/test')
def test_view():
from threading import Thread
t = Thread(target=test_multi_threading_query)
t.start()
return ''
# main.py
app = Flask(__main__)
#db.py
db = SQLAlchemy()
class Post(db.Model):
id = db.Column(db.Integer, primary_key=True)
foo = db.Column(db.Integer)
https://flask.palletsprojects.com/en/1.1.x/appcontext/
In folder [Root]/src/app, I have a file services_factory.py, for example:
class Describing:
def __init__(self):
pass
def get_description(self):
pass
class APIService(Describing):
def __init__(self):
pass
def get_description(self):
return 'Here provide services for APIs'
class DatabaseService(Describing):
def __init__(self):
pass
def get_description(self):
return 'Here provide services for Database'
class Injector:
def __init__(self):
pass
def get_service(self, type='API'):
services = {
"API": APIService,
"DB": DatabaseService
}
return services[type]()
At the end of file services_factory.py, I add an unittest, ex:
def test_services_injector():
injector = Injector()
api_service = injector.get_service('API')
db_service = injector.get_service('DB')
assert api_service.get_description() == 'Here provide services for APIs'
assert db_service.get_description() == 'Here provide services for Database'
Then, cmd: $ pytest src/app/services_injector.py, it worked nicely.
But when I create a file test_services_factory.py in [Root]/tests/app, for example:
import unittest
from unittest.mock import patch
def test_services_injector():
assert 'a' == 'a'
I can't import the classes in my services_factory.py.
So, how can I quickly fix this problem?
A Cloudwatch log is an object with Log Group > Log Stream > Log Events on AWS. I am trying to write tests for this, but the moto mocking raises a client error when applied to boto3.client('logs'). I am looking at other ways to mock the behavior of the log. How would you write a test for this function?
For example:
client = boto3.client('logs')
def get_recent_log_stream_name(logGroupName):
response = client.describe_log_streams(
logGroupName=logGroupName,
orderBy='LastEventTime',
descending=True,
limit=1)
logStreamName = response['logStreams'][0]['logStreamName']
return logStreamName
I would write the test using moto like this:
import boto3
from moto import mock_logs
def get_recent_log_stream_name(logs, logGroupName):
"""Function under test"""
response = logs.describe_log_streams(
logGroupName=logGroupName,
orderBy='LastEventTime',
descending=True,
limit=1)
log_stream_name = response['logStreams'][0]['logStreamName']
return log_stream_name
#mock_logs
def test_get_recent_log_stream_name():
"""Test function"""
log_group_name = 'test-group'
log_stream_name = 'test-stream'
logs = boto3.client('logs')
logs.create_log_group(logGroupName=log_group_name)
logs.create_log_stream(
logGroupName=log_group_name,
logStreamName=log_stream_name,
)
assert get_recent_log_stream_name(logs, log_group_name) == log_stream_name
test_get_recent_log_stream_name()