Consider the following:
class MockResponse:
status_code = 200
#staticmethod
def json():
return {'key': 'value'}
# where api_session is a fixture
def test_api_session_get(monkeypatch, api_session) -> None:
def mock_get(*args, **kwargs):
return MockResponse()
monkeypatch.setattr(requests.Session, 'get', mock_get)
response = api_session.get('endpoint/') # My wrapper around requests.Session
assert response.status_code == 200
assert response.json() == {'key': 'value'}
monkeypatch.assert_called_with(
'endpoint/',
headers={
'user-agent': 'blah',
},
)
How can I assert that the get I am patching gets called with '/endpoint' and headers? When I run the test now I get the following failure message:
FAILED test/utility/test_api_session.py::test_api_session_get - AttributeError: 'MonkeyPatch' object has no attribute 'assert_called_with'
What am I doing wrong here? Thanks to all those of who reply in advance.
Going to add another response that uses monkeypatch rather than "you can't use monkeypatch"
Since python has closures, here is my poor man's way of doing such things with monkeypatch:
patch_called = False
def _fake_delete(keyname):
nonlocal patch_called
patch_called = True
assert ...
monkeypatch.setattr("mymodule._delete", _fake_delete)
res = client.delete(f"/.../{delmeid}"). # this is a flask client
assert res.status_code == 200
assert patch_called
In your case, since we are doing similar things with patching an HTTP servers method handler, you could do something like (not saying this is pretty):
param_called = None
def _fake_delete(param):
nonlocal param_called
patch_called = param
assert ...
monkeypatch.setattr("mymodule._delete", _fake_delete)
res = client.delete(f"/.../{delmeid}")
assert res.status_code == 200
assert param_called == "whatever this should be"
You need a Mock object to call assert_called_with - monkeypatch does not provide that out of the box. You can use unittest.mock.patch with side_effect instead to achieve this:
from unittest import mock
import requests
...
#mock.patch('requests.Session.get')
def test_api_session_get(mocked, api_session) -> None:
def mock_get(*args, **kwargs):
return MockResponse()
mocked.side_effect = mock_get
response = api_session.get('endpoint/')
...
mocked.assert_called_with(
'endpoint/',
headers={
'user-agent': 'blah',
},
)
Using side_effect is needed to still get a mock object (mocked in this case, of type MagickMock), instead of just setting your own object in patch, otherwise you won't be able to use the assert_called_... methods.
Related
I need help in implementing the logic to count number of successful post calls which are asynchronous in nature (status_code=200) as well as failed_calls (status_code != 200)
I am new to coroutines. Would appreciate if someone can suggest a better way of making a post asynchronous call which can be retried, polled for status, and that can emit metrics for successful post requests as well.
Following is my code:
asyncio.get_event_loop().run_in_executor(
None,
self.publish_actual,
event_name,
custom_payload,
event_message_params,
)
which calls publish_actual:
def publish_actual(
self,
event_name: str,
custom_payload={},
event_message_params=[],
):
"""Submits a post request using the request library
:param event_name: name of the event
:type event_name: str
:param key: key for a particular application
:param custom_payload: custom_payload, defaults to {}
:type custom_payload: dict, optional
:param event_message_params: event_message_params, defaults to []
:type event_message_params: list, optional
"""
json_data = {}
path = f"/some/path"
self.request(path, "POST", json=json_data)
which calls following request function
def request(self, api_path, method="GET", **kwargs):
try:
self._validate_configuration()
headers = {}
api_endpoint = self.service_uri.to_url(api_path)
logger.debug(api_endpoint)
if "headers" in kwargs and kwargs["headers"]:
headers.update(kwargs["headers"])
headers = {"Content-Type": "application/json"}
begin = datetime.now()
def build_success_metrics(response, *args, **kwargs):
tags = {
"name": "success_metrics",
"domain": api_endpoint,
"status_code": 200,
}
build_metrics(tags)
def check_for_errors(response, *args, **kwargs):
response.raise_for_status()
response = self.session.request(
method=method,
url=api_endpoint,
headers=headers,
timeout=self.timeout,
hooks={"response": [build_success_metrics, check_for_errors]},
**kwargs,
)
end = datetime.now()
logger.debug(
f"'{method}' request against endpoint '{api_endpoint}' took {round((end - begin).total_seconds() * 1000, 3)} ms"
)
logger.debug(f"response: {response}")
except RequestException as e:
tags = {
"name": "error_metrics",
"domain": api_endpoint,
"exception_class": e.__class__.__name__,
}
build_metrics(tags)
return f"Exception occured: {e}"
Let me know if anything else is required from my end to explain what exactly I have done and what I am trying to achieve.
There is not much await and async in your example so I've just addressed the counting part of your question in general terms in asyncio. asyncio.Queue is good for this because you can separate out the counting from the cause quite simply.
import asyncio
import aiohttp
class Count():
def __init__(self, queue: asyncio.Queue):
self.queue = queue
self.good = 0
self.bad = 0
async def count(self):
while True:
result = await self.queue.get()
if result == 'Exit':
return
if result == 200:
self.good += 1
else:
self.bad += 1
async def request(q: asyncio.Queue):
async with aiohttp.ClientSession() as session:
for _ in range(5): # just poll 30 times in this instance
await asyncio.sleep(0.1)
async with session.get(
'https://httpbin.org/status/200%2C500', ssl=False
) as response:
q.put_nowait(response.status)
q.put_nowait('Exit')
async def main():
q = asyncio.Queue()
cnt = Count(q)
tasks = [cnt.count(), request(q)]
await asyncio.gather(*[asyncio.create_task(t) for t in tasks])
print(cnt.good, cnt.bad)
if __name__ == "__main__":
asyncio.run(main())
Output is random given httpbin response. Should add to 5.
4 1
Here is my unit test case written using monkey patch library in python :
#pytest.mark.asyncio
async def test_my_method(monkeypatch):
app.dependency_overrides[check_token] = override_token_expired
async def mock_cosmos_query(*args, **kwargs):
return fake_spaces
monkeypatch.setattr('app.routers.space.cosmos_query', mock_cosmos_query)
response = client.get("/api/spaces/TestSpace", headers={"authorization": "fake user"}, params={"expand": "false"})
jsonObj = response.json()
assert response.status_code == 200
assert jsonObj['name'] == 'TestSpace'
monkeypatch.undo() // NOT WORKING
The problem is that when I do monkeypatch.undo(), it doesn't undo monkeypatch.setattr() and hence rest of my test cases fails.
My expectation is that when I do monkeypatch.undo(), it should reset monkeypatch and the bahviour of monkey patch do not leak to other test cases.
Can someone please guid me here ?
I have async call, for example
from httpx import AsyncClient, Response
client = AsyncClient()
my_call = client.get(f"{HOST}/api/my_method") # async call
And I want to pass it to some retry logic like
async def retry_http(http_call):
count = 5
status, response = None, None
while count > 0:
response: Response = await http_call
if response.status_code == 200:
break
count -= 1
if response.status_code in (502, 504):
await asyncio.sleep(2)
else:
break
if response.status_code != 200:
return {
"success": False,
"result": {
"error": f"Response Error",
"response_code": response.status_code,
"response": response.text,
}
}
return response.json()
await retry_http(my_call)
but I got
RuntimeError
cannot reuse already awaited coroutine
Are there any method to make my_call an reusable coroutine ?
It is not possible in Python - a co-routine, once created, have an internal state that can't be easily duplicated - so once it runs, the internal state changes, including the internal line of code that is in execution, and there is no way to "rewind" that.
The most simple approach is to do like in #RyabchenkoAlexander's answer and accept the co-routine function and its parameters separately, and create the co-routine inside your retry function.
An alternative that is a nice Python idiom is to decorate the co-routine function - you make your retry_http a decorator instead, which wraps the underlying co-routine function in the retrying code.
Then, if the functions where you want this behavior are in your code, you can use the decorator syntax (#name prefixing the function definion) so that all calls will have the retry behavior, or you can apply it as a plain expression to get a new, retriable, co-routine function. Your final call could be:
result = await (retry_http(client.get) (f"{HOST}/api/my_method"))
(note the extra pair of parentheses around client.get, decorating it)
The decorator itself could be:
def retry_http(coro_func):
async def wrapper(*args, **kw):
# your original code - just replacing the await expression
...
while count > 5:
...
result = await coro_func(*args, **kw)
...
...
return result
return wrapper
As for your original intent: it would actually be possible to introspect a coroutine object, its internal variables and passed parameter, to recreate a co-routine object that has not yet started - however, that would involve using introspection to locate the original callable, and making the call again - it would be cumbersome, could be slow, and for little gain. I will outline the requirements, nonetheless:
A co-routine object has the cr_code and cr_frame attributes - you'd need to retrieve the function associated with the code object in cr_code- probably using the garbage colector API, or recreate a new function re-using the same code object, by calling types.FunctionType with the same parameters - and the local and global variables can be retrieved from the frame object in cr_frame.
can be fixed in next way
async def retry_http(http_call, *args, **kwargs):
count = 5
status, response = None, None
while count > 0:
response: Response = await http_call(*args, **kwargs)
if response.status_code == 200:
break
count -= 1
if response.status_code in (502, 504):
await asyncio.sleep(2)
else:
break
if response.status_code != 200:
return {
"success": False,
"result": {
"error": f"Response Error",
"response_code": response.status_code,
"response": response.text,
}
}
return response.json()
client = AsyncClient()
await retry_http(client.get, f"{HOST}/api/my_method")
I have a method which is calling two different end points and validating there response.
def foo_bar:
status_1 = requests.post(
"http://myapi/test/status1", {},
headers=headers)
status_2 = requests.post(
"http://myapi/test/status2", {},
headers=headers)
# and check the responses ...
I want to mock the both the url in pytest like this:
def foo_test:
with requests_mock.Mocker() as m1:
m1.post('http://myapi/test/status1',
json={},
headers={'x-api-key': my_api_key})
m1.post('http://myapi/test/status2',
json={},
headers={'x-api-key': my_api_key})
It always throws the error
**NO mock address: http://myapi/test/status2**
seems like its only mocking first url.
So is there any way to mock more than one url in one method?
Yes there is. From the docs: "There is a special symbol at requests_mock.ANY which acts as the wildcard to match anything. It can be used as a replace for the method and/or the URL."
import requests_mock
with requests_mock.Mocker() as rm:
rm.post(requests_mock.ANY, text='resp')
I am not sure if this is the best way but it works for me. You can assert afterwards which URLs were called with:
urls = [r._request.url, for r in rm._adapter.request_history]
I think you have something else going on, it's very normal to mock out a single path like this at a time so that you can return different values from different paths simply. Your example works for me:
import requests
import requests_mock
with requests_mock.Mocker() as m1:
my_api_key = 'key'
m1.post('http://myapi/test/status1',
json={},
headers={'x-api-key': my_api_key})
m1.post('http://myapi/test/status2',
json={},
headers={'x-api-key': my_api_key})
headers = {'a': 'b'}
status_1 = requests.post("http://myapi/test/status1", {}, headers=headers)
status_2 = requests.post("http://myapi/test/status2", {}, headers=headers)
assert status_1.status_code == 200
assert status_2.status_code == 200
Yes, there is a way!
You need to use additional_matcher callback (see docs) and requests_mock.ANY as URL.
Your example (with context manager)
import requests
import requests_mock
headers = {'key': 'val', 'another': 'header'}
def my_matcher(request):
url = request.url
mocked_urls = [
"http://myapi/test/status1",
"http://myapi/test/status2",
]
return url in mocked_urls # True or False
# as Context manager
with requests_mock.Mocker() as m1:
m1.post(
requests_mock.ANY, # Mock any URL before matching
additional_matcher=my_matcher, # Mock only matched
json={},
headers=headers,
)
r = requests.post('http://myapi/test/status1')
print(f"{r.text} | {r.headers}")
r = requests.post('http://myapi/test/status2')
print(f"{r.text} | {r.headers}")
# r = requests.get('http://myapi/test/status3').text # 'NoMockAddress' exception
Adaptation for pytest
Note: import requests_mock library with alias (because requests_mock is a fixture in pytest tests)
See example for pytest framework, GET method and your URLs:
# test_some_module.py
import requests
import requests_mock as req_mock
def my_matcher(request):
url = request.url
mocked_urls = [
"http://myapi/test/status1",
"http://myapi/test/status2",
]
return url in mocked_urls # True or False
def test_mocking_several_urls(requests_mock): # 'requests_mock' is fixture here
requests_mock.get(
req_mock.ANY, # Mock any URL before matching
additional_matcher=my_matcher, # Mock only matched
text="Some fake response for all matched URLs",
)
... Do your requests ...
# GET URL#1 -> response "Some fake response for all matched URLs"
# GET URL#2 -> response "Some fake response for all matched URLs"
# GET URL#N -> Raised exception 'NoMockAddress'
I am trying to check the response status code after trigerring some API with a POST method, Response status code is of Magicmock instance type, i am checking whether the status code is inbetween 400 and 500 using comparison operator which works in python 2 but raises TypeError in python 3
import mock
response = <MagicMock name='Session().post()' id='130996186'>
Below code works in python 2
if (400 <= response.status_code <= 500):
print('works')
But when executed in python 3, raises
TypeError: '<=' not supported between instances of 'int' and 'MagicMock'
class BMRAPI(object):
root_url = None
def __init__(self, user, api_key, root_url=BMR_URL,
api_uri=RESULTS_API_URI):
self.log =
logging.getLogger("BMRframework.Reporting.BMR6.BMRAPI")
self.root_url = root_url
self.url = urljoin(root_url, api_uri)
self.log.info("Connecting to BMR REST API: %s" % self.url)
self.session = requests.Session()
auth = 'ApiKey {0}:{1}'.format(user, api_key)
self.session.headers.update({
'Content-type': 'application/json',
'Accept': 'text/plain',
'Authorization': auth})
self.session.trust_env = False # bypass the proxy
self.log.debug("Authenticating as: %s" % user)
self.log.debug("Using API Key: %s" % api_key)`enter code here`
self.log.info("Connection to REST API successful")
def url_for_resource(self, resource_name):
return urljoin(self.url, resource_name) + "/"
def create(self, resource_name, data):
response = self.session.post(self.url_for_resource(resource_name),
json.dumps(data), timeout=TIMEOUT)
return self.handle_response(response)
def handle_response(self, response):
if (400 <= response.status_code <= 500):
print('mars')
Below is the UNit test case
#mock.patch("requests.Session")
def BMRAPI(Session):
api = BMRAPI('http://1.2.3.4/', 'dummy_user', '12345')
data = {'hello': 123}
api.create('testresource', data)
This isn't exactly a fix, more of a workaround.
Instead of making that <= comparison, write a separate method:
def is_4xx_or_5xx_code(status_code):
return 400 <= status_code <= 500
if is_4xx_or_5xx_code(status_code=response.status_code):
print('works')
Then mock it in your tests.
#mock.patch('path.to_code.under_test.is_4xx_or_5xx_code')
def test_your_method(mock_status_code):
mock_status_code.return_value = True
# rest of the test.