How to set mocked exception behavior on Python? - python-3.x

I am using an external library (github3.py) that defines an internal exception (github3.exceptions.UnprocessableEntity). It doesn't matter how this exception is defined, so I want to create a side effect and set the attributes I use from this exception.
Tested code not-so-minimal example:
import github3
class GithubService:
def __init__(self, token: str) -> None:
self.connection = github3.login(token=token)
self.repos = self.connection.repositories()
def create_pull(self, repo_name: str) -> str:
for repo in self.repos:
if repo.full_name == repo_name:
break
try:
created_pr = repo.create_pull(
title="title",
body="body",
head="head",
base="base",
)
except github3.exceptions.UnprocessableEntity as github_exception:
extra = ""
for error in github_exception.errors:
if "message" in error:
extra += f"{error['message']} "
else:
extra += f"Invalid field {error['field']}. " # testing this case
return f"{repo_name}: {github_exception.msg}. {extra}"
I need to set the attributes msg and also errors from the exception. So I tried in my test code using pytest-mock:
#pytest.fixture
def mock_github3_login(mocker: MockerFixture) -> MockerFixture:
"""Fixture for mocking github3.login."""
mock = mocker.patch("github3.login", autospec=True)
mock.return_value.repositories.return_value = [
mocker.Mock(full_name="staticdev/nope"),
mocker.Mock(full_name="staticdev/omg"),
]
return mock
def test_create_pull_invalid_field(
mocker: MockerFixture, mock_github3_login: MockerFixture,
) -> None:
exception_mock = mocker.Mock(errors=[{"field": "head"}], msg="Validation Failed")
mock_github3_login.return_value.repositories.return_value[1].create_pull.side_effect = github3.exceptions.UnprocessableEntity(mocker.Mock())
mock_github3_login.return_value.repositories.return_value[1].create_pull.return_value = exception_mock
response = GithubService("faketoken").create_pull("staticdev/omg")
assert response == "staticdev/omg: Validation Failed. Invalid field head."
The problem with this code is that, if you have side_effect and return_value, Python just ignores the return_value.
The problem here is that I don't want to know the implementation of UnprocessableEntity to call it passing the right arguments to it's constructor. Also, I didn't find other way using just side_effect. I also tried to using return value and setting the class of the mock and using it this way:
def test_create_pull_invalid_field(
mock_github3_login: MockerFixture,
) -> None:
exception_mock = Mock(__class__ = github3.exceptions.UnprocessableEntity, errors=[{"field": "head"}], msg="Validation Failed")
mock_github3_login.return_value.repositories.return_value[1].create_pull.return_value = exception_mock
response = GithubService("faketoken").create_pull("staticdev/omg")
assert response == "staticdev/omg: Validation Failed. Invalid field head."
This also does not work, the exception is not thrown. So I don't know how to overcome this issue given the constraint I don't want to see the implementation of UnprocessableEntity. Any ideas here?

So based on your example, you don't really need to mock github3.exceptions.UnprocessableEntity but only the incoming resp argument.
So the following test should work:
def test_create_pull_invalid_field(
mocker: MockerFixture, mock_github3_login: MockerFixture,
) -> None:
mocked_response = mocker.Mock()
mocked_response.json.return_value = {
"message": "Validation Failed", "errors": [{"field": "head"}]
}
repo = mock_github3_login.return_value.repositories.return_value[1]
repo.create_pull.side_effect = github3.exceptions.UnprocessableEntity(mocked_response)
response = GithubService("faketoken").create_pull("staticdev/omg")
assert response == "staticdev/omg: Validation Failed. Invalid field head."
EDIT:
If you want github3.exceptions.UnprocessableEntity to be completely abstracted, it won't be possible to mock the entire class as catching classes that do not inherit from BaseException is not allowed (See docs). But you can get around it by mocking the constructor only:
def test_create_pull_invalid_field(
mocker: MockerFixture, mock_github3_login: MockerFixture,
) -> None:
def _initiate_mocked_exception(self) -> None:
self.errors = [{"field": "head"}]
self.msg = "Validation Failed"
mocker.patch.object(
github3.exceptions.UnprocessableEntity, "__init__",
_initiate_mocked_exception
)
repo = mock_github3_login.return_value.repositories.return_value[1]
repo.create_pull.side_effect = github3.exceptions.UnprocessableEntity
response = GithubService("faketoken").create_pull("staticdev/omg")
assert response == "staticdev/omg: Validation Failed. Invalid field head."

Related

"Error 'ResourceSummaryCollection' object is not subscriptable" OCI PythonSDK

I have python script that queries OCI Compute Instances with a defined tag and value of test and will take the results and stop any instance with those tags. I am expecting the query to return the result as a json object of according to this:
https://docs.oracle.com/en-us/iaas/Content/Search/Tasks/queryingresources.htm
However whenever I take the result and apply it to a variable I get the following error:
"Error 'ResourceSummaryCollection' object is not subscriptable"
I noticed that the query response is not returning as a json object. It is returning as a list.
Update:
This is the error I am getting after updating my script: "Error 'list' object has no attribute 'identifier'". I was expecting the query to return as a json object. When I try to convert the list into a json object via the json dumps method I get object is not serializable. Bottom line is, how do I pass the OCID "Identifier" from the query?
My function:
def do(signer):
print("Searching for untagged instance", flush=True)
# results = ""
# message = ""
# resp = ""
try:
search_client = oci.resource_search.ResourceSearchClient(config={}, signer=signer)
print("Search client initialized",flush=True)
PredefinedTag = "apps"
key= "test"
value= "test"
structured_search = oci.resource_search.models.StructuredSearchDetails(
query="query instance resources where (definedTags.namespace = '{}' && definedTags.key = '{}' && definedTags.value = '{}')".format(PredefinedTag,key,value),
type='Structured',
matching_context_type=oci.resource_search.models.SearchDetails.MATCHING_CONTEXT_TYPE_NONE)
print("Step1",flush=True)
#old
results = search_client.search_resources(structured_search).data
# print(results.items)
# print(results.items.identifier)
# print(results['items'][0]['identifier'])
print("Step2",flush=True)
instanceId = results(results.items.identifier)
# instanceId = results['items'][0]['identifier']
print("Step3",flush=True)
resp = perform_action(signer, instanceId , 'STOP')
print("Step4",flush=True)
except oci.exceptions.ServiceError as e:
print('RQS Search failed with Service Error: {0}'.format(e),flush=True)
raise
except oci.exceptions.RequestException as e:
print('RQS Search failed w/ a Request exception. {0}'.format(e),flush=True)
raise
return resp
Was able to reference the response from the query:
def do(signer):
print("Searching for untagged instance", flush=True)
try:
search_client = oci.resource_search.ResourceSearchClient(config={}, signer=signer)
print("Search client initialized",flush=True)
PredefinedTag = "apps"
key= "test"
value= "test"
structured_search = oci.resource_search.models.StructuredSearchDetails(
query="query instance resources where (definedTags.namespace = '{}' && definedTags.key = '{}' && definedTags.value = '{}')".format(PredefinedTag,key,value),
type='Structured',
matching_context_type=oci.resource_search.models.SearchDetails.MATCHING_CONTEXT_TYPE_NONE)
print("Step1",flush=True)
results = search_client.search_resources(structured_search)
for result in results.data.items:
print(result.identifier + " has availability tag checking status")
print(result.identifier + " status is " + result.lifecycle_state)
instanceId = result.identifier
resp = perform_action(signer, instanceId , 'START')
except oci.exceptions.ServiceError as e:
print('RQS Search failed with Service Error: {0}'.format(e),flush=True)
raise
except oci.exceptions.RequestException as e:
print('RQS Search failed w/ a Request exception. {0}'.format(e),flush=True)
raise
return resp

Caching values on a module level and unit testing

Below is a module for querying and caching AWS STS tokens, the intention is to avoid querying STS if there is a valid token.
class Credentials:
def __init__(self):
self.sts_credentials = None
self.token_expiry_time = None
def is_token_expired(self):
current_time_with_buffer = datetime.now() + timedelta(minutes=2)
return not self.token_expiry_time or self.token_expiry_time < current_time_with_buffer
CREDENTIALS_ = Credentials()
def get_credentials():
if CREDENTIALS_.is_token_expired():
sts_client = boto3.client('sts')
LOGGER.info("The credentials are either empty or expiring, refreshing")
try:
sts_token = sts_client.assume_role(
RoleArn=os.environ["KINESIS_ASSUME_ROLE"],
RoleSessionName=str(uuid.uuid4()))
except Exception as e:
LOGGER.error(f"Error occurred while trying to assume role with {os.environ['KINESIS_ASSUME_ROLE']}", e)
raise e
CREDENTIALS_.sts_credentials = {
"aws_access_key_id": sts_token['Credentials']['AccessKeyId'],
"aws_secret_access_key": sts_token['Credentials']['SecretAccessKey'],
"aws_session_token": sts_token['Credentials']['SessionToken']
}
CREDENTIALS_.token_expiry_time = sts_token["Credentials"]["Expiration"]
return CREDENTIALS_.sts_credentials
One of the unit tests is as below, this passes in isolation, but fails when run alongside other tests, the reason being CREDENTIALS_ variable, which is modified by other tests, I can set this value to None, but I want to know what is the cleaner way of clearing the cached value
def test_get_credentials_refreshes_token_if_about_to_expire(sts_response, credentials):
with mock.patch("boto3.client") as mock_boto_client:
mock_assume_role = mock_boto_client.return_value.assume_role
mock_assume_role.return_value = sts_response
get_credentials()
actual_credentials = get_credentials()
calls = [call('sts'),
call().assume_role(RoleArn='arn:aws:iam::000000000000:role/dummyarn', RoleSessionName=ANY),
call('sts'),
call().assume_role(RoleArn='arn:aws:iam::000000000000:role/dummyarn', RoleSessionName=ANY)]
assert credentials == actual_credentials
mock_boto_client.assert_has_calls(calls)
The cleaner way would be to make sure that your unit tests are performing unit tests. This means that for every unit there should be no interaction with other units. Since you are using a global variable CREDENTIALS_, this is going to be nearly impossible.
1) easy fix
An easy fix would be to pass CREDENTIALS_ as input argument. Then you can create a fake CREDENTIALS_ object during each of the tests, that are tailored to your test conditions.
2) Better fix
A better solution would be, besides using the credential input argument, to break up the logic inside the get_credentials. By splitting it into smaller functions, you can separate the server logic and the credential updating. Making it easier to Mock and test. A possible division of the whole function would be:
get_sts_token
update_credentials
get_credentials
Now the get_sts_token has connections to the server, but the update_credentials and get_credentials do not have to directly interact with it.
Code
Example 1)
def update_credentials(credentials):
if credentials.is_token_expired():
sts_client = boto3.client('sts')
LOGGER.info("The credentials are either empty or expiring, refreshing")
try:
sts_token = sts_client.assume_role(
RoleArn=os.environ["KINESIS_ASSUME_ROLE"],
RoleSessionName=str(uuid.uuid4()))
except Exception as e:
LOGGER.error(f"Error occurred while trying to assume role with {os.environ['KINESIS_ASSUME_ROLE']}", e)
raise e
credentials.sts_credentials = {
"aws_access_key_id": sts_token['Credentials']['AccessKeyId'],
"aws_secret_access_key": sts_token['Credentials']['SecretAccessKey'],
"aws_session_token": sts_token['Credentials']['SessionToken']
}
credentials.token_expiry_time = sts_token["Credentials"]["Expiration"]
return credentials
# Where you need the credentials
CREDENTIALS_ = update_credentials(CREDENTIALS_)
CREDENTIALS_.sts_credentials
Now you can insert your own CREDENTIALS_ object in the test.
Example 2)
def get_sts_token():
sts_client = boto3.client('sts')
LOGGER.info("The credentials are either empty or expiring, refreshing")
try:
sts_token = sts_client.assume_role(
RoleArn=os.environ["KINESIS_ASSUME_ROLE"],
RoleSessionName=str(uuid.uuid4()))
except Exception as e:
LOGGER.error(f"Error occurred while trying to assume role with {os.environ['KINESIS_ASSUME_ROLE']}", e)
raise e
return sts_token
def update_credentials(credentials, sts_token):
credentials.sts_credentials = {
"aws_access_key_id": sts_token['Credentials']['AccessKeyId'],
"aws_secret_access_key": sts_token['Credentials']['SecretAccessKey'],
"aws_session_token": sts_token['Credentials']['SessionToken']
}
return credentials
def get_credentials(credentials: Credentials):
if credentials.is_token_expired():
sts_token = get_sts_token()
credentials = update_credentials(credentials, sts_token)
return credentials.sts_credentials

How to write unit_test for a function which only updates a global dict type variable - Python(pytest)

Unit-Test code for a function which does a validation operation and updates the Global dict - result_count = {'test_method':{'Total_tested': 0, 'passed': 0, 'failed': 0}
below is the function
def validate_response(testmethod, response, expected_data):
ra = response.json()
expected = expected_data['payload']
if (response.status_code == expected['response_status']) \
and (result_count[testmethod['folder']]["FAILED"] < 10):
------code logic - checks using jsondiff and re.expressions--
else:
missmatch = 'response status code missmatch'
update_result(testmethod, 'status_code', expected_data, response, missmatch, fail=True)
result_count[testmethod['folder']]['FAILED'] += 1
Need to write test for the above function. to check if the result_count is updated properly.
Regular expression library & jsondiff is also used in the mentioned function.
help required for mocking the global variable and using the same for testing.
During executing the test script below i was getting a key error for the global variable result_count. That implies that the code is unable to access the result_count.. the key error after the update - is throwing TypeError: '<=' not supported between instances of 'MagicMock' and 'int' as Error
The current issue is that the result_count is not updated when the below line code is executed.
partner_test.validate_response(test_input_mocker.method, response, expected_data)
my unit test script is as below
#patch("tests.p_test.result")
#patch("tests.p_test.result_count")
def test_validate_response_pass(result_count_mocker, monkeypatch, result_mocker, test_input_mocker):
# Build data for validate response function
response = Resp(200, {'message': 'pong'})
response_data = response.json()
expected_data = {some_test_data}
# global variable import and initialize result, result_count
from p_test import result, result_count
result_count.update(result_count_mocker.data)
result.update(result_mocker.data)
result_count_mocker.return_value = result_count_mocker.data
def update_result_mocker(*args):
mock operations here
return None
monkeypatch.setattr(partner_test, "update_result", update_result_mocker)
p_test.validate_response(test_input_mocker.method, response, expected_data)
Resp() in the test fnction is a response class created to mock the response object.
the issue can be resolved using #patch.dict(dict_name, values) below is the solution
#patch.dict(p_test.result, {dict_values })
#patch.dict(p_test.result_count, {dict_values})
def test_validate_response(monkeypatch, test_input_mocker, result_count_test):
response = Resp(200, {'message': 'pong'})
response_data = response.json()
expected_data = {----your data-----}
def update_result_mocker(*args):
# mock_operations
monkeypatch.setattr(p_test, "update_result", update_result_mocker)
p_test.validate_response(test_input_mocker.method, response, expected_data)
print(p_test.result_count)

Getting TypeError: Object of type set is not JSON serializable, when trying to post a dictionary through a REST-API

Trying to POST through REST-API, some JSON object, but having some issue and getting the error TypeError: Object of type set is not JSON serializable
Getting error when trying to post the agentResource_info to a REST-API
def agentresource(self):
global fogarea_info
while self.deviceID_cimiresource is None:
time.sleep(0.1)
deviceID = self.userID
dID = str(deviceID)
# devID = {"device": dID}
MyleaderID = {str(self.deviceID_cimiresource)}
while self._running:
dynamic = dynamic_info()
devdyna = json.loads(dynamic)
devwifiip = devdyna['wifiAddress']
devethip = devdyna['ethernetAddress']
devips = [*devethip, *devwifiip]
devIP = str(', '.join(devips))
devip = str(devIP)
r2 = requests.get("http://cimi:8201/api/device-dynamic",headers={"slipstream-authn-info": "internal ADMIN"}, verify=False)
dynamics_info = r2.json()
rs_info = dynamics_info['deviceDynamics']
ips1 = [item['wifiAddress'] for item in rs_info]
ips2 = [item['ethernetAddress'] for item in rs_info]
ips3 = [*ips1, *ips2]
childips = str(ips3)
backupip = 'None'
authenticated = True
connect = True
isleader = True
agentResource_info = json.dumps({"device_id": MyleaderID, "device_ip": devip, "leader_id": dID, "leader_ip": devip, "authenticated": authenticated, "connected": connect, "isLeader": isleader, "backup_ip": backupip, "childrenIPs": childips})
#print("I'm here can you see me!!!!")
#agentRes_info = json.dumps(agentResource_info)
#print(agentResource_info)
try:
if self.count is 0:
r1 = requests.post("http://cimi:8201/api/agent-resource",headers={"slipstream-authn-info": "internal ADMIN"},json=agentRes_info, verify=False)
print('r1', r1, r1.request, r1.reason, r1.json())
self.count = r1.json()['count']
else:
r1 = requests.put("http://cimi:8201/api/agent-resource",headers={"slipstream-authn-info": "internal ADMIN"},json=agentRes_info, verify=False)
print('r1', r1, r1.request, r1.reason, r1.json())
except ConnectionError as e:
print(e)
r1 = "No response"
print(r1)
time.sleep(60)
You wrote:
MyleaderID = {str(self.deviceID_cimiresource)}
and snakecharmerb points out that set
won't serialize as it isn't a native JSON type.
Better to assign a str instead.
Also, pep-8 asks that you use lowercase snake_case
for such variable names: my_leader_id

Google Matrix API - python return Nonetype error

"Update"
*Finally resolved the issue, changed the try except to include TypeError and also use pass instead of continue in the except.
"End of update"
I wrote code to search for distance between two locations using Google Distance Matrix API. The origin location are fixed, however for the destination, I get it from an xlsx file. I was expecting to get Dictionary with Destination as the Key and the distance as value. When I run the code below, after certain loop I'm stumbled with this error code:
TypeError: Expected a lat/lng dict or tuple, but got NoneType
Can you help me understand the cause of the error? Here is the code (pygmap.py):
import googlemaps
import openpyxl
#get origin and destination locations
def cleanWB(file_path):
destination = list()
wb = openpyxl.load_workbook(filename=file_path)
ws = wb.get_sheet_by_name('Sheet1')
for i in range(ws.max_row):
cellValueLocation = ws.cell(row=i+2,column=1).value
destination.append(cellValueLocation)
#remove duplicates from destination list
unique_location = list(set(destination))
return unique_location
def getDistance(origin, destination):
#Google distance matrix API key
gmaps = googlemaps.Client(key = 'INSERT API KEY')
distance = gmaps.distance_matrix(origin, destination)
distance_status = distance['rows'][0]['elements'][0]['status']
if distance_status != 'ZERO_RESULTS':
jDistance = distance['rows'][0]['elements'][0]
distance_location = jDistance['distance']['value']
else:
distance_location = 0
return distance_location
And I run it using this code:
import pygmap
unique_location = pygmap.cleanWB('C:/Users/an_id/Documents/location.xlsx')
origin = 'alam sutera'
result = {}
for i in range(len(unique_location)):
try:
result[unique_location[i]] = pygmap.getDistance(origin, unique_location[i])
except (KeyError, TypeError):
pass
If I print results it will show that I have successfully get 46 results
result
{'Pondok Pinang': 25905, 'Jatinegara Kaum': 40453, 'Serdang': 1623167, 'Jatiasih
': 44737, 'Tanah Sereal': 77874, 'Jatikarya': 48399, 'Duri Kepa': 20716, 'Mampan
g Prapatan': 31880, 'Pondok Pucung': 12592, 'Johar Baru': 46791, 'Karet': 26889,
'Bukit Duri': 34039, 'Sukamaju': 55333, 'Pasir Gunung Selatan': 42140, 'Pinangs
ia': 30471, 'Pinang Ranti': 38099, 'Bantar Gebang': 50778, 'Sukabumi Utara': 204
41, 'Kembangan Utara': 17708, 'Kwitang': 25860, 'Kuningan Barat': 31231, 'Cilodo
ng': 58879, 'Pademangan Barat': 32585, 'Kebon Kelapa': 23452, 'Mekar Jaya': 5381
0, 'Kampung Bali': 1188894, 'Pajang': 30008, 'Sukamaju Baru': 53708, 'Benda Baru
': 19965, 'Sukabumi Selatan': 19095, 'Gandaria Utara': 28429, 'Setia Mulya': 635
34, 'Rawajati': 31724, 'Cireundeu': 28220, 'Cimuning': 55712, 'Lebak Bulus': 273
61, 'Kayuringin Jaya': 47560, 'Kedaung Kali Angke': 19171, 'Pagedangan': 16791,
'Karang Anyar': 171165, 'Petukangan Selatan': 18959, 'Rawabadak Selatan': 42765,
'Bojong Sari Baru': 26978, 'Padurenan': 53216, 'Jati Mekar': 2594703, 'Jatirang
ga': 51119}
Resolved the issue to include TypeError in the Try Except. And also use pass instead of continue
import pygmap
unique_location = pygmap.cleanWB('C:/Users/an_id/Documents/location.xlsx')
origin = 'alam sutera'
result = {}
#get getPlace
for i in range(len(unique_location)):
try:
result[unique_location[i]] = pygmap.getDistance(origin, unique_location[i])
except (KeyError, TypeError):
pass
I skipped some locations using this solution though.

Resources