I'm looking for guidance calling a class in main vs methods.
A simple example would be a single method script.
# filename: one_funct.py
import argparse
import my_utils
from my_module_name import MyAwesomeClass
def my_one_func(class_inst, log):
aw_work_work = class_inst.method()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Run One Function')
parser.add_argument('class_config', type=str, help='Class configuration filepath.')
opts = parser.parse_args()
_log = my_utils.setup_logging('run_one_funct')
class_config_dict = my_utils.get_config_dict(opts.class_config)
my_class_inst = MyAwesomeClass(class_config_dict)
my_one_func(my_class_inst, _log)
I've only ever been shown that invoking the class in main will mean I have to do it in my unittests when testing my_one_func(). But, there has to be more to whether or not it's better/worse doing it in the script function.
I'm aware doing it in main allows me to pass the single class instance to multiple methods. For instance, assuming all methods required the same class, if I had a 2...N function script I could do...
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Run One Function')
parser.add_argument('class_config', type=str, help='Class configuration filepath.')
opts = parser.parse_args()
_log = my_utils.setup_logging('run_one_funct')
class_config_dict = my_utils.get_config_dict(opts.class_config)
my_class_inst = MyAwesomeClass(class_config_dict)
my_func_1(my_class_inst, _log)
my_func_2(my_class_inst, _log)
...
my_func_N(my_class_inst, _log)
What I don't know is whether or not I should. I've been looking for an answer for a while now. Apologies if it does exist here.
Related
This is my sample code:
from path.lib import DBInterface
class MyClass:
def __init__(self):
self.something = "something"
def _my_method(self, some_key, new_setup):
with DBInterface(self.something) as ic:
current_setup = ic.get(some_key)
if current_setup != new_setup:
with DBInterface(self.something) as ic:
ic.set(new_setup)
def public_method(self, some_key, new_setup):
return self._my_method(some_key, new_setup)
(my actual code is bit more complex, but i cant put it here on public :)
Now, what I want to do is, I want to completely mock the imported class DBInterface, because I do not want my unittests to do anything in DB.
BUT I also need the ic.get(some_key) to return some value, or to be more precise, I need to set the value it returns, because thats the point of my unittests, to test if the method behave properly according to value returned from DB.
This is how far I got:
class TestMyClass:
def test_extractor_register(self, mocker):
fake_db = mocker.patch.object('my_path.my_lib.DBInterface')
fake_db.get.return_value = None
# spy_obj = mocker.spy(MyClass, "_my_method")
test_class = MyClass()
# Test new registration in _extractor_register
result = test_class.public_method(Tconf.test_key, Tconf.test_key_setup)
fake_db.assert_has_calls([call().__enter__().get(Tconf.test_key),
call().__enter__().set(Tconf.test_key, Tconf.test_key_setup)])
# spy_obj.assert_called_with(ANY, Tconf.test_key, Tconf.test_key_setup)
assert result.result_status.status_code == Tconf.status_ok.status_code
assert result.result_data == MyMethodResult.new_reg
But i am unable to set return value for call().__enter__().get(Tconf.test_key).
I have been trying many approaches:
fake_db.get.return_value = None
fake_db.__enter__().get.return_value = None
fake_db.__enter__.get = Mock(return_value=None)
mocker.patch.object(MyClass.DBInterface, "get").return_value = None
None of that is actually working and I am running out of options I can think about.
Without having more code or errors that are being produced, it's tough to provide a conclusive answer.
However, if you truly only need to specify a return value for set() I would recommend using MagicMock by virtue of patch --
from unittest.mock import patch
#patch("<MyClassFile>.DBInterface", autospec=True)
def test_extractor_register(mock_db):
mock_db.set.return_value = "some key"
# Rest of test code
Taken from this answer:
Python mock AWS SSM
I now have this code:
test_2.py
from unittest import TestCase
import boto3
import pytest
from moto import mock_ssm
#pytest.yield_fixture
def s3ssm():
with mock_ssm():
ssm = boto3.client("ssm")
yield ssm
#mock_ssm
class MyTest(TestCase):
def setUp(self):
ssm = boto3.client("ssm")
ssm.put_parameter(
Name="/mypath/password",
Description="A test parameter",
Value="this is it!",
Type="SecureString",
)
def test_param_getting(self):
import real_code
resp = real_code.get_variable("/mypath/password")
assert resp["Parameter"]["Value"] == "this is it!"
and this is my code to test (or a cut down example):
real_code.py
import boto3
class ParamTest:
def __init__(self) -> None:
self.client = boto3.client("ssm")
pass
def get_parameters(self, param_name):
print(self.client.describe_parameters())
return self.client.get_parameters_by_path(Path=param_name)
def get_variable(param_name):
p = ParamTest()
param_details = p.get_parameters(param_name)
return param_details
I have tried a number of solutions, and switched between pytest and unittest quite a few times!
Each time I run the code, it doesn't reach out to AWS so it seems something is affecting the boto3 client, but it doesn't return the parameter. If I edit real_code.py to not have a class inside it the test passes.
Is it not possible to patch the client inside the class in the real_code.py file? I'm trying to do this without editing the real_code.py file at all if possible.
Thanks,
The get_parameters_by_path returns all parameters that are prefixed with the supplied path.
When providing /mypath, it would return /mypath/password.
But when providing /mypath/password, as in your example, it will only return parameters that look like this: /mypath/password/..
If you are only looking to retrieve a single parameter, the get_parameter call would be more suitable:
class ParamTest:
def __init__(self) -> None:
self.client = boto3.client("ssm")
pass
def get_parameters(self, param_name):
# Decrypt the value, as it is stored as a SecureString
return self.client.get_parameter(Name=param_name, WithDecryption=True)
Edit: Note that Moto behaves the same as AWS in this.
From https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameters_by_path:
[The Path-parameter is t]he hierarchy for the parameter. [...] The hierachy is the parameter name except the last part of the parameter. For the API call to succeeed, the last part of the parameter name can't be in the path.
I am using pytest, with coveragerc to measure code coverage of my source file, which is given below
import argparse
import sys
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--input_dir', type=str, required=True, help='fully qualified directory path')
arguments = parser.parse_known_args(sys.argv[1:])
my_func(arguments.input_dir)
My coveragerc configuration is given below
[report]
exclude_lines =
if __name__ == .__main__.:
pytest config
[pytest]
python_files=*_test.py
addopts = --cov='.' --cov-config=.coveragerc
Tests to validate functionality in my_func exist, but not for the lines of code under main function. I would like to be able to exclude all lines within the main function from the code coverage because its functionality just initialises the parser and calls the another function.
Let's say we have following setup (copied & shortened from the Hydra docs):
Configuration file: config.yaml
db:
driver: mysql
user: omry
pass: secret
Python file: my_app.py
import hydra
#hydra.main(config_path="config.yaml")
def my_app(cfg):
print(cfg.pretty())
if __name__ == "__main__":
my_app()
This works well when we can use a decorator on the function my_app. Now I would like (for small scripts and testing purposes, but that is not important) to get this cfg object outside of any function, just in a plain python script. From what I understand how decorators work, it should be possible to call
import hydra
cfg = hydra.main(config_path="config.yaml")(lambda x: x)()
print(cfg.pretty())
but then cfg is just None and not the desired configuration object. So it seems that the decorator does not pass on the returned values. Is there another way to get to that cfg ?
Use the Compose API:
from hydra import compose, initialize
from omegaconf import OmegaConf
initialize(config_path="conf", job_name="test_app")
cfg = compose(config_name="config", overrides=["db=mysql", "db.user=me"])
print(OmegaConf.to_yaml(cfg))
This will only compose the config and will not have side effects like changing the working directory or configuring the Python logging system.
None of the above solutions worked for me. They gave errors:
'builtin_function_or_method' object has no attribute 'code'
and
GlobalHydra is already initialized, call
Globalhydra.instance().clear() if you want to re-initialize
I dug further into hydra and realised I could just use OmegaConf to load the file directly. You don't get overrides but I'm not fussed about this.
import omegaconf
cfg = omegaconf.OmegaConf.load(path)
I found a rather ugly answer but it works - if anyone finds a more elegant solution please let us know!
We can use a closure or some mutable object. In this example we define a list outside and append the config object:
For hydra >= 1.0.0 you have to use config_name instead, see documentation.
import hydra
c = []
hydra.main(config_name="config.yaml")(lambda x:c.append(x))()
cfg = c[0]
print(cfg)
For older versions:
import hydra
c = []
hydra.main(config_path="config.yaml")(c.append)()
cfg = c[0]
print(cfg.pretty())
anther ugly answer, but author said this may be crush in next version
Blockquote
from omegaconf import DictConfig
from hydra.utils import instantiate
from hydra._internal.utils import _strict_mode_strategy, split_config_path, create_automatic_config_search_path
from hydra._internal.hydra import Hydra
from hydra.utils import get_class
class SomeThing:
...
def load_from_yaml(self, config_path, strict=True):
config_dir, config_file = split_config_path(config_path)
strict = _strict_mode_strategy(strict, config_file)
search_path = create_automatic_config_search_path(
config_file, None, config_dir
)
hydra = Hydra.create_main_hydra2(
task_name='sdfs', config_search_path=search_path, strict=strict
)
config = hydra.compose_config(config_file, [])
config.pop('hydra')
self.config = config
print(self.config.pretty())
This is my solution
from omegaconf import OmegaConf
class MakeObj(object):
""" dictionary to object.
Thanks to https://stackoverflow.com/questions/1305532/convert-nested-python-dict-to-object
Args:
object ([type]): [description]
"""
def __init__(self, d):
for a, b in d.items():
if isinstance(b, (list, tuple)):
setattr(self, a, [MakeObj(x) if isinstance(x, dict) else x for x in b])
else:
setattr(self, a, MakeObj(b) if isinstance(b, dict) else b)
def read_yaml(path):
x_dict = OmegaConf.load(path)
x_yamlstr = OmegaConf.to_yaml(x_dict)
x_obj = MakeObj(x_dict)
return x_yamlstr, x_dict, x_obj
x_yamlstr, x_dict, x_obj = read_yaml('config/train.yaml')
print(x_yamlstr)
print(x_dict)
print(x_obj)
print(dir(x_obj))
I have a question for pytest
I would like to run same pytest script with multiple threads.
But,i am not sure how to create and run thread which is passing more than one param. (And running thread with pytest..)
for example I have
test_web.py
from selenium import webdriver
import pytest
class SAMPLETEST:
self.browser = webdriver.Chrome()
self.browser.get(URL)
self.browser.maximize_window()
def test_title(self):
assert "Project WEB" in self.browser.title
def test_login(self):
print('Testing Login')
ID_BOX = self.broswer.find_element_by_id("ProjectemployeeId")
PW_BOX = self.broswer.find_element_by_id("projectpassword")
ID_BOX.send_keys(self.ID) # this place for ID. This param come from thread_run.py
PW_BOX.send_keys(self.PW) # this place for PW. It is not working. I am not sure how to get this data from threa_run.py
PW_BOX.submit()
IN thread_run.py
import threading
import time
from test_web import SAMPLETEST
ID_List = ["0","1","2","3","4","5","6","7"]
PW_LIST = ["0","1","2","3","4","5","6","7"]
threads = []
print("1: Create thread")
for I in range(8):
print("Append thread" + str(I))
t = threading.Thread(target=SAMPLETEST, args=(ID_List[I], PW_LIST[I]))
threads.append(t)
for I in range(8):
print("Start thread:" + str(I))
threads[I].start()
i was able to run thread to run many SAMPLETEST class without pytest.
However, it is not working with pytest.
My question is.
First, how to initialize self.brower in insde of SAMPLETEST? I am sure below codes will not be working
self.browser = webdriver.Chrome()
self.browser.get(URL)
self.browser.maximize_window()
Second, in thread_run.py, how can i pass the two arguments(ID and Password) when I run thread to call SAMPLTEST on test_web.py?
ID_BOX.send_keys(self.ID) # this place for ID. This param come from thread_run.py
ID_BOX.send_keys(self.ID)
PW_BOX.send_keys(self.PW)
I was trying to build constructor (init) in SAMPLETEST class but it wasn't working...
I am not really sure how to run threads (which passing arguments or parameter ) with pytest.
There are 2 scenarios which i could read from this:
prepare the test data and pass in parameters to your test method which could be achieved by pytest-generate-tests and parameterise concept. You can refer to the documentation here
In case of running pytest in multi threading - Pytest-xdist or pytest-parallel
I had a similar issue and got it resolved by passing an argument in form of a list.
Like:,
I replaced below line
thread_1 = Thread(target=fun1, args=10)
with
thread_1 = Thread(target=fun1, args=[10])