I would like to create my own pytest fixture where I can insert what I want it to do in the setup and teardown phase.
I am looking for something like this (in this example, i create a file that's needed for the test):
#pytest.fixture
def file(path, content):
def setup():
# check that file does NOT exist
if os.path.isfile(path):
raise Exception('file already exists')
# put contents in the file
with open(path, 'w') as file:
file.write(content)
def teardown():
os.remove(path)
and I would like to be able to use it like this:
def test_my_function(file):
file('/Users/Me/myapplication/info.txt', 'ham, eggs, orange juice')
assert my_function('info') == ['ham', 'eggs', 'orange juice']
I am aware there is already a tempdir fixture in pytest that has similar functionality. Unfortunately, that fixture only creates files somewhere within the /tmp directory, and I need files in my application.
Thanks!
UPDATE:
I'm getting pretty close. The following almost works, but it doesn't set the PATH variable global to the fixture like I expected. I'm wondering if I can create a class instead of a function for my fixture.
#pytest.fixture
def file(request):
PATH = None
def setup(path, content):
PATH = path
# check that file does NOT exist
if os.path.isfile(PATH):
raise Exception('file already exists')
# put contents in the file
with open(PATH, 'w+') as file:
file.write(content)
def teardown():
os.remove(PATH)
request.addfinalizer(teardown)
return setup
This is a bit crazy, but here is a solution:
#pytest.fixture
def file(request):
class File:
def __call__(self, path, content):
self.path = path
# check that file does NOT exist
if os.path.isfile(self.path):
raise Exception('file already exists')
# put contents in the file
with open(self.path, 'w+') as file:
file.write(content)
def teardown(self):
os.remove(self.path)
obj = File()
request.addfinalizer(obj.teardown)
return obj
Related
I have a function that does some file operations and
makes an entry for that IP to /etc/hosts file for DNS resolution
def add_hosts_entry():
ip_addr = "1.2.3.4"
HOST_FILE_PATH = "/etc/hosts"
reg_svc_name = "SVC_NAME"
try:
with open(HOST_FILE_PATH, 'r+') as fp:
lines = fp.readlines()
fp.seek(0)
fp.truncate()
for line in lines:
if not reg_svc_name in line:
fp.write(line)
fp.write(f"{ip_addr}\t{reg_svc_name}\n")
except FileNotFoundError as ex:
LOGGER.error(f"Failed to read file. Details: {repr(ex)}")
sys.exit(1)
LOGGER.info(
f"Successfully made entry in /etc/hosts file:\n{ip_addr}\t{reg_svc_name}"
)
I want to test that there is indeed an IP entry in the file that I
made.
and that there is only 1 IP address that maps to reg_svc_name
I found how to mock open().
I have this so far but not sure how to check for above two cases:
#pytest.fixture
def mocker_etc_hosts(mocker):
mocked_etc_hosts_data = mocker.mock_open(read_data=etc_hosts_sample_data)
mocker.patch("builtins.open", mocked_etc_hosts_data)
def test_add_hosts_entry(mocker_etc_hosts):
with caplog.at_level(logging.INFO):
registry.add_hosts_entry()
# how to assert??
Solution 1
Don't mock the open functionality because we want it to actually update a file that we can check. Instead, intercept it and open a test file instead of the actual file used in the source code. Here, we will use tmp_path to create a temporary file to be updated for the test.
src.py
def add_hosts_entry():
ip_addr = "1.2.3.4"
HOST_FILE_PATH = "/etc/hosts"
reg_svc_name = "SVC_NAME"
try:
with open(HOST_FILE_PATH, 'r+') as fp:
lines = fp.readlines()
fp.seek(0)
fp.truncate()
for line in lines:
if not reg_svc_name in line:
fp.write(line)
fp.write(f"{ip_addr}\t{reg_svc_name}\n")
except FileNotFoundError as ex:
print(f"Failed to read file. Details: {repr(ex)}")
else:
print(f"Successfully made entry in /etc/hosts file:\n{ip_addr}\t{reg_svc_name}")
test_src.py
import pytest
from src import add_hosts_entry
#pytest.fixture
def etc_hosts_content_raw():
return "some text\nhere\nSVC_NAME\nand the last!\n"
#pytest.fixture
def etc_hosts_content_updated():
return "some text\nhere\nand the last!\n1.2.3.4\tSVC_NAME\n"
#pytest.fixture
def etc_hosts_file(tmp_path, etc_hosts_content_raw):
file = tmp_path / "dummy_etc_hosts"
file.write_text(etc_hosts_content_raw)
return file
#pytest.fixture
def mocker_etc_hosts(mocker, etc_hosts_file):
real_open = open
def _mock_open(file, *args, **kwargs):
print(f"Intercepted. Would open {etc_hosts_file} instead of {file}")
return real_open(etc_hosts_file, *args, **kwargs)
mocker.patch("builtins.open", side_effect=_mock_open)
def test_add_hosts_entry(
mocker_etc_hosts, etc_hosts_file, etc_hosts_content_raw, etc_hosts_content_updated
):
assert etc_hosts_file.read_text() == etc_hosts_content_raw
add_hosts_entry()
assert etc_hosts_file.read_text() == etc_hosts_content_updated
Output
$ pytest -q -rP
. [100%]
============================================== PASSES ===============================================
_______________________________________ test_add_hosts_entry ________________________________________
--------------------------------------- Captured stdout call ----------------------------------------
Intercepted. Would open /tmp/pytest-of-nponcian/pytest-13/test_add_hosts_entry0/dummy_etc_hosts instead of /etc/hosts
Successfully made entry in /etc/hosts file:
1.2.3.4 SVC_NAME
1 passed in 0.05s
If you're interested, you can display the temporary dummy file too to see the result of the process:
$ cat /tmp/pytest-of-nponcian/pytest-13/test_add_hosts_entry0/dummy_etc_hosts
some text
here
and the last!
1.2.3.4 SVC_NAME
Solution 2
Mock open as well as the .write operation. Once mocked, see all the calls to the mocked .write via call_args_list. This isn't recommended as it would feel like we are writing a change-detector test which is tightly coupled to how the source code was implemented line by line rather than checking the behavior.
I am trying to open a file read it's content and write to it by using the contents that were read earlier. I am opening the file in 'a+' mode. I can't use 'r+' mode since it won't create a file if it doesn't exist.
a+ will put the pointer in the end of the file.
You can save it with tell() for later writing.
Then use seek(0,0) to return to file beginning for reading.
tell()
seek()
Default open
Using the default a(+) option, it is not possible, as provided in the documentation:
''mode is an optional string that specifies the mode in which the file
is opened. It defaults to 'r' which means open for reading in text
mode. Other common values are 'w' for writing (truncating the file if
it already exists), 'x' for creating and writing to a new file, and
'a' for appending (which on some Unix systems, means that all writes
append to the end of the file regardless of the current seek position).''
Alternative
Using the default open, this is not possible.However we can of course create our own file handler, that will create a file in r and r+ mode when it doesn't exists.
A minimal working example that works exactly like open(filename, 'r+', *args, **kwargs), would be:
import os
class FileHandler:
def __init__(self, filename, mode='r', buffering=None, encoding=None, errors=None, newline=None, closefd=True):
self.filename = filename
self.mode = mode
self.kwargs = dict(buffering=buffering, encoding=encoding, errors=errors, newline=newline, closefd=closefd)
if self.kwargs['buffering'] is None:
del self.kwargs['buffering']
def __enter__(self):
if self.mode.startswith('r') and not os.path.exists(self.filename):
with open(self.filename, 'w'): pass
self.file = open(self.filename, self.mode, **self.kwargs)
return self.file
def __exit__(self, exc_type, exc_val, exc_tb):
self.file.close()
Now when you use the following code:
with FileHandler("new file.txt", "r+") as file:
file.write("First line\n")
file.write("Second line\n")
file.seek(0, 0)
file.write("Third line\n")
It will generate a new file new file.txt, when it doesn't exists, with the context:
Third line
Second line
If you would use the open you will receive a FileNotFoundError, if the file doesn't exists.
Notes
I am only creating a new file when the mode starts with an r, all other files are handled as would be by the normal open function.
For some reason passing buffering=None, directly to the open function crashes it with an TypeError: an integer is required (got type NoneType), therefore I had to remove it from the key word arguments if it was None. Even though it is the default argument according to the documentation (if any one knows why, please tell me)
Edit
The above code didn't handle the following cases:
file = FileHandler("new file.txt", "r+")
file.seek(0, 0)
file.write("Welcome")
file.close()
In order to support all of the open use cases, the above class can be adjusted by using __getattr__ as follows:
import os
class FileHandler:
def __init__(self, filename, mode='r', buffering=None, encoding=None, errors=None, newline=None, closefd=True):
self.filename = filename
self.mode = mode
self.kwargs = dict(buffering=buffering, encoding=encoding, errors=errors, newline=newline, closefd=closefd)
if self.kwargs['buffering'] is None:
del self.kwargs['buffering']
if self.mode.startswith('r') and not os.path.exists(self.filename):
with open(self.filename, 'w'): pass
self.file = open(self.filename, self.mode, **self.kwargs)
def __enter__(self):
return self.file
def __exit__(self, exc_type, exc_val, exc_tb):
self.file.close()
def __getattr__(self, item):
if hasattr(self.file, item):
return getattr(self.file, item)
raise AttributeError(f"{type(self).__name__}, doesn't have the attribute {item!r}")
I've written some python code that needs to read a config file at /etc/myapp/config.conf . I want to write a unit test for what happens if that file isn't there, or contains bad values, the usual stuff. Lets say it looks like this...
""" myapp.py
"""
def readconf()
""" Returns string of values read from file
"""
s = ''
with open('/etc/myapp/config.conf', 'r') as f:
s = f.read()
return s
And then I have other code that parses s for its values.
Can I, through some magic Python functionality, make any calls that readconf makes to open redirect to custom locations that I set as part of my test environment?
Example would be:
main.py
def _open_file(path):
with open(path, 'r') as f:
return f.read()
def foo():
return _open_file("/sys/conf")
test.py
from unittest.mock import patch
from main import foo
def test_when_file_not_found():
with patch('main._open_file') as mopen_file:
# Setup mock to raise the error u want
mopen_file.side_effect = FileNotFoundError()
# Run actual function
result = foo()
# Assert if result is expected
assert result == "Sorry, missing file"
Instead of hard-coding the config file, you can externalize it or parameterize it. There are 2 ways to do it:
Environment variables: Use a $CONFIG environment variable that contains the location of the config file. You can run the test with an environment variable that can be set using os.environ['CONFIG'].
CLI params: Initialize the module with commandline params. For tests, you can set sys.argv and let the config property be set by that.
In order to mock just calls to open in your function, while not replacing the call with a helper function, as in Nf4r's answer, you can use a custom patch context manager:
from contextlib import contextmanager
from types import CodeType
#contextmanager
def patch_call(func, call, replacement):
fn_code = func.__code__
try:
func.__code__ = CodeType(
fn_code.co_argcount,
fn_code.co_kwonlyargcount,
fn_code.co_nlocals,
fn_code.co_stacksize,
fn_code.co_flags,
fn_code.co_code,
fn_code.co_consts,
tuple(
replacement if call == name else name
for name in fn_code.co_names
),
fn_code.co_varnames,
fn_code.co_filename,
fn_code.co_name,
fn_code.co_firstlineno,
fn_code.co_lnotab,
fn_code.co_freevars,
fn_code.co_cellvars,
)
yield
finally:
func.__code__ = fn_code
Now you can patch your function:
def patched_open(*args):
raise FileNotFoundError
with patch_call(readconf, "open", "patched_open"):
...
You can use mock to patch a module's instance of the 'open' built-in to redirect to a custom function.
""" myapp.py
"""
def readconf():
s = ''
with open('./config.conf', 'r') as f:
s = f.read()
return s
""" test_myapp.py
"""
import unittest
from unittest import mock
import myapp
def my_open(path, mode):
return open('asdf', mode)
class TestSystem(unittest.TestCase):
#mock.patch('myapp.open', my_open)
def test_config_not_found(self):
try:
result = myapp.readconf()
assert(False)
except FileNotFoundError as e:
assert(True)
if __name__ == '__main__':
unittest.main()
You could also do it with a lambda like this, if you wanted to avoid declaring another function.
#mock.patch('myapp.open', lambda path, mode: open('asdf', mode))
def test_config_not_found(self):
...
I have a simple luigi pipeline.
import luigi
import subprocess
import row_count_test
class TaskOne(luigi.Task):
def requires(self):
return None
def run(self):
output = row_count_test()
if output:
with self.output().open('w') as open_file:
open_file.write('{}'.format(output))
def output(self):
return luigi.LocalTarget('TaskOne.txt')
class TaskTwo(luigi.Task):
def requires(self):
return TaskOne()
def run(self):
subprocess.call('rm *.txt', shell = True)
if __name__ == "__main__":
luigi.run()
I run the following code through command line:
python luigi_demo.py --scheduler-host localhost TaskTwo
I want to be able to save the terminal output to a log file. I also want to be able to add a time stamp to the log file name. I know there's a way to do it through bash commands. Is there a way to do this using luigi? I looked at the luigi.cfg documentation and it wasn't too helpful. A simple example would be greatly appreciated.
You just have to changes the following to your TaskTwo.
import datetime as dt
class TaskTwo(luigi.Task):
date= luigi.DateSecondParameter(default=dt.datetime.now())
def output(self):
# Here you create a file with your date in it.
return luigi.LocalTarget('path/to/your/log/file%s.txt' % self.date)
def requires(self):
return TaskOne()
def run(self):
self.output().open('w') as f:
subprocess.call('rm *.txt', shell = True,stdout=f)
Also, on a side note if you want to delete the file created in the Taskone then you can remove all the code in the run() then just add self.input().remove()
class TaskTwo(luigi.Task):
date= luigi.DateSecondParameter(default=dt.datetime.now())
def output():
return luigi.LocalTarget('path/to/your/log/file%s.txt' % self.date)
def requires(self):
return TaskOne()
def run(self):
# this should remove the file created in the Task one.
self.input().remove()
I need to generate a temporary file to replace a system file with new contents.
I have the code, and it works, but I was thinking if there is some module that automatically does that, along the lines of:
with tempfile.NamedTemporaryFileFor('/etc/systemfile', delete=False) as fp:
...
This would create a temporary file with the same permissions and owner of the original file and in the same folder. Then I would write my new contents and atomically replace the original systemfile with the new one. Heck, the context manager could do the replacement for me when the file is closed!
My current code is:
with tempfile.NamedTemporaryFile(delete=False, dir='/etc') as fp:
tempfilename = fp.name
fp.write('my new contents')
orig_stat = os.stat('/etc/systemfile')
os.chown(tempfilename, orig_stat.st_uid, orig_stat.st_gid)
os.chmod(tempfilename, orig_stat.st_mode)
os.replace(tempfilename, '/etc/systemfile')
There is no such context manager in tempfile but it's not too difficult to write your own context manager:
class NamedTemporaryFileFor(object):
def __init__(self, orig_path, delete=True, dir=None):
self.orig_path = orig_path
self.named_temp_file = tempfile.NamedTemporaryFile(delete=delete, dir=dir)
def __enter__(self):
self.named_temp_file.__enter__()
return self
def write(self, *args, **kwargs):
return self.named_temp_file.write(*args, **kwargs)
def __exit__(self, exc, value, tb):
orig_stat = os.stat(self.orig_path)
os.chown(self.named_temp_file.name, orig_stat.st_uid, orig_stat.st_gid)
os.chmod(self.named_temp_file.name, orig_stat.st_mode)
self.named_temp_file.__exit__(exc, value, tb)
if __name__ == "__main__":
with NamedTemporaryFileFor(sys.argv[1], delete=False, dir="/etc") as fp:
f.write(b'my new content')
(Note that I've had to pass a byte string to the write method to make the example work)