I'm not quite sure how to implement a context manager for a custom class. Basically it's just accepting a file name in __init__, opening the file in __enter__ and closing it in __exit__.
E.g., like:
class BlaFile:
def __init__(self, file_name):
self.file_name = file_name
def __enter__(self):
self.file = open(self.file_name, "rb")
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.file.close()
def do_stuff_with_file():
# This will fail when I'm using this class without 'with'.
return self.file.read(1)
However, when I'm not going to instantiate / use the class with the 'with' statement, I will not be able to use any functions of it which access the file, as __enter__ has never been called; and thus the file is not open.
Maybe I'm too much oriented towards C#'s using keyword here; but shouldn't I be able to correctly use an instance of the class even when I'm not using it with the with keyword? Right now I'm forced to use it with it - is that the typical usage in Python?
If you want the file to be opened regardless of whether or not it is used in a with, you need to open the file in your __init__ method. Since the user might forget to close the file, I would suggest that you register the closing to happen at exit:
from atexit import register
class BlaFile:
def __init__(self, file_name):
self.file_name = file_name
self.file = open(file_name, 'rb')
register(self.file.close)
def __enter__(self):
pass
def __exit__(self, exc_type, exc_val, exc_tb):
self.file.close()
def do_stuff_with_file(self):
return self.file.read(1)
You need to use open()
to open with reading you need to change to:
class BlaFile:
def __init__(self, file_name):
self.file_name = file_name
self.file = open(self.file_name, "r+b")
def close(self):
self.file.close()
def do_stuff_with_file(self):
return self.file.read(1)
That should do the trick.
Related
The problem is the lock. The multiprocessing needs a lock but not for the single process. For example, consider the following code:
Class Test():
def __init__(self, rlock = None):
self.tlock = rlock
def do_test(self, invalue):
with self.tlock:
return invalue + 1
For the multiprocessing, I need to use the tlock but when I use the class for single process, I don't need it. So the line with self.tlock doesn't make sense for a single process execution.
My immediate thought is to write in the following way:
def do_test(self, invalue):
if self.tlock is not None:
with self.tlock:
return invalue + 1
else:
return invalue + 1
But this looks so awkward as I would have handful of methods of this pattern inside the class.
Is there any elegant and efficient way to write the code for the code reuse?
You can create a dummy class that can support context managers, and use that instead of storing the lock if it is None (i.e, no multiprocessing is involved):
class DummyLock:
def __enter__(self):
pass
def __exit__(self, exc_type, exc_val, exc_tb):
pass
class Test:
def __init__(self, rlock=None):
self.tlock = rlock
if self.tlock is None:
self.tlock = DummyLock()
All other methods in the class will not need to be changed unless they are accessing self.rlock specific methods (like self.rlock.acquire()) instead of using context managers (with self.rlock:).
I have a collection of ever more specialized classes which correspond to collections of the same kind of data (temperature, density, etc) but for different drifts, for example, one subclass has dimensions (nx, ny) and a different suclass has dimensions (ncv), and I want to reflect that in the docstrings, for having a better documentation using Sphinx.
After reading many very useful threads here in Stack Overflow, I have arrived to this model:
import numpy as np
from functools import wraps
def class_decorator(cls):
import ipdb; ipdb.set_trace()
clsdict = {}
mro = cls.mro()
mro.reverse()
for tmp in mro[1:]: ##Ignore object class parent.
clsdict.update(tmp.__dict__)
for name, method in clsdict.items():
if hasattr(method, '__og_doc__'):
try:
method.__doc__ = method.__og_doc__.format(**clsdict)
except:
pass
else:
try:
method.__og_doc__ = method.__doc__
method.__doc__ = method.__doc__.format(**clsdict)
except:
pass
return cls
def mark_documentation(fn):
if not hasattr(fn, '__og_doc__'):
try:
fn.__og_doc__ = fn.__doc__
except:
pass
#wraps(fn)
def wrapped(*args, **kwargs):
return fn(*args, **kwargs)
return wrapped
def documented_property(fn):
if not hasattr(fn, '__og_doc__'):
try:
fn.__og_doc__ = fn.__doc__
except:
pass
#wraps(fn)
def wrapped(*args, **kwargs):
return fn(*args, **kwargs)
prp= property(wrapped)
prp.__og_doc__ = fn.__og_doc__
return prp
#class_decorator
class Base(object):
_GRID_DIM = 'nx, ny'
_TYPE = 'BaseData'
def __init__(self, name):
self.name = name
def shape(self):
""" This docstring contains the type '{_TYPE}' of class."""
print('Simple')
def operation(self, a, b, oper=np.sum, **kwargs):
""" Test for functions with args and kwargs in {_TYPE}"""
return oper([a,b])
#classmethod
def help(cls, var):
try:
print(get(cls, var).__doc__)
except:
print("No docstring yet.")
#class_decorator
class Advanced(Base):
_GRID_DIM = 'ncv'
_TYPE = 'AdvancedData'
def __init__(self,name):
super().__init__(name)
#property
#mark_documentation
# #documented_property
def arkansas(self):
"""({_GRID_DIM}, ns): Size of Arkansaw."""
return 'Yeah'
I am aiming to get the correctly formatted docstring when I call the help method or I use Sphinx, so that:
> adv = Advanced('ADV')
> adv.help("arkansas")
(ncv, ns): Size of Arkansaw.
> adv.help("operation")
Test for functions with args and kwargs in AdvancedData
I have managed to make it work so far, except for properties, because I assigned __og_doc__ to the function, but the property does not have that attribute. My last attempt at monkeypatching this, documented_property, fails because property is inmutable (as expected), and I cannot come up with any way to avoid this roadblock.
Is there any way around this problem?
I'm trying to write a unit test class in python but feel like I'm missing something fundamental as it's not doing what I would expect. Here is my class:
from unittest import TestCase
class MyTestClass(TestCase):
def __init__(self):
self.file_name = None
def setUp(self):
self.file_name = 'give this file a name'
return self.file_name
def test_a_file_name(self):
assert self.file_name == 'give this file a name', 'fail'
tester = MyTestClass()
tester.setUp()
tester.test_a_file_name()
I would expect when running this that the test would pass but I'm getting a __init__() takes 1 positional argument but 2 were given error and I can't see why?
When running unittest.main your class that inherits from TestCase gets handed the test method to call. As such you need to allow your class to be handed that argument and pass it on to the parent class __init__.
from unittest import TestCase, main
class MyTestClass(TestCase):
# accept arbitrary positional and keyword arguments
def __init__(self, *args, **kwargs):
self.file_name = None
# pass them on to the parent
super().__init__(*args, **kwargs)
def setUp(self):
self.file_name = 'give this file a name'
return self.file_name
def test_a_file_name(self):
assert self.file_name == 'give this file a name', 'fail'
if __name__ == '__main__':
main()
As you noticed, you also don't need to handle instantiation and method calling. unittest.main() will do that for you.
In the future, if you ever get an error with arguments, a helpful debugging tip is throwing in an *args, **kwargs and printing them to see what is being handed that you're not handling.
I am trying to open a file read it's content and write to it by using the contents that were read earlier. I am opening the file in 'a+' mode. I can't use 'r+' mode since it won't create a file if it doesn't exist.
a+ will put the pointer in the end of the file.
You can save it with tell() for later writing.
Then use seek(0,0) to return to file beginning for reading.
tell()
seek()
Default open
Using the default a(+) option, it is not possible, as provided in the documentation:
''mode is an optional string that specifies the mode in which the file
is opened. It defaults to 'r' which means open for reading in text
mode. Other common values are 'w' for writing (truncating the file if
it already exists), 'x' for creating and writing to a new file, and
'a' for appending (which on some Unix systems, means that all writes
append to the end of the file regardless of the current seek position).''
Alternative
Using the default open, this is not possible.However we can of course create our own file handler, that will create a file in r and r+ mode when it doesn't exists.
A minimal working example that works exactly like open(filename, 'r+', *args, **kwargs), would be:
import os
class FileHandler:
def __init__(self, filename, mode='r', buffering=None, encoding=None, errors=None, newline=None, closefd=True):
self.filename = filename
self.mode = mode
self.kwargs = dict(buffering=buffering, encoding=encoding, errors=errors, newline=newline, closefd=closefd)
if self.kwargs['buffering'] is None:
del self.kwargs['buffering']
def __enter__(self):
if self.mode.startswith('r') and not os.path.exists(self.filename):
with open(self.filename, 'w'): pass
self.file = open(self.filename, self.mode, **self.kwargs)
return self.file
def __exit__(self, exc_type, exc_val, exc_tb):
self.file.close()
Now when you use the following code:
with FileHandler("new file.txt", "r+") as file:
file.write("First line\n")
file.write("Second line\n")
file.seek(0, 0)
file.write("Third line\n")
It will generate a new file new file.txt, when it doesn't exists, with the context:
Third line
Second line
If you would use the open you will receive a FileNotFoundError, if the file doesn't exists.
Notes
I am only creating a new file when the mode starts with an r, all other files are handled as would be by the normal open function.
For some reason passing buffering=None, directly to the open function crashes it with an TypeError: an integer is required (got type NoneType), therefore I had to remove it from the key word arguments if it was None. Even though it is the default argument according to the documentation (if any one knows why, please tell me)
Edit
The above code didn't handle the following cases:
file = FileHandler("new file.txt", "r+")
file.seek(0, 0)
file.write("Welcome")
file.close()
In order to support all of the open use cases, the above class can be adjusted by using __getattr__ as follows:
import os
class FileHandler:
def __init__(self, filename, mode='r', buffering=None, encoding=None, errors=None, newline=None, closefd=True):
self.filename = filename
self.mode = mode
self.kwargs = dict(buffering=buffering, encoding=encoding, errors=errors, newline=newline, closefd=closefd)
if self.kwargs['buffering'] is None:
del self.kwargs['buffering']
if self.mode.startswith('r') and not os.path.exists(self.filename):
with open(self.filename, 'w'): pass
self.file = open(self.filename, self.mode, **self.kwargs)
def __enter__(self):
return self.file
def __exit__(self, exc_type, exc_val, exc_tb):
self.file.close()
def __getattr__(self, item):
if hasattr(self.file, item):
return getattr(self.file, item)
raise AttributeError(f"{type(self).__name__}, doesn't have the attribute {item!r}")
I found the default implementation of python's multiprocessing.Queue lacking, in that it's not iterable like any other collection. So I went about the effort of creating a 'subclass' of it, adding the feature in. As you can see from the code below, it's not a proper subclass, as multiprocess.Queue isn't a direct class itself, but a factory function, and the real underlying class is multiprocess.queues.Queue. I don't have the understanding nor effort to expend necessary to go about mimicking the factory function just so I can inherit from the class properly, so I simply had the new class create it's own instance from the factory and treat it as the superclass. Here is the code;
from multiprocessing import Queue, Value, Lock
import queue
class QueueClosed(Exception):
pass
class IterableQueue:
def __init__(self, maxsize=0):
self.closed = Value('b', False)
self.close_lock = Lock()
self.queue = Queue(maxsize)
def close(self):
with self.close_lock:
self.closed.value = True
self.queue.close()
def put(self, elem, block=True, timeout=None):
with self.close_lock:
if self.closed.value:
raise QueueClosed()
else:
self.queue.put(elem, block, timeout)
def put_nowait(self, elem):
self.put(elem, False)
def get(self, block=True):
if not block:
return self.queue.get_nowait()
elif self.closed.value:
try:
return self.queue.get_nowait()
except queue.Empty:
return None
else:
val = None
while not self.closed.value:
try:
val = self.queue.get_nowait()
break
except queue.Empty:
pass
return val
def get_nowait(self):
return self.queue.get_nowait()
def join_thread(self):
return self.queue.join_thread()
def __iter__(self):
return self
def __next__(self):
val = self.get()
if val == None:
raise StopIteration()
else:
return val
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
This allows me to instantiate an IterableQueue object just like a normal multiprocessing.Queue, put elements into it like normal, and then inside child consumers, simply loop over it like so;
from iterable_queue import IterableQueue
from multiprocessing import Process, cpu_count
import os
def fib(n):
if n < 2:
return n
return fib(n-1) + fib(n-2)
def consumer(queue):
print(f"[{os.getpid()}] Consuming")
for i in queue:
print(f"[{os.getpid()}] < {i}")
n = fib(i)
print(f"[{os.getpid()}] {i} > {n}")
print(f"[{os.getpid()}] Closing")
def producer():
print("Enqueueing")
with IterableQueue() as queue:
procs = [Process(target=consumer, args=(queue,)) for _ in range(cpu_count())]
[p.start() for p in procs]
[queue.put(i) for i in range(36)]
print("Finished")
if __name__ == "__main__":
producer()
and it works almost seamlessly; the consumers exit the loop once the queue has been closed, but only after exhausting all remaining elements. However, I was unsatisfied with the lack of inherited methods. In an attempt to mimic actual inheritance behavior, I tried adding the following meta function call to the class;
def __getattr__(self, name):
if name in self.__dict__:
return self.__dict__[name]
else:
return self.queue.__getattr__[name]
However, this fails when instances of the IterableQueue class are manipulated inside child multiprocessing.Process threads, as the class's __dict__ property is not preserved within them. I attempted to remedy this in a hacky manner by replacing the class's default __dict__ with a multiprocessing.Manager().dict(), like so;
def __init__(self, maxsize=0):
self.closed = Value('b', False)
self.close_lock = Lock()
self.queue = Queue(maxsize)
self.__dict__ = Manager().dict(self.__dict__)
However on doing so, I received an error stating RuntimeError: Synchronized objects should only be shared between processes through inheritance. So my question is, how should I go about inheriting from the Queue class properly such that the subclass has inherited access to all of it's properties? In addition, while the queue is empty but not closed, the consumers all sit in a busy loop instead of a true IO block, taking up valuable cpu resources. If you have any suggestions on concurrency and race condition issues I might run into with this code, or how I might solve the busy loop issue, I'd be willing to take suggestions therein as well.
Based on code provided by MisterMiyagi, I created this general purpose IterableQueue class which can accept arbitrary input, blocks properly, and does not hang on queue close;
from multiprocessing.queues import Queue
from multiprocessing import get_context
class QueueClosed(Exception):
pass
class IterableQueue(Queue):
def __init__(self, maxsize=0, *, ctx=None):
super().__init__(
maxsize=maxsize,
ctx=ctx if ctx is not None else get_context()
)
def close(self):
super().put((None, False))
super().close()
def __iter__(self):
return self
def __next__(self):
try:
return self.get()
except QueueClosed:
raise StopIteration
def get(self, *args, **kwargs):
result, is_open = super().get(*args, **kwargs)
if not is_open:
super().put((None, False))
raise QueueClosed
return result
def put(self, val, *args, **kwargs):
super().put((val, True), *args, **kwargs)
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
The multiprocess.Queue wrapper only serves to use the default context.
def Queue(self, maxsize=0):
'''Returns a queue object'''
from .queues import Queue
return Queue(maxsize, ctx=self.get_context())
When inheriting, you can replicate this in the __init__ method. This allows you to inherit the entire Queue behaviour. You only need to add the iterator methods:
from multiprocessing.queues import Queue
from multiprocessing import get_context
class IterableQueue(Queue):
"""
``multiprocessing.Queue`` that can be iterated to ``get`` values
:param sentinel: signal that no more items will be received
"""
def __init__(self, maxsize=0, *, ctx=None, sentinel=None):
self.sentinel = sentinel
super().__init__(
maxsize=maxsize,
ctx=ctx if ctx is not None else get_context()
)
def close(self):
self.put(self.sentinel)
# wait until buffer is flushed...
while self._buffer:
time.sleep(0.01)
# before shutting down the sender
super().close()
def __iter__(self):
return self
def __next__(self):
result = self.get()
if result == self.sentinel:
# re-queue sentinel for other listeners
self.put(result)
raise StopIteration
return result
Note that the sentinel to indicate end-of-queue is compared by equality, because identity is not preserved across processes. The often-used queue.Queue sentinel object() does not work properly with this.