I would like to reliably call the __del__ method of an object immediately when it gets deleted. I am aware that this has been asked for python 2, but the documentation of python 3 claims that the __del__ method is called when the reference count reaches 0. However, this does not seem to be the case with my program
import time
class MyClass:
object_list = []
def __init__(self):
MyClass.object_list.append(self)
print("object initialized")
def __del__(self):
print("executing cleanup steps")
MyClass.object_list.remove(self)
a = MyClass()
b = MyClass()
c = MyClass()
print(len(MyClass.object_list))
del b
time.sleep(1)
print("sleep finished")
print(len(MyClass.object_list))
Try it online
What I want to achieve is to remove b from the object_list immediately when it is destroyed and also make the variable b inaccessible. So simply calling b.__del__() is not enough, I would need to also render b inaccessible.
Note: running first b.__del__() and then del b will lead to complaints when exiting the python interpreter.
Interestingly enough, a very similar program seems to call b.__del__() immediately when del b is executed.
import time
class MyClass:
counter = 0
def __init__(self, value):
self.value = value
print("object initialized")
MyClass.counter+=1
def __del__(self):
print("executing cleanup steps")
MyClass.counter-=1
a = MyClass(5)
b = MyClass(6)
c = MyClass(7)
print(MyClass.counter)
del b
time.sleep(1)
print("sleep finished")
print(MyClass.counter)
Try it online
EDIT
#Pranav Hosangadi pointed out that there is still a reference to the object in the object_list. What a shameful oversight of mine. Now I came up with a workaround by calling first b.__del__() and then del b, but it is a two step process. I would like to have it in one step, if possible.
import time
class MyClass:
object_list = []
def __init__(self, value):
self.value = value
MyClass.object_list.append(self)
print("object initialized")
def __del__(self):
print("executing cleanup steps")
try:
MyClass.object_list.remove(self)
except:
pass
a = MyClass(1)
b = MyClass(2)
c = MyClass(3)
print(len(MyClass.object_list))
b.__del__()
del b
time.sleep(1)
print("sleep finished")
print(len(MyClass.object_list))
Related
I have a long running task, which for example's sake I have made an infinite while loop:
def long_task(parent, progress_callback):
top = 100000
x = 0
while True:
if x < top:
if not parent.stop:
progress_callback.emit(x)
x += 1
else:
break
else:
x = 0
progress_callback.emit(x)
x += 1
I have a Worker class that subclasses QRunnable, and then I can override the run() method with whatever function is passed to the Worker.
class ThreadWorker(QtCore.QRunnable):
def __init__(self, fn, *args, **kwargs):
super(ThreadWorker, self).__init__()
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = ThreadWorkerSignals()
self.kwargs['progress_callback'] = self.signals.progress
self.running = False
#QtCore.pyqtSlot()
def run(self):
self.running = True
try:
result = self.fn(*self.args, **self.kwargs)
except:
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit((exctype, value, traceback.format_exc()))
else:
self.signals.result.emit(result) # Return the result of the processing
finally:
self.signals.finished.emit() # Done
I create two instances of Worker within my MainWindow, and pass the same long-running task to each worker. Both workers are added to my MainWindow's QThreadPool and then I call start(worker) on each to begin the worker's run() method. I now have two threads running the infinite loop:
class MainWindow(QtWidgets.QMainWindow):
def __init__(self):
super().__init__()
## NOT SHOWING THE REST OF INIT CODE
def create_workers(self):
self.worker1 = ThreadWorker(self.long_task, parent=self)
self.worker1.signals.progress.connect(lambda x: self.long_label_1.setText(str(x)))
self.worker2 = ThreadWorker(self.long_task, parent=self)
self.worker2.signals.progress.connect(lambda x: self.long_label_2.setText(str(x)))
self.threadpool.start(self.worker1)
self.threadpool.start(self.worker2)
self.stop = False
Please note the self.stop attribute above - this also belongs to the MainWindow class.
All I want to do is break the loop (interrupt the run() method of a worker) when I press a button.
As you can see, I am referencing parent.stop during every iteration of the worker's while loop. Right now, if I press my button, MainWindow's stop attribute turns True and the loop breaks when the worker class sees this change.
def stop_tasks(self):
self.stop = True
This works fine and accomplishes my goal, but I am wondering if this is dangerous and if there is a better way to do this? I only ask because it seems risky to reference an outside class attribute from within a separate running thread, and I don't know what could go wrong.
I'd like to use named constants whereever possible instead of providing literal values or longish function signatures with a lot of boolean args.
Therefore i like pythons enum.Flag or enum.Enum.
More precisely, I would like to pass an argument to a function which holds a bit combination of enum.Flags. And i would like to avoid writing module.TheFlags.flagX for every set flag I would like to pass to the function. The flags should replace the boolean args.
I came up with following code:
import enum
class AvailableFlags(enum.Flag):
flag1 = enum.auto()
flag2 = enum.auto()
class FuncFlags:
def __init__(self):
self._flags = AvailableFlags(0)
#property
def flag1(self):
self._flags |= AvailableFlags.flag1
return self
#property
def flag2(self):
self._flags |= AvailableFlags.flag2
return self
def __str__(self):
return str(self._flags.value)
def func(setup_flags: FuncFlags):
print(setup_flags)
if __name__ == "__main__":
func(FuncFlags().flag1)
func(FuncFlags().flag2)
func(FuncFlags().flag1.flag2)
func(FuncFlags())
It creates instances of FuncFlags and then mis-uses the properties to set single flags returning the changed object itself.
However, one would expect that the property does NOT change object state.
Therefore, this is obviously not a clean solution despite that it works, though.
So, my question is, how this can be implemented in a clean, reusable way?
I'm not really clear on what you are trying to accomplish, but perhaps this helps?
import enum
class AvailableFlags(enum.Flag):
flag1 = enum.auto()
flag2 = enum.auto()
flag1, flag2 = AvailableFlag
def func(setup_flags: AvailableFlags):
print(setup_flags)
if __name__ == "__main__":
func(flag1)
func(flag2)
func(flag1|flag2)
func()
Meanwhile, I found an answer by adding another level of indirection.
I want to share it here if it is of interest for someone else.
Object state is maintained as every invokation of a flag creates a new instance from the current instance by setting an additional flag.
If we attempt to access an undefined flag an exception is raised (not shown).
import enum
class AvailableFlags(enum.Flag):
flag1 = enum.auto()
flag2 = enum.auto()
class FlagHelper:
def __init__(self, cls, value = 0):
self._cls = cls
self._flags = self._cls(value)
def __getattr__(self, item):
if item in self._cls.__members__:
return self.__class__(self._flags | getattr(self._cls, item))
getattr(self._cls, item) # Let attribute error pass through
def __str__(self):
return str(self._flags.value)
class FuncFlags(FlagHelper):
def __init__(self, value = 0):
super().__init__(AvailableFlags, value)
def func(setup_flags: FuncFlags):
print(setup_flags)
if __name__ == "__main__":
ff = FuncFlags()
func(ff.flag1)
func(ff.flag2)
func(ff.flag1.flag2)
func(ff)
Output:
1
2
3
0
Recently while getting my hands on with Python Class concepts, I came upon this observation and was not able to understand.
When I try and create instance out of the below class interactively(Python console), I also get the Finding __len__ line in output.
class MyClass(object):
counter = 0
data = 'Class Variable'
def __init__(self):
self.counter += 1
self.value = -1
def __str__(self):
return "Instance {} is the {} instance".format(self.__class__.__name__, self.counter)
def __getattr__(self, item):
print(f'Finding {item}')
return self.__dict__.get(item, f'Attr {item} not available, {self.__dict__}')
def __setattr__(self, key, value):
if key not in self.__dict__:
self.__dict__[key] = value
def __delattr__(self, item):
print(f'Deleting attr: {item}')
if item in self.__dict__:
del self.__dict__[item]
else:
print(f'Cannot find {item} in {self.__dict__}')
if __name__ == '__main__':
inst = MyClass()
print(inst.id)
But running it as a top level module, doesn't add this additional line in output.
Can someone help me understand, why Finding __len__ output would be displayed interactively.
Below is an interactive output,
import WS1
x = WS1.MyClass()
Finding __len__
x.name = 'Yathin'
Finding __len__
When I do not crate object for CP class, the operations are not captured. I am referring to the code below, Can somebody help me understand why we need obj creation in this case
from abc import ABC, abstractmethod
class P(ABC):
def __init__(self):
super().__init__()
self._pre_map = {}
self._pre_order = []
def set_pre(self, tag_value):
index = len(self._pre_map)
print(index)
self._pre_map[index] = tag_value
self._pre_order.append(index)
def execute(self):
pass
class CP(P):
def __init__(self):
super().__init__()
def execute(self):
self.prnt()
def prnt(self):
print (self._pre_map)
print (self._pre_order)
#Working
print("\n++++++++ working")
obj = CP()
obj.set_pre("test string added")
obj.execute()
#Not Working
print("\n+++++++ not working")
CP().set_pre("test string added")
CP().execute()
It produces,
++++++++working
0
{0: 'test string added'}
[0]
+++++++not working
0
{}
[]
When you call the class the second time with CP.execute(), you have created a completely new instance of the CP class. It is not going to have the text string you specified.
If you actually wanted it to print the values like the working one you can make the functions return self after each call in the P class. If you did that you could do something like this.
from abc import ABC, abstractmethod
class P(ABC):
def __init__(self):
super().__init__()
self._pre_map = {}
self._pre_order = []
def set_pre(self, tag_value):
index = len(self._pre_map)
print(index)
self._pre_map[index] = tag_value
self._pre_order.append(index)
##need to return self here
return self
def execute(self):
pass
class CP(P):
def __init__(self):
super().__init__()
def execute(self):
self.prnt()
def prnt(self):
print (self._pre_map)
print (self._pre_order)
#Working
print("\n++++++++ working")
obj = CP()
obj.set_pre("test string added")
obj.execute()
#Not Working
print("\n+++++++ not working: but now working after returning self in the P class")
CP().set_pre("test string added").execute()
++++++++ working
0
{0: 'test string added'}
[0]
+++++++ not working: but now working after returning self in the P class
0
{0: 'test string added'}
[0]
This would print the result you want.
The reason for the difference is the fact that in the first one, you are creating an instance, and using that instance the whole way through, whereas in the second one, you are using two different instances of your class.
The two different instances cannot share their attributes, so you are unable to recall what happened. If you really don't want to use a dedicated variable, change your P class to look like this:
class P(ABC):
...
def set_pre(self, tag_value):
index = len(self._pre_map)
print(index)
self._pre_map[index] = tag_value
self._pre_order.append(index)
return self
...
And use CP().set_pre("test string added").execute()
I found the default implementation of python's multiprocessing.Queue lacking, in that it's not iterable like any other collection. So I went about the effort of creating a 'subclass' of it, adding the feature in. As you can see from the code below, it's not a proper subclass, as multiprocess.Queue isn't a direct class itself, but a factory function, and the real underlying class is multiprocess.queues.Queue. I don't have the understanding nor effort to expend necessary to go about mimicking the factory function just so I can inherit from the class properly, so I simply had the new class create it's own instance from the factory and treat it as the superclass. Here is the code;
from multiprocessing import Queue, Value, Lock
import queue
class QueueClosed(Exception):
pass
class IterableQueue:
def __init__(self, maxsize=0):
self.closed = Value('b', False)
self.close_lock = Lock()
self.queue = Queue(maxsize)
def close(self):
with self.close_lock:
self.closed.value = True
self.queue.close()
def put(self, elem, block=True, timeout=None):
with self.close_lock:
if self.closed.value:
raise QueueClosed()
else:
self.queue.put(elem, block, timeout)
def put_nowait(self, elem):
self.put(elem, False)
def get(self, block=True):
if not block:
return self.queue.get_nowait()
elif self.closed.value:
try:
return self.queue.get_nowait()
except queue.Empty:
return None
else:
val = None
while not self.closed.value:
try:
val = self.queue.get_nowait()
break
except queue.Empty:
pass
return val
def get_nowait(self):
return self.queue.get_nowait()
def join_thread(self):
return self.queue.join_thread()
def __iter__(self):
return self
def __next__(self):
val = self.get()
if val == None:
raise StopIteration()
else:
return val
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
This allows me to instantiate an IterableQueue object just like a normal multiprocessing.Queue, put elements into it like normal, and then inside child consumers, simply loop over it like so;
from iterable_queue import IterableQueue
from multiprocessing import Process, cpu_count
import os
def fib(n):
if n < 2:
return n
return fib(n-1) + fib(n-2)
def consumer(queue):
print(f"[{os.getpid()}] Consuming")
for i in queue:
print(f"[{os.getpid()}] < {i}")
n = fib(i)
print(f"[{os.getpid()}] {i} > {n}")
print(f"[{os.getpid()}] Closing")
def producer():
print("Enqueueing")
with IterableQueue() as queue:
procs = [Process(target=consumer, args=(queue,)) for _ in range(cpu_count())]
[p.start() for p in procs]
[queue.put(i) for i in range(36)]
print("Finished")
if __name__ == "__main__":
producer()
and it works almost seamlessly; the consumers exit the loop once the queue has been closed, but only after exhausting all remaining elements. However, I was unsatisfied with the lack of inherited methods. In an attempt to mimic actual inheritance behavior, I tried adding the following meta function call to the class;
def __getattr__(self, name):
if name in self.__dict__:
return self.__dict__[name]
else:
return self.queue.__getattr__[name]
However, this fails when instances of the IterableQueue class are manipulated inside child multiprocessing.Process threads, as the class's __dict__ property is not preserved within them. I attempted to remedy this in a hacky manner by replacing the class's default __dict__ with a multiprocessing.Manager().dict(), like so;
def __init__(self, maxsize=0):
self.closed = Value('b', False)
self.close_lock = Lock()
self.queue = Queue(maxsize)
self.__dict__ = Manager().dict(self.__dict__)
However on doing so, I received an error stating RuntimeError: Synchronized objects should only be shared between processes through inheritance. So my question is, how should I go about inheriting from the Queue class properly such that the subclass has inherited access to all of it's properties? In addition, while the queue is empty but not closed, the consumers all sit in a busy loop instead of a true IO block, taking up valuable cpu resources. If you have any suggestions on concurrency and race condition issues I might run into with this code, or how I might solve the busy loop issue, I'd be willing to take suggestions therein as well.
Based on code provided by MisterMiyagi, I created this general purpose IterableQueue class which can accept arbitrary input, blocks properly, and does not hang on queue close;
from multiprocessing.queues import Queue
from multiprocessing import get_context
class QueueClosed(Exception):
pass
class IterableQueue(Queue):
def __init__(self, maxsize=0, *, ctx=None):
super().__init__(
maxsize=maxsize,
ctx=ctx if ctx is not None else get_context()
)
def close(self):
super().put((None, False))
super().close()
def __iter__(self):
return self
def __next__(self):
try:
return self.get()
except QueueClosed:
raise StopIteration
def get(self, *args, **kwargs):
result, is_open = super().get(*args, **kwargs)
if not is_open:
super().put((None, False))
raise QueueClosed
return result
def put(self, val, *args, **kwargs):
super().put((val, True), *args, **kwargs)
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
The multiprocess.Queue wrapper only serves to use the default context.
def Queue(self, maxsize=0):
'''Returns a queue object'''
from .queues import Queue
return Queue(maxsize, ctx=self.get_context())
When inheriting, you can replicate this in the __init__ method. This allows you to inherit the entire Queue behaviour. You only need to add the iterator methods:
from multiprocessing.queues import Queue
from multiprocessing import get_context
class IterableQueue(Queue):
"""
``multiprocessing.Queue`` that can be iterated to ``get`` values
:param sentinel: signal that no more items will be received
"""
def __init__(self, maxsize=0, *, ctx=None, sentinel=None):
self.sentinel = sentinel
super().__init__(
maxsize=maxsize,
ctx=ctx if ctx is not None else get_context()
)
def close(self):
self.put(self.sentinel)
# wait until buffer is flushed...
while self._buffer:
time.sleep(0.01)
# before shutting down the sender
super().close()
def __iter__(self):
return self
def __next__(self):
result = self.get()
if result == self.sentinel:
# re-queue sentinel for other listeners
self.put(result)
raise StopIteration
return result
Note that the sentinel to indicate end-of-queue is compared by equality, because identity is not preserved across processes. The often-used queue.Queue sentinel object() does not work properly with this.