I have the following function which is called from a function of a class. I would like to call this as a non blocking thread or daemon process. And I want it to be stopped gracefully taking care of any IO locks and DB locks (which happen inside foo()).
def worker(fnames):
while True:
for f in fnames:
while(not os.path.isfile(f)):
time.sleep(SLEEP_INTERVAL)
while os.stat(f).st_size < 1000000:
time.sleep(SLEEP_INTERVAL)
file= open(f, 'rb')
#Do Something
foo()
os.remove(f)
I looked in to first answer on this link and modified the code. Do I create a driver function for the above function, run the driver function from a new thread, and pass the _stop event flag from the below thread to it? Or is there a better way?
class StoppableThread(threading.Thread):
def __init__(self, target, timeout):
super(StoppableThread, self).__init__()
self._target = target
self._timeout = timeout
self._stop = threading.Event()
def run(self):
while not self.stopped():
self._stop.wait(self._timeout) # instead of sleeping
if self.stopped():
continue
self._target()
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
EDIT:
I managed to think of the following solution, but I find it very nasty. Any help is appreciated:
def run(self):
while True:
for f in fnames:
while not self._stop.isSet():
while(not os.path.isfile(f)):
self._stop.wait(SLEEP_INTERVAL)
if self._stop.isSet():
break;
while os.stat(f).st_size < 1000000:
self._stop.wait(SLEEP_INTERVAL)
if self._stop.isSet():
break;
if self._stop.isSet():
continue;
file= open(f, 'rb')
#Do Something
foo()
os.remove(f)
if self._stop.isSet():
break;
if self._stop.isSet():
break;
Related
I have a long running task, which for example's sake I have made an infinite while loop:
def long_task(parent, progress_callback):
top = 100000
x = 0
while True:
if x < top:
if not parent.stop:
progress_callback.emit(x)
x += 1
else:
break
else:
x = 0
progress_callback.emit(x)
x += 1
I have a Worker class that subclasses QRunnable, and then I can override the run() method with whatever function is passed to the Worker.
class ThreadWorker(QtCore.QRunnable):
def __init__(self, fn, *args, **kwargs):
super(ThreadWorker, self).__init__()
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = ThreadWorkerSignals()
self.kwargs['progress_callback'] = self.signals.progress
self.running = False
#QtCore.pyqtSlot()
def run(self):
self.running = True
try:
result = self.fn(*self.args, **self.kwargs)
except:
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit((exctype, value, traceback.format_exc()))
else:
self.signals.result.emit(result) # Return the result of the processing
finally:
self.signals.finished.emit() # Done
I create two instances of Worker within my MainWindow, and pass the same long-running task to each worker. Both workers are added to my MainWindow's QThreadPool and then I call start(worker) on each to begin the worker's run() method. I now have two threads running the infinite loop:
class MainWindow(QtWidgets.QMainWindow):
def __init__(self):
super().__init__()
## NOT SHOWING THE REST OF INIT CODE
def create_workers(self):
self.worker1 = ThreadWorker(self.long_task, parent=self)
self.worker1.signals.progress.connect(lambda x: self.long_label_1.setText(str(x)))
self.worker2 = ThreadWorker(self.long_task, parent=self)
self.worker2.signals.progress.connect(lambda x: self.long_label_2.setText(str(x)))
self.threadpool.start(self.worker1)
self.threadpool.start(self.worker2)
self.stop = False
Please note the self.stop attribute above - this also belongs to the MainWindow class.
All I want to do is break the loop (interrupt the run() method of a worker) when I press a button.
As you can see, I am referencing parent.stop during every iteration of the worker's while loop. Right now, if I press my button, MainWindow's stop attribute turns True and the loop breaks when the worker class sees this change.
def stop_tasks(self):
self.stop = True
This works fine and accomplishes my goal, but I am wondering if this is dangerous and if there is a better way to do this? I only ask because it seems risky to reference an outside class attribute from within a separate running thread, and I don't know what could go wrong.
Since threading.Timer is a subclass of Thread, I would expect that the .join() in this script would cause the code to print "woof" once a second, continually:
import threading
def target_action(arg):
print arg
def start_timer_proc(interval, arg):
timer = threading.Timer(interval, target_action, [arg])
timer.start()
return timer
def main():
timer = start_timer_proc(1.0, "woof")
timer.join()
print("...exiting")
main()
Instead, it prints out "woof" once and then terminates (without any error message). What am I missing?
Here's what I really wanted (based loosely on https://stackoverflow.com/a/12435256/558639):
import threading
class IntervalTimer(threading.Thread):
def __init__(self, target_action, interval, args=[]):
threading.Thread.__init__(self)
self.event = threading.Event()
self.target_action = target_action
self.interval = interval
self.args = args
def start(self):
while not self.event.wait(self.interval):
self.target_action(*self.args)
def target_action(arg):
print arg
def start_timer_proc(interval, arg):
timer = IntervalTimer(target_action, interval, [arg])
timer.start()
return timer
def main():
timer = start_timer_proc(1.0, "woof")
print timer
timer.join()
print("...exiting")
main()
Note that I didn't need to change my target_action() or start_timer_proc() methods, except to instantiate an IntervalTimer rather than a Timer.
I have a code something like this,
import threading
class Mythread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
print('do some processing')
if __name__=='__main__':
while Ture:
val = raw_input('next thread')
t = MyThread()
t.start()
t.join()
The question is how can I carry on with main function without blocking the main because t.join() stop the main until t does not finish?
You should put code in the "code" tag or else it's not really readable.
And you just have to do something like that.
if name == 'main':
#Thread creation
allThreads = []
while True:
val = raw_input('next thread')
newThread = MyThread()
newThread.start()
allThreads.append(newThread)
#You can do something here
#Waiting for all threads to stop
for thread in allThreads:
thread.join()
I modified the following code from first answer on this link.
class StoppableThread(threading.Thread):
"""Thread class with a stop() method. The thread itself has to check
regularly for the stopped() condition."""
def __init__(self, target, timeout):
super(StoppableThread, self).__init__()
self._target = target
self._timeout = timeout
self._stop = threading.Event()
self.awake = threading.Event()
def run(self):
while(not self._stop.isSet()):
self.awake.clear()
time.sleep(self._timeout)
self.awake.set()
self._target()
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
Once I create an instance of this class and set it to daemon process, I would like to terminate it at a later time, when the thread is sleeping, else wait for it to complete the _target() function and then terminate. I am able to handle the latter case by calling stop method. However, I have no idea of terminating it when the _awake event object is set to False. Can someone please help?
Your thread doesn't have to explicitly sleep. It can simply wait for another thread to ask it to stop.
def run(self):
while(not self._stop.isSet()):
self.awake.clear()
self._stop.wait(self._timeout) # instead of sleeping
if self._stop.isSet():
continue
self.awake.set()
self._target()
For this purpose, you don't need the awake event at all. (You might still need it if another thread wants to check its "status". I don't know if that's a requirement you have).
Without awake, your code will be:
class StoppableThread(threading.Thread):
def __init__(self, target, timeout):
super(StoppableThread, self).__init__()
self._target = target
self._timeout = timeout
self._stop = threading.Event()
def run(self):
while not self.stopped():
self._stop.wait(self._timeout) # instead of sleeping
if self.stopped():
continue
self._target()
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
I have a python 3.4.3, postgreSQL 9.4, aiopg-0.7.0. An example of multi-threaded applications, was taken from this site. How to use the pool? The thread hangs when the operation of the select.
import time
import asyncio
import aiopg
import functools
from threading import Thread, current_thread, Event
from concurrent.futures import Future
class B(Thread):
def __init__(self, start_event):
Thread.__init__(self)
self.loop = None
self.tid = None
self.event = start_event
def run(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self.tid = current_thread()
self.loop.call_soon(self.event.set)
self.loop.run_forever()
def stop(self):
self.loop.call_soon_threadsafe(self.loop.stop)
def add_task(self, coro):
"""this method should return a task object, that I
can cancel, not a handle"""
def _async_add(func, fut):
try:
ret = func()
fut.set_result(ret)
except Exception as e:
fut.set_exception(e)
f = functools.partial(asyncio.async, coro, loop=self.loop)
if current_thread() == self.tid:
return f() # We can call directly if we're not going between threads.
else:
# We're in a non-event loop thread so we use a Future
# to get the task from the event loop thread once
# it's ready.
fut = Future()
self.loop.call_soon_threadsafe(_async_add, f, fut)
return fut.result()
def cancel_task(self, task):
self.loop.call_soon_threadsafe(task.cancel)
#asyncio.coroutine
def test(pool, name_task):
while True:
print(name_task, 'running')
with (yield from pool.cursor()) as cur:
print(name_task, " select. ")
yield from cur.execute("SELECT count(*) FROM test")
count = yield from cur.fetchone()
print(name_task, ' Result: ', count)
yield from asyncio.sleep(3)
#asyncio.coroutine
def connect_db():
dsn = 'dbname=%s user=%s password=%s host=%s' % ('testdb', 'user', 'passw', '127.0.0.1')
pool = yield from aiopg.create_pool(dsn)
print('create pool type =', type(pool))
# future.set_result(pool)
return (pool)
event = Event()
b = B(event)
b.start()
event.wait() # Let the loop's thread signal us, rather than sleeping
loop_db = asyncio.get_event_loop()
pool = loop_db.run_until_complete(connect_db())
time.sleep(2)
t = b.add_task(test(pool, 'Task1')) # This is a real task
t = b.add_task(test(pool, 'Task2'))
while True:
time.sleep(10)
b.stop()
Not return result in 'yield from cur.execute("SELECT count(*) FROM test")'
Long story short: you cannot share aiopg pool object from different event loops.
Every aiopg.Pool is coupled to event loop. If you don't specify loop parameter explicitly it is taken from asyncio.get_event_loop() call.
So it your example you have a pool coupled to event loop from main thread.
When you execute db query from separate thread you trying to accomplish it by executing thread's loop, not the main one. It doesn't work.