Python 3 control multiprocessing - python-3.x

I am trying to write function using Python multiprocessing that i can control it and pass "command" to cleanly terminate the process.
I looked for few examples and tried it out ,but didn't seems to work fro me
So basically i need to to run separate process function code that doing some while loop action
and when needed stop it by passing somehow command and exit
Please advice
Thanks
example 1
from multiprocessing import Process, Queue
def start_process(queue):
while True:
try:
m = queue.get()
if m == 'exit':
print ('cleaning up worker...')
# add here your cleaning up code
break
else:
print (m)
except KeyboardInterrupt:
print ('ignore CTRL-C from worker')
if __name__ == '__main__':
queue = Queue()
process = Process(target=start_process, args=(queue,))
process.start()
queue.put(12)
try:
process.join()
except KeyboardInterrupt:
print ('wait for worker to cleanup...')
queue.put('exit')
process.join()
example 2
import multiprocessing
import time
class MyProcess(multiprocessing.Process):
def __init__(self, ):
multiprocessing.Process.__init__(self)
self.exit = multiprocessing.Event()
def run(self):
while not self.exit.is_set():
pass
print ("You exited!")
def shutdown(self):
print ("Shutdown initiated")
self.exit.set()
if __name__ == "__main__":
process = MyProcess()
process.start()
print ("Waiting for a while")
time.sleep(3)
process.shutdown()
time.sleep(3)
print ("Child process state: %d" % process.is_alive())

both examples works fine for me - perhaps you're misunderstanding how they should work?
in the first example, when the main thread runs, it starts the child and sends 12. then it waits to join the child. at that point everything is stalled because the child is waiting for 'exit'. but if you then hit ctrl-C the 'exit' is sent, the child exits, and the second join is successful:
> python3.3 example1.py
12
^Cignore CTRL-C from worker
wait for worker to cleanup...
cleaning up worker...
>
if you just want the parent to send 'exit' and then for everything to end, use:
def start_process(queue):
while True:
try:
m = queue.get()
if m == 'exit':
print ('cleaning up worker...')
# add here your cleaning up code
break
else:
print (m)
except KeyboardInterrupt:
print ('ignore CTRL-C from worker')
print('goodbye cruel world')
if __name__ == '__main__':
queue = Queue()
process = Process(target=start_process, args=(queue,))
process.start()
queue.put(12)
print ('sending exit')
queue.put('exit')
process.join()
which gives:
> python3.3 my-example.py
sending exit
12
cleaning up worker...
goodbye cruel world
>
your second example also works (with the indentation fixed):
> python3.3 example2.py
Waiting for a while
Shutdown initiated
You exited!
Child process state: 0
>
(just wait a little). not sure what else you could have expected here.

Related

How to pass data between 3 threads that contain while True loops in Python?

Im trying to generate data in two threads and get that data in a separate thread that prints the data.
3 threads, 2 threads generate data , 1 thread consumes the data generated.
The Problem: not getting both generated data into the consumer thread
How can I pass data generated in 2 threads and deliver it in the consumer thread?
#from threading import Thread
import concurrent.futures
import time
# A thread that produces data
def producer(out_q):
while True:
# Produce some data
global data
data = data + 2
out_q.put(data)
# Another thread that produces data
def ac(out2_q):
while True:
global x
x = x + 898934567
out2_q.put(data)
# A thread that consumes data
def consumer(in_q):
while True:
# Get BOTH produced data from 2 threads
data = in_q.get()
# Process the data
time.sleep(.4)
print(data, end=' ', flush=True)
x=0
data = 0
q = Queue()
with concurrent.futures.ThreadPoolExecutor() as executor:
t1 = executor.submit(consumer, q)
t2 = executor.submit(producer,q)
t3 = executor.submit(ac, q)```
I recommend to go with threading.Thread in this case. Please see the code below and follow comments. Feel free to ask questions.
from threading import Thread, Event
from queue import Queue
import time
def producer_one(q: Queue, e: Event):
while not e.is_set():
q.put("one")
time.sleep(1)
print("Producer # one stopped")
def producer_two(q: Queue, e: Event):
while not e.is_set():
q.put("two")
time.sleep(2)
print("Producer # two stopped")
def consumer(q: Queue):
while True:
item = q.get()
print(item)
q.task_done() # is used to unblock queue - all tasks were done
time.sleep(2)
# will never be printed ! - since it is daemon thread
print("All work is done by consumer!")
if __name__ == '__main__':
_q = Queue() # "connects" threads
_e = Event() # is used to stop producers from the Main Thread
# create threads block
producer_th1 = Thread(target=producer_one, args=(_q, _e, ))
producer_th2 = Thread(target=producer_two, args=(_q, _e, ))
# daemon means that thread will be stopped when main thread stops
consumer_th = Thread(target=consumer, args=(_q, ), daemon=True)
try:
# starts block:
producer_th1.start()
producer_th2.start()
consumer_th.start()
time.sleep(20)
_e.set() # ask producers to stop
except KeyboardInterrupt:
_e.set() # ask producer threads to stop
print("Asked Producer Threads to stop")
finally:
producer_th1.join() # main thread is block until producer_th1 is not stopped
producer_th2.join() # main thread is block until producer_th2 is not stopped
_q.join() # now wait consumer to finish all tasks from queue
print("Queue is empty and program will be finished soon")
time.sleep(2) # just wait 2 seconds to show that consumer stops with main thread
print("All done!")

stop an endless function with threading

how do you kill an endless function in Python?
I would like the function to execute for 5 seconds and then be stopped, but thread.terminate() doesn't seem to work, I get the following error
AttributeError: 'Thread' object has no attribute 'terminate'
here is the code
import threading, time
def endless():
while True:
pass
p = threading.Thread(target=endless, name="endless")
p.start()
time.sleep(5)
if p.is_alive():
p.terminate()
p.join()
As mentioned in the comment, use Process if you want to force terminate the callback function.
from multiprocessing import Process, freeze_support
import time
def endless():
while True:
pass
if __name__ == '__main__': # required for windows
freeze_support()
p = Process(target=endless)
p.start()
time.sleep(5)
if p.is_alive():
p.terminate()
p.join()

how to stop a thread started by a timer, in main thread

I am trying to start a thread to listen to the incoming messages from a socket. so it contains an infinite loop. but when I try to close the gui, it hangs there, and does not close it. here is more simplified code without using any gui.
import threading,time,sys
def f(x):
while True:
time.sleep(0.5)
print(x)
timer = threading.Timer(0.1,f,("some text",) )
timer.start()
time.sleep(2)
print("time to stop")
sys.exit()
as you see the line sys.exit() won't end all threads (main thread and thread started by timer).
now I was wondering how to kill that specific thread which started by the timer.
thank you for your help
I finally find a solution for it. somehow we can use global variables to end an endless loop inside a thread, and therefore close it.
import threading,time
def f(x):
global z
while True:
time.sleep(0.5)
print(x)
if not z:
break
global z
z = True
timer = threading.Timer(0.1,f,("some text",) )
timer.start()
time.sleep(2)
print("time to stop")
z = False

How to stop a function outside of it in python

I want to know how to stop a running function outside of it. Here is how it should be:
def smth():
time.sleep(5) # Just an example
smth.stop()
Thanks for your help
Here's an example using the multiprocessing library:
from multiprocessing import Process
import time
def foo():
print('Starting...')
time.sleep(5)
print('Done')
p = Process(target=foo) #make process
p.start() #start function
time.sleep(2) #wait 2 secs
p.terminate() #kill it
print('Killed')
Output:
Starting...
Killed
Basically, what this code does is:
Create a process p which runs the function foo when started
Wait 2 seconds to simulate doing other stuff
End the process p with p.terminate()
Since p never passes time.sleep(5) in foo, it doesn't print 'Done'
Run this code online

How to stop the multithreaded process by ctrl+c

I'm trying to kill the process during executing, but it failed to stop it.
My thread is infinite loop thread won't be terminated until received kill signal
I can only terminated by kill command on terminal
def signal_handler(*args):
print("Killed by user")
teardown()
sys.exit(0)
def install_signal():
for sig in (SIGABRT, SIGILL, SIGINT, SIGSEGV, SIGTERM):
signal(sig, signal_handler)
class Monitor(object):
...
def run(self):
"""Run the monitor thread
Add the tasks to threads list
"""
try:
threads = {
"streaming": Streaming(
self.args["rtsp_link"],
int(self.args["duration"]),
int(self.args["period"])
),
"telnet_vid": p,
"telnet_aud": c,
}
for sub_task in threads.values():
sub_task.setDaemon(True)
sub_task.start()
for sub_task in threads.values():
sub_task.join()
time.sleep(1)
logging.info("Completed Monitor Tasks")
except KeyboardInterrupt:
print("Ok ok, quitting")
sys.exit(1)
except BaseException as e:
print("Got BaseException")
traceback.print_exc(file=sys.stdout)
raise e
def main():
try:
install_signal()
monitor = Monitor('tests/test_configuration.txt')
monitor.run()
except KeyboardInterrupt:
print("Ok ok, quitting")
sys.exit(1)
If I add timeout to join() the main thread won't be blocked will terminated in few seconds and can not received keyboard interrupt anymore
I referred to this blog
while len(running_threads) > 0 :
try:
print("To add join")
# Join all threads using a timeout so it doesn't block
# Filter out threads which have been joined or are None
running_threads = [t.join(1) for t in running_threads if t is not None and t.isAlive()]
except KeyboardInterrupt:
print("Ctrl+C received! Sending kill to threads!!!")
for t in running_threads:
t.kill_received = True
Replace:
for sub_task in threads.values():
sub_task.join() # no timeout
with:
running_threads = threads.values()
while running_threads:
for t in running_threads:
t.join(.1) # with timeout
running_threads = [t for t in running_threads if t.is_alive()]

Resources