Im trying to make a simple thread in python3 where the test1 will run until a certain amount of number and then sleep while the test2 will still be running and also when it reaches a certain number it will go to sleep.
My code goes like this:
def test2(count):
if count == 8:
print("sleep for 4 sec")
time.sleep(3.0)
print("test2 thread = {}".format(count))
def test1(count):
if count == 5:
print("sleep for 5 sec")
time.sleep(3.0)
print("test1 thread = {}".format(count))
for num in range(0,10):
t1 = threading.Thread(target=test1, args=(num,))
t2 = threading.Thread(target=test2, args=(num,))
t1.start()
t2.start()
Also, i been coding python before but without using thread and now i wanted to have a go on it and hope this will end well :)
ohh, and additionally the output doesn't matter if they overlap.
The threading.Thread() creates new thread and t1.start() just dispatch it.
This code:
for num in range(0,10):
t1 = threading.Thread(target=test1, args=(num,))
t2 = threading.Thread(target=test2, args=(num,))
t1.start()
t2.start()
actually creates and start 2 new threads per iteration. At the end you have 20 threads + master thread.
Also when you start thread you should wait until it ends or run it as daemon thread. With daemon thread you are saying I don't care what you do and when you end.
Basic thread usage can looks like this:
import threading
def do_stuff():
print("Stuff on thread {}".format(threading.get_ident()))
print("Main thread {}".format(threading.get_ident()))
t = threading.Thread(target=do_stuff) # Specify what should be running in new thread
t.start() # Dispatch thread
t.join() # Wait until the thread is done
Note: threading.get_ident() gives you unique identifier of the thread where this function is called.
Now from you example if you want start 2 independent threads you can do this:
import threading
import time
def test2():
for count in range(0, 10):
if count == 8:
print("test2: sleep for 4 sec")
time.sleep(3.0)
print("test2: thread = {}".format(count))
def test1():
for count in range(0, 10):
if count == 5:
print("test 1: sleep for 5 sec")
time.sleep(3.0)
print("test1: thread = {}".format(count))
t1 = threading.Thread(target=test1)
t2 = threading.Thread(target=test2)
t1.start()
t2.start()
t1.join()
t2.join()
But you might want to synchronize those threads and send them some item at the "same" time.
import threading
# Create threads
t1 = threading.Thread(target=test1)
t2 = threading.Thread(target=test2)
# Run threads
t1.start()
t2.start()
# Go through some list or whatever
for num in range(0,10):
# send num to t1
# send num to t2
# wait for t1 and t2
pass
# Wait until threads are finished with their jobs
t1.join()
t2.join()
For sending value to other thread we can user queue.Queue. You can safely put there value in one thread and second thread can read it or wait until there is something (or multiple thread can write and multiple thread can read).
import threading
import time
import queue
def test2(q):
while True:
count = q.get() # Get data from the q2 queue
if count == 8:
print("test2: sleep for 4 sec")
time.sleep(3.0)
print("test2: thread = {}".format(count))
def test1(q):
while True:
count = q.get() # Get data from the q1 queue
if count == 5:
print("test 1: sleep for 5 sec")
time.sleep(3.0)
print("test1: thread = {}".format(count))
# Creates queues
q1 = queue.Queue()
q2 = queue.Queue()
# Create threads
t1 = threading.Thread(target=test1, args=(q1, ))
t2 = threading.Thread(target=test2, args=(q2, ))
# Run threads
t1.start()
t2.start()
# Go through some list or whatever
for num in range(0, 10):
# send num to t1
q1.put(num)
# send num to t2
q2.put(num)
# wait for t1 and t2
# ???
# Wait until threads are finished with their jobs
t1.join()
t2.join()
Oh wait... how can we know that threads are done with their work and we can send another value? Well we can use Queue again. Create new pair and sending e.g. True at the end of the test? function and then wait read in main loop from those queues. But for sending state information we should use threading.Event.
import threading
import time
import queue
def test2(q, e):
while True:
count = q.get() # Get data from the q2 queue
if count == 8:
print("test2: sleep for 4 sec")
time.sleep(3.0)
print("test2: thread = {}".format(count))
e.set() # Inform master the processing of given value is done
def test1(q, e):
while True:
count = q.get() # Get data from the q1 queue
if count == 5:
print("test 1: sleep for 5 sec")
time.sleep(3.0)
print("test1: thread = {}".format(count))
e.set() # Inform master the processing of given value is done
# Creates queues
q1 = queue.Queue()
q2 = queue.Queue()
# Create events
e1 = threading.Event()
e2 = threading.Event()
# Create threads
t1 = threading.Thread(target=test1, args=(q1, e1))
t2 = threading.Thread(target=test2, args=(q2, e2))
# Run threads
t1.start()
t2.start()
# Go through some list or whatever
for num in range(0, 10):
# send num to t1
q1.put(num)
# send num to t2
q2.put(num)
# wait for t1
e1.wait()
# wait for t2
e2.wait()
# Wait until threads are finished with their jobs
t1.join()
t2.join()
Now we are almost there but the script never ends. It's because the test? functions (threads) waits in infinite loop for data (from queues q1/q2). We need some way how to tell them "Ok, that's all folks". For that we can say None value in queues means end. The result following:
import threading
import time
import queue
def test2(q, e):
while True:
count = q.get() # Get data from the q2 queue
if count is None: # Exit on None value
return
if count == 8:
print("test2: sleep for 4 sec")
time.sleep(3.0)
print("test2: thread = {}".format(count))
e.set() # Inform master the processing of given value is done
def test1(q, e):
while True:
count = q.get() # Get data from the q1 queue
if count is None: # Exit on None value
return
if count == 5:
print("test 1: sleep for 5 sec")
time.sleep(3.0)
print("test1: thread = {}".format(count))
e.set() # Inform master the processing of given value is done
# Creates queues
q1 = queue.Queue()
q2 = queue.Queue()
# Create events
e1 = threading.Event()
e2 = threading.Event()
# Create threads
t1 = threading.Thread(target=test1, args=(q1, e1))
t2 = threading.Thread(target=test2, args=(q2, e2))
# Run threads
t1.start()
t2.start()
# Go through some list or whatever
for num in range(0, 10):
# send num to t1
q1.put(num)
# send num to t2
q2.put(num)
# wait for t1
e1.wait()
# wait for t2
e2.wait()
# Inform threads to exit
q1.put(None)
q2.put(None)
# Wait until threads are finished with their jobs
t1.join()
t2.join()
Note: instead of using parameters in threads "main" functions you can use global variables, because global variables or class attributes are shared across all threads. But usually it is bad practice.
Be aware of gotchas coming with threading, for example exception handling is not so easy. Imagine that function test1 raises exception before calling e.set(). Then the master thread never ends waiting on e1.wait().
Also CPython (the most common implementation of the Python) has something called GIL, which basically (with some exceptions) allows running only 1 thread at a time and the others are sleeping.
Threading documentation
Queue documentation
Related
Trying below code.
import concurrent.futures
import time
def do_it():
with concurrent.futures.ThreadPoolExecutor() as my_executor:
t1 = my_executor.submit(doing, 3)
ret_value = t1.result()
t2 = my_executor.submit(some_func)
return f"doing return is {ret_value}"
def doing(num):
print(f"Calculating Square for {num}")
return num*num
def some_func():
print("sleep for 6 sec")
time.sleep(6)
print("done sleeping 6 secs")
start = time.perf_counter()
print(do_it())
finish = time.perf_counter()
print(f"total time {finish-start}")
Getting below output:
Calculating Square for 3
sleep for 6 sec
done sleeping 6 secs
doing return is 9
total time 6.0060749100002795
But i was expecting (and want):
Calculating Square for 3
sleep for 6 sec
doing return is 9
total time <time much much less than 6>
<then after 6 sec>
done sleeping 6 secs
I want the return value of t1 Asap and let t2 continue. How can i achieve it. appreciate your help.
What you wrote here:
t1 = my_executor.submit(doing, 3)
ret_value = t1.result()
t2 = my_executor.submit(some_func)
makes the two functions (doing and some_func) running sequentially instead of concurrently because you explicitly blocked and awaited the value of the first one using .result() before launching the second one.
If you want to run the two functions concurrently, then you must submit them before awaiting them:
def do_it():
with concurrent.futures.ThreadPoolExecutor() as my_executor:
t1 = my_executor.submit(doing, 3)
t2 = my_executor.submit(some_func)
ret_value = t1.result()
return f"doing return is {ret_value}"
Here, t1 and t2 runs concurrently, they are submitted at (almost) the same time. You then await for the result of t1 via .result() and return its value, this is what you probably want.
However, if you need to await the first available result between the two functions, you can use the wait function or the as_completed one, take a look at the documentation to learn how to use them.
Edit
The with statement opens a context manager that calls the .shutdown() of the executor before continuing. This method waits for all futures to complete before returning, hence do_it() only returns when t1 and t2 completed.
If you want to return as soon as t2 started, pass the executor as a parameter to avoid calling .shutdown() in this function:
import concurrent.futures
import time
def do_it(executor):
ret_value = doing(3)
t2 = executor.submit(some_func)
return f"doing return is {ret_value}"
def doing(num):
print(f"Calculating Square for {num}")
return num*num
def some_func():
print("sleep for 6 sec")
time.sleep(6)
print("done sleeping 6 secs")
with concurrent.futures.ThreadPoolExecutor() as executor:
start = time.perf_counter()
print(do_it(executor))
finish = time.perf_counter()
print(f"total time {finish-start}")
You said that t2 needs the value of t1, so those calls cannot be made concurrent, you can execute t1 serially.
Also, note that it does not really make sense to submit t2 inside the do_it function, right after t1 completed. You could submit t2 after do_it has returned, which is more logical and simpler:
import concurrent.futures
import time
def do_it():
ret_value = doing(3)
return f"doing return is {ret_value}"
def doing(num):
print(f"Calculating Square for {num}")
return num*num
def some_func():
print("sleep for 6 sec")
time.sleep(6)
print("done sleeping 6 secs")
with concurrent.futures.ThreadPoolExecutor() as executor:
start = time.perf_counter()
print(do_it())
executor.submit(some_func)
finish = time.perf_counter()
print(f"total time {finish-start}")
This does not gives the exact output you want (two print statements are interleaved), but this does not matters anyway, the result is the same, t2 is launched as soon as t1 completed.
I am trying to grasp the Lock in multithreading module in python. But for some reason it is not locking the objects and lets the next thread run without waiting for the lock to release.
Here is the code:
from threading import Thread, Lock
import time
database_value = 0
def increase(lock):
global database_value
lock.acquire()
local_copy = database_value
local_copy += 1
time.sleep(0.1)
database_value = local_copy
lock.release()
if __name__ == '__main__':
lock = Lock()
print('start value',database_value)
thread1 = Thread(target =increase, args = (lock,))
thread2 = Thread(target =increase, args = (lock,))
print('start')
#start
thread1.start()
thread2.start()
#join
print('join')
thread1.join()
thread1.join()
print('end value', database_value)
The Output I am expecting is:
start value 0
start
join
end value 2
But the Output I get:
start value 0
start
join
end value 1
At the join step, you wait for thread1 instead of thread2.
#join
print('join')
thread1.join()
thread1.join() # Should be thread2
If you change it below, it will work.
#join
print('join')
thread1.join()
thread2.join()
Im trying to generate data in two threads and get that data in a separate thread that prints the data.
3 threads, 2 threads generate data , 1 thread consumes the data generated.
The Problem: not getting both generated data into the consumer thread
How can I pass data generated in 2 threads and deliver it in the consumer thread?
#from threading import Thread
import concurrent.futures
import time
# A thread that produces data
def producer(out_q):
while True:
# Produce some data
global data
data = data + 2
out_q.put(data)
# Another thread that produces data
def ac(out2_q):
while True:
global x
x = x + 898934567
out2_q.put(data)
# A thread that consumes data
def consumer(in_q):
while True:
# Get BOTH produced data from 2 threads
data = in_q.get()
# Process the data
time.sleep(.4)
print(data, end=' ', flush=True)
x=0
data = 0
q = Queue()
with concurrent.futures.ThreadPoolExecutor() as executor:
t1 = executor.submit(consumer, q)
t2 = executor.submit(producer,q)
t3 = executor.submit(ac, q)```
I recommend to go with threading.Thread in this case. Please see the code below and follow comments. Feel free to ask questions.
from threading import Thread, Event
from queue import Queue
import time
def producer_one(q: Queue, e: Event):
while not e.is_set():
q.put("one")
time.sleep(1)
print("Producer # one stopped")
def producer_two(q: Queue, e: Event):
while not e.is_set():
q.put("two")
time.sleep(2)
print("Producer # two stopped")
def consumer(q: Queue):
while True:
item = q.get()
print(item)
q.task_done() # is used to unblock queue - all tasks were done
time.sleep(2)
# will never be printed ! - since it is daemon thread
print("All work is done by consumer!")
if __name__ == '__main__':
_q = Queue() # "connects" threads
_e = Event() # is used to stop producers from the Main Thread
# create threads block
producer_th1 = Thread(target=producer_one, args=(_q, _e, ))
producer_th2 = Thread(target=producer_two, args=(_q, _e, ))
# daemon means that thread will be stopped when main thread stops
consumer_th = Thread(target=consumer, args=(_q, ), daemon=True)
try:
# starts block:
producer_th1.start()
producer_th2.start()
consumer_th.start()
time.sleep(20)
_e.set() # ask producers to stop
except KeyboardInterrupt:
_e.set() # ask producer threads to stop
print("Asked Producer Threads to stop")
finally:
producer_th1.join() # main thread is block until producer_th1 is not stopped
producer_th2.join() # main thread is block until producer_th2 is not stopped
_q.join() # now wait consumer to finish all tasks from queue
print("Queue is empty and program will be finished soon")
time.sleep(2) # just wait 2 seconds to show that consumer stops with main thread
print("All done!")
I am learning to thread one of my scripts and I have a question. I have three classes that does their own thing:
class odd(object):
def count():
num = 1
while True:
num += 2
class even(object):
def count():
num = 0
while True:
num += 2
class singles(object):
def count():
num = 0
while True:
num +=1
And my main starts three threads:
if __name__ == '__main__':
print('Starting...')
t1 = Thread(name='thread1', target=even.count)
t2 = Thread(name='thread2', target=odd.count)
t3 = Thread(name='thread3', target=singles.count)
t1.start()
t2.start()
t3.start()
while True:
pass
However when I do top -H -p <pid> to view threads spawned by my python process I see 4 total threads, 1 running, and 3 sleeping. I was expecting to see 3 total threads and 3 running. Can someone explain to me what is happening? Where is the 4th thread coming from? Why aren't all my threads running?
I wrote a script that uses 2 queues and 3 types of worker: producer, consumer (CPU-bound task), writer (I need to write the results sequentially).
This is the simplified version of my code:
from queue import Queue
from threading import Thread
def compute_single_score(data):
#do lots of calculations
return 0.0
def producer(out_q, data_to_compute):
while stuff:
data = data_to_compute.popitem()
out_q.put(data)
out_q.put(_sentinel)
def consumer(in_q, out_q):
while True:
data = in_q.get()
if data is _sentinel:
in_q.put(_sentinel)
break
out_q.put([data[0], compute_single_score(*data)])
in_q.task_done()
def writer(in_q):
while True:
data = in_q.get()
if data is _sentinel:
in_q.put(_sentinel)
break
in_q.task_done()
if __name__ == '__main__':
_sentinel = object()
jobs_queue = Queue()
scores_queue = Queue()
t1 = Thread(target=producer, args=(jobs_queue, data_to_compute,))
t2 = Thread(target=consumer, args=(jobs_queue,scores_queue,))
t3 = Thread(target=consumer, args=(jobs_queue,scores_queue,))
t4 = Thread(target=consumer, args=(jobs_queue,scores_queue,))
t5 = Thread(target=consumer, args=(jobs_queue,scores_queue,))
t6 = Thread(target=consumer, args=(jobs_queue,scores_queue,))
t7 = Thread(target=consumer, args=(jobs_queue,scores_queue,))
t8 = Thread(target=consumer, args=(jobs_queue,scores_queue,))
t9 = Thread(target=writer, args=(scores_queue,))
t1.start(); t2.start(); t3.start(); t4.start(); t5.start(); t6.start(); t7.start(); t8.start(); t9.start()
jobs_queue.join()
scores_queue.join()
print('File written')
It immediately prints out 'File written', instead waiting for the queues to be empty. Consequently the script doesn't exit although all the calculations are performed. Two threads seem to remain active.
Thanks a lot for your support.
It does wait for queues to be empty. But since putting things in queue happens in threads then it reaches .join() line faster then .put() happens. So when it does reach .join() queues are empty.
Now I'm not sure what you are trying to achieve simply because a producer has a while stuff loop. I assume that you want to continue processing until this condition is true. In particular you have to wait until t1 thread quits, i.e.
t1.start(); t2.start(); t3.start(); t4.start(); t5.start(); t6.start(); t7.start(); t8.start(); t9.start()
t1.join() # <-- this is important
jobs_queue.join()
scores_queue.join()
print('File written')
Otherwise you won't be able to synchronize it.
Side note 1: due to GIL there is no point in creating CPU bound threads. If your threads are not doing any IO (and they don't) then it will perform better when single-threaded. Well at least multiple consumer threads are pointless.
Side note 2: Do not use commas. It's not pythonic. Instead do this:
threads = []
threads.append(Thread(target=producer, args=(jobs_queue, data_to_compute,)))
threads.append(Thread(target=writer, args=(scores_queue,)))
for i in range(10):
threads.append(Thread(target=consumer, args=(jobs_queue,scores_queue,)))
for t in threads:
t.start()
threads[0].join()
Side note 3: You should handle case when queues are empty. data = in_q.get() will block forever meaning that your script won't quit (unless threads are marked as daemon). You should do for example:
try:
data = in_q.get(timeout=1)
except queue.Empty:
# handle empty queue here, perhaps quit if t1 is not alive
# otherwise just continue the loop
if not t1.is_alive(): # <-- you have to pass t1 to the thread
break
else:
continue
and then join all threads at the end (see side note 2) of the main thread:
for t in threads:
t.start()
for t in threads:
t.join()
print('File written')
And now you don't even have to join queues.
This is the code I used in the end (according to the requirements illustrated before):
from multiprocessing import JoinableQueue
from multiprocessing import Process
def compute_single_score(data):
#do lots of calculations
return 0.0
def producer(out_q, data_to_compute):
while stuff:
data = data_to_compute.popitem()
out_q.put(data)
def consumer(in_q, out_q):
while True:
try:
data = in_q.get(timeout=5)
except:
break
out_q.put([data[0], compute_single_score(*data)])
in_q.task_done()
def writer(in_q):
while True:
try:
data = in_q.get(timeout=5)
except:
break
#write
in_q.task_done()
if __name__ == '__main__':
jobs_queue = JoinableQueue()
scores_queue = JoinableQueue()
processes = []
processes.append(Process(target=producer, args=(jobs_queue, data_to_compute,)))
processes.append(Process(target=writer, args=(scores_queue,)))
for i in range(10):
processes.append(Process(target=consumer, args=(jobs_queue,scores_queue,)))
for p in processes:
p.start()
processes[1].join()
scores_queue.join()
print('File written')
I hope it will be of help for somebody else.