I am learning about QRunnable and I have the following code:
from PyQt5.QtCore import QThreadPool, QRunnable
class SomeObjectToDoComplicatedStuff(QRunnable):
def __init__(self, name):
QRunnable.__init__(self)
self.name = name
def run(self):
print('running', self.name)
a = 10
b = 30
c = 0
for i in range(5000000):
c += a**b
print('done', self.name)
pool = QThreadPool.globalInstance()
pool.setMaxThreadCount(10)
batch_size = 100
workers = [None] * batch_size
for i in range(batch_size):
worker = SomeObjectToDoComplicatedStuff('object ' + str(i))
workers[i] = worker
pool.start(worker)
print('All cued')
pool.waitForDone()
# processing the results back
for i in range(batch_size):
print(workers[i].name, ' - examining again.')
I see that indeed there are different processes being alternated, but all is happening on a single core.
How can I make this code run using all the processor cores?
PS: This code is just a simplification of a super complicated number crunching application I am making. In it, I want to to do Monte Carlo in several threads and the worker itself is a complex optimization problem.
I have tried the python multiprocessing module but it doesn't handle scipy too well.
Not sure how much use this will be, but a multiprocessing version of your example script would be something like this:
from multiprocessing import Pool
class Worker(object):
def __init__(self, name):
self.name = name
def run(self):
print('running', self.name)
a = 10
b = 30
c = 0
for i in range(5000000):
c += a**b
print('done', self.name)
return self.name, c
def caller(worker):
return worker.run()
def run():
pool = Pool()
batch_size = 10
workers = (Worker('object%d' % i) for i in range(batch_size))
result = pool.map(caller, workers)
for item in result:
print('%s = %s' % item)
if __name__ == '__main__':
run()
How can I make this code run using all the processor cores?
Using PyQt (QRunner/QThread and likely), I think it's almost impossible because they (the python version, not the C++) are using the GIL.
The easiest solution would be to use multiprocessing, but since you have some problem using it along scipy you should look for some non-standard library.
I suggest you to take a look at ipyparallel, AFAIK they're developed under the same umbrella, so they're likely to work seamlessy.
Related
I am using the following code to process some pictures for my ML project and I would like to parallelize it.
import multiprocessing as mp
import concurrent.futures
def track_ids(seq):
'''The func is so big I can not put it here'''
ood = {}
for i in seq:
# I load around 500 images and process them
ood[i] = some Value
return ood
seqs = []
for seq in range(1, 10):# len(seqs)+1):
seq = txt+str(seq)
seqs.append(seq)
# serial call of the function
track_ids(seq)
#parallel call of the function
with concurrent.futures.ProcessPoolExecutor(max_workers=mp.cpu_count()) as ex:
ood_id = ex.map(track_ids, seqs)
if I run the code serially it takes 3.0 minutes but for parallel with concurrent, it takes 3.5 minutes.
can someone please explain why is that? and present a way to solve the problem.
btw, I have 12 cores.
Thanks
Here's a brief example of how one might go about profiling multiprocessing code vs serial execution:
from multiprocessing import Pool
from cProfile import Profile
from pstats import Stats
import concurrent.futures
def track_ids(seq):
'''The func is so big I can not put it here'''
ood = {}
for i in seq:
# I load around 500 images and process them
ood[i] = some Value
return ood
def profile_seq():
p = Profile() #one and only profiler instance
p.enable()
seqs = []
for seq in range(1, 10):# len(seqs)+1):
seq = txt+str(seq)
seqs.append(seq)
# serial call of the function
track_ids(seq)
p.disable()
return Stats(p), seqs
def track_ids_pr(seq):
p = Profile() #profile the child tasks
p.enable()
retval = track_ids(seq)
p.disable()
return (Stats(p, stream="dummy"), retval)
def profile_parallel():
p = Profile() #profile stuff in the main process
p.enable()
with concurrent.futures.ProcessPoolExecutor(max_workers=mp.cpu_count()) as ex:
retvals = ex.map(track_ids_pr, seqs)
p.disable()
s = Stats(p)
out = []
for ret in retvals:
s.add(ret[0])
out.append(ret[1])
return s, out
if __name__ == "__main__":
stat, retval = profile_parallel()
stat.print_stats()
EDIT: Unfortunately I found out that pstat.Stats objects cannot be used normally with multiprocessing.Queue because it is not pickleable (which is needed for the operation of concurrent.futures). Evidently it normally will store a reference to a file for the purpose of writing statistics to that file, and if none is given, it will by default grab a reference to sys.stdout. We don't actually need that reference however until we actually want to print out the statistics, so we can just give it a temporary value to prevent the pickle error, and then restore an appropriate value later. The following example should be copy-paste-able and run just fine rather than the pseudocode-ish example above.
from multiprocessing import Queue, Process
from cProfile import Profile
from pstats import Stats
import sys
def isprime(x):
for d in range(2, int(x**.5)):
if x % d == 0:
return False
return True
def foo(retq):
p = Profile()
p.enable()
primes = []
max_n = 2**20
for n in range(3, max_n):
if isprime(n):
primes.append(n)
p.disable()
retq.put(Stats(p, stream="dummy")) #Dirty hack: set `stream` to something picklable then override later
if __name__ == "__main__":
q = Queue()
p1 = Process(target=foo, args=(q,))
p1.start()
p2 = Process(target=foo, args=(q,))
p2.start()
s1 = q.get()
s1.stream = sys.stdout #restore original file
s2 = q.get()
# s2.stream #if we are just adding this `Stats` object to another the `stream` just gets thrown away anyway.
s1.add(s2) #add up the stats from both child processes.
s1.print_stats() #s1.stream gets used here, but not before. If you provide a file to write to instead of sys.stdout, it will write to that file)
p1.join()
p2.join()
Consider the following snippet:
import numpy as np
import multiprocessing as mp
import time
def work_standalone(args):
return 2
class Worker:
def __init__(self):
self.data = np.random.random(size=(10000, 10000))
# leave a trace whenever init is called
with open('rnd-%d' % np.random.randint(100), 'a') as f:
f.write('init called\n')
def work_internal(self, args):
return 2
def _run(self, target):
with mp.Pool() as pool:
tasks = [[idx] for idx in range(16)]
result = pool.imap(target, tasks)
for res in result:
pass
def run_internal(self):
self._run(self.work_internal)
def run_standalone(self):
self._run(work_standalone)
if __name__ == '__main__':
t1 = time.time()
Worker().run_standalone()
t2 = time.time()
print(f'Standalone took {t2 - t1:.3f} seconds')
t3 = time.time()
Worker().run_internal()
t4 = time.time()
print(f'Internal took {t3 - t4:.3f} seconds')
I.e. we have an object containing a large variable that uses multiprocessing to parallelize some work that has nothing to do with that large variable, i.e. does not read from or write to. The location of the worker process has a huge impact on the runtime:
Standalone took 0.616 seconds
Internal took 19.917 seconds
Why is this happening? I am completely lost. Note that __init__ is only called twice, so the random data is not created for every new process in the pool. The only reason I can think of why this would be slow is that data is copied around, but that would not make sense since it is never used anywhere, and python is supposed to use copy-on-write semantics. Also note that the difference disappears if you make run_internal a static method.
The issue you have is due to the target you are calling from the pool. That target is the function with the reference to Worker instance.
Now, you're right that the __init__() is only called twice. But remember, when you send anything to and from the processes, python will need to pickle the data first.
So, because your target is self.work_internal(), python has to pickle the Worker() instance every time the imap is called. This leads to one issue, self.data being copied over again and again.
The following is the proof. I just added 1 "input" statements, and fixed the last time of time calculation.
import numpy as np
import multiprocessing as mp
import time
def work_standalone(args):
return 2
class Worker:
def __init__(self):
self.data = np.random.random(size=(10000, 10000))
# leave a trace whenever init is called
with open('rnd-%d' % np.random.randint(100), 'a') as f:
f.write('init called\n')
def work_internal(self, args):
return 2
def _run(self, target):
with mp.Pool() as pool:
tasks = [[idx] for idx in range(16)]
result = pool.imap(target, tasks)
input("Wait for analysis")
for res in result:
pass
def run_internal(self):
self._run(self.work_internal)
# self._run(work_standalone)
def run_standalone(self):
self._run(work_standalone)
def work_internal(target):
with mp.Pool() as pool:
tasks = [[idx] for idx in range(16)]
result = pool.imap(target, tasks)
for res in result:
pass
if __name__ == '__main__':
t1 = time.time()
Worker().run_standalone()
t2 = time.time()
print(f'Standalone took {t2 - t1:.3f} seconds')
t3 = time.time()
Worker().run_internal()
t4 = time.time()
print(f'Internal took {t4 - t3:.3f} seconds')
You can run the code, when it shows up "wait for analysis", go and check the memory usage.
Like so
Then on the second time you see the message, press enter. And observe the memory usage increasing and decreasing again.
On the other hand, if you change self._run(self.work_internal) to self._run(work_standalone) you would notice that the speed is very fast, and the memory is not increasing, as well as the time taken is a lot shorter than doing self.work_internal.
Solution
One way to solve your issue is to set self.data as a static class variable. In normal cases, this would prevent instances from having to copy/reinit the variable again. This also prevented the issue from occuring.
class Worker:
data = np.random.random(size=(10000, 10000))
def __init__(self):
pass
...
Tried to write tfrecord w/ and w/o multithreading, and found the speed difference is not much (w/ 4 threads: 434 seconds; w/o multithread 590 seconds). Not sure if I used it correctly. Is there any better way to write tfrecord faster?
import tensorflow as tf
import numpy as np
import threading
import time
def generate_data(shape=[15,28,60,1]):
return np.random.uniform(size=shape)
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def write_instances_to_tfrecord(tfrecord_file, filenames):
tfrecord_writer = tf.python_io.TFRecordWriter(tfrecord_file)
for i, filename in enumerate(filenames):
curr_MFCC = generate_data()
curr_MFCC_raw = curr_MFCC.tostring()
curr_filename_raw = str(filename)+'-'+str(i)
example = tf.train.Example(features=tf.train.Features(
feature={
'MFCC': _bytes_feature(curr_MFCC_raw),
'filename': _bytes_feature(curr_filename_raw)
})
)
tfrecord_writer.write(example.SerializeToString())
tfrecord_writer.close()
def test():
threading_start = time.time()
coord = tf.train.Coordinator()
threads = []
for thread_index in xrange(4):
args = (str(thread_index), range(200000))
t = threading.Thread(target=write_instances_to_tfrecord, args=args)
t.start()
threads.append(t)
coord.join(threads)
print 'w/ threading takes', time.time()-threading_start
start = time.time()
write_instances_to_tfrecord('5', range(800000))
print 'w/o threading takes', time.time()-start
if __name__ == '__main__':
test()
When using python threading, due to the GIL restriction in the cPython implementation, the CPU utilization will be capped at 1 core. No matter how many threads you add, you will not see a speed up.
A simple solution in your case would be to use the multiprocessing module.
The code is almost exactly the same as what you have, just switch threads to processes:
from multiprocessing import Process
coord = tf.train.Coordinator()
processes = []
for thread_index in xrange(4):
args = (str(thread_index), range(200000))
p = Process(target=write_instances_to_tfrecord, args=args)
p.start()
processes.append(p)
coord.join(processes)
I tested this on my own tfrecord writer code, and got a linear scaling speedup. Total number of processes is limited by memory.
It's better to use Tensorflow computation graph to take advantage of multithreading since each session and graph can be run in different threads. With computation graph, it's about 40 times faster.
I am working on a project called monte carlo tree search(MCTS). In my project, there is a main loop which is time limited. The psudo code for MCTS module goes like this:
import time
class mctsagent:
def select_node(self):
# do sth
def expand(self):
# do sth
def roll_out(self):
# do sth
def backup(self):
# do sth
def search(self, time_budget):
# main loop
startTime = time.clock()
num_rollouts = 0
while (time.clock() - startTime < time_budget):
result1 = self.select_node()
result2 = self.expand(result1)
result3 = self.roll_out(result2)
self.backup(result3)
num_rollouts += 1
I want to use threads to have as many as iterations possible.
The question is that how can I develope a class which could make use of threading module to run the search function in different threads.
for example develope a module like this:
class MCTS_Thread:
def __init__(self, agent, time_budget):
self.agent = agent # MCTS module
self.time_budget = time_budget # The time for running loop
def run(self):
self.agent.search(self.time_budget)
and then instead of search function (# main loop) I use this:
def search(self, time_budget):
threads = []
for i in range(3):
# 3 threads running search
t = MCTS_Thread(self, time_budget)
threads.append(t)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
The only thing matters here is that the functions in search function have to run in the same order in each thread.
I have a method which needs to process through a large database, that would take hours/days to dig through
The arguments are stored in a (long) list of which max X should be processed in one batch. The method does not need to return anything, yet i return "True" for "fun"...
The function is working perfectly when I'm iterating through it linearly (generating/appending the results in other tables not seen here), yet I am unable to get apply_async or map_async work. (it worked before in other projects)
Any hint of what might I be doing wrong would be appreciated, thanks in advance!
See code below:
import multiprocessing as mp
class mainClass:
#loads of stuff
def main():
multiprocess = True
batchSize = 35
mC = mainClass()
while True:
toCheck = [key for key, value in mC.lCheckSet.items()] #the tasks are stored in a dictionary, I'm referring to them with their keys, which I turn to a list here for iteration.
if multiprocess == False:
#this version works perfectly fine
for i in toCheck[:batchSize]:
mC.check(i)
else:
#the async version does not, either with apply_async...
with mp.Pool(processes = 8) as pool:
temp = [pool.apply_async(mC.check, args=(toCheck[n],)) for n in range(len(toCheck[:batchSize]))]
results = [t.get() for t in temp]
#...or as map_async
pool = mp.Pool(processes = 8)
temp = pool.map_async(mC.check, toCheck[:batchSize])
pool.close()
pool.join()
if __name__=="__main__":
main()
The "smell" here is that you are instantiating your maincClass on the main Process, just once, and then trying to call a method on it on the different processes - but note that when you pass mC.check to your process pool, it is a method already bound to the class instantiated in this process.
I'd guess there is where your problem lies. Although that could possibly work - and it does - I made this simplified version and it works as intended :
import multiprocessing as mp
import random, time
class MainClass:
def __init__(self):
self.value = 1
def check(self, arg):
time.sleep(random.uniform(0.01, 0.3))
print(id(self),self.value, arg)
def main():
mc = MainClass()
with mp.Pool(processes = 4) as pool:
temp = [pool.apply_async(mc.check, (i,)) for i in range(8)]
results = [t.get() for t in temp]
main()
(Have you tried just adding some prints to make sure the method is not running at all?)
So, the problem lies likely in some complex state in your MainClass that does not make it to the parallel processes in a good way. A possible work-around is to instantiate your mainclasses inside each process - that can be easily done since MultiProcessing allow you to get the current_process, and use this object as a namespace to keep data in the process instantiated in the worker Pool, across different calls to apply async.
So, create a new check function like the one bellow - and instead of instantiating your mainclass in the mainprocess, instantiate it inside each process in the pool:
import multiprocessing as mp
import random, time
def check(arg):
process = mp.current_process
if not hasattr(process, "main_class"):
process.main_class = MainClass()
process.main_class.check(arg)
class MainClass:
def __init__(self):
self.value = random.randrange(100)
def check(self, arg):
time.sleep(random.uniform(0.01, 0.3))
print(id(self),self.value, arg)
def main():
mc = MainClass()
with mp.Pool(processes = 2) as pool:
temp = [pool.apply_async(check, (i,)) for i in range(8)]
results = [t.get() for t in temp]
main()
I got to this question with the same problem, my apply_async calls not called at all, but the reason on my case was that the parameters number on apply_async call was different to the number on function declaration