I spent nearly the whole day with this and came to the end of my knowledge:
I want to change a shared multiprocessing.Value string in the subprocess, but python hangs as soon as the subprocess is trying to change the shared value.
Below an example code:
from multiprocessing import Process, Value, freeze_support
from ctypes import c_wchar_p
def test(x):
with x.get_lock():
x.value = 'THE TEST WORKED'
return
if __name__ == "__main__":
freeze_support()
value = Value(c_wchar_p, '')
p = Process(target=test, args = (value,))
p.start()
print(p.pid)
# this try block is to also allow p.run()
try:
p.join()
p.terminate()
except:
pass
print(value.value)
What I tried and does not work:
I tried ctypes c_wchar_p and c_char_p, but both result in the same freezing.
I tried also without x.get_lock()
I tried also without freeze_support()
What works (but does not help):
Using a float as the shared value (value = Value('d',0) and x.value = 1).
Running the Process without starting a subprocess (replace p.start() with p.run() )
I am using Windows 10 64 bit and Python 3.6.4 (Spyder, but also tried outside of Spyder).
Any help welcome!
A shared pointer won't work in another process because the pointer is only valid in the process in which it was created. Instead, use an array:
import multiprocessing as mp
def test(x):
x.value = b'Test worked!'
if __name__ == "__main__":
x = mp.Array('c',15)
p = mp.Process(target=test, args = (x,))
p.start()
p.join()
print(x.value)
Output:
b'Test worked!'
Note that array type 'c' is specialized and returns a SynchronizedString vs. other types that return SynchronizedArray. Here's how to use type 'u' for example:
import multiprocessing as mp
from ctypes import *
def test(x):
x.get_obj().value = 'Test worked!'
if __name__ == "__main__":
x = mp.Array('u',15)
p = mp.Process(target=test, args = (x,))
p.start()
p.join()
print(x.get_obj().value)
Output:
Test worked!
Note that operations on the wrapped value that are non-atomic such as += that do read/modify/write should be protected with a with x.get_lock(): context manager.
Related
I am using the following code to process some pictures for my ML project and I would like to parallelize it.
import multiprocessing as mp
import concurrent.futures
def track_ids(seq):
'''The func is so big I can not put it here'''
ood = {}
for i in seq:
# I load around 500 images and process them
ood[i] = some Value
return ood
seqs = []
for seq in range(1, 10):# len(seqs)+1):
seq = txt+str(seq)
seqs.append(seq)
# serial call of the function
track_ids(seq)
#parallel call of the function
with concurrent.futures.ProcessPoolExecutor(max_workers=mp.cpu_count()) as ex:
ood_id = ex.map(track_ids, seqs)
if I run the code serially it takes 3.0 minutes but for parallel with concurrent, it takes 3.5 minutes.
can someone please explain why is that? and present a way to solve the problem.
btw, I have 12 cores.
Thanks
Here's a brief example of how one might go about profiling multiprocessing code vs serial execution:
from multiprocessing import Pool
from cProfile import Profile
from pstats import Stats
import concurrent.futures
def track_ids(seq):
'''The func is so big I can not put it here'''
ood = {}
for i in seq:
# I load around 500 images and process them
ood[i] = some Value
return ood
def profile_seq():
p = Profile() #one and only profiler instance
p.enable()
seqs = []
for seq in range(1, 10):# len(seqs)+1):
seq = txt+str(seq)
seqs.append(seq)
# serial call of the function
track_ids(seq)
p.disable()
return Stats(p), seqs
def track_ids_pr(seq):
p = Profile() #profile the child tasks
p.enable()
retval = track_ids(seq)
p.disable()
return (Stats(p, stream="dummy"), retval)
def profile_parallel():
p = Profile() #profile stuff in the main process
p.enable()
with concurrent.futures.ProcessPoolExecutor(max_workers=mp.cpu_count()) as ex:
retvals = ex.map(track_ids_pr, seqs)
p.disable()
s = Stats(p)
out = []
for ret in retvals:
s.add(ret[0])
out.append(ret[1])
return s, out
if __name__ == "__main__":
stat, retval = profile_parallel()
stat.print_stats()
EDIT: Unfortunately I found out that pstat.Stats objects cannot be used normally with multiprocessing.Queue because it is not pickleable (which is needed for the operation of concurrent.futures). Evidently it normally will store a reference to a file for the purpose of writing statistics to that file, and if none is given, it will by default grab a reference to sys.stdout. We don't actually need that reference however until we actually want to print out the statistics, so we can just give it a temporary value to prevent the pickle error, and then restore an appropriate value later. The following example should be copy-paste-able and run just fine rather than the pseudocode-ish example above.
from multiprocessing import Queue, Process
from cProfile import Profile
from pstats import Stats
import sys
def isprime(x):
for d in range(2, int(x**.5)):
if x % d == 0:
return False
return True
def foo(retq):
p = Profile()
p.enable()
primes = []
max_n = 2**20
for n in range(3, max_n):
if isprime(n):
primes.append(n)
p.disable()
retq.put(Stats(p, stream="dummy")) #Dirty hack: set `stream` to something picklable then override later
if __name__ == "__main__":
q = Queue()
p1 = Process(target=foo, args=(q,))
p1.start()
p2 = Process(target=foo, args=(q,))
p2.start()
s1 = q.get()
s1.stream = sys.stdout #restore original file
s2 = q.get()
# s2.stream #if we are just adding this `Stats` object to another the `stream` just gets thrown away anyway.
s1.add(s2) #add up the stats from both child processes.
s1.print_stats() #s1.stream gets used here, but not before. If you provide a file to write to instead of sys.stdout, it will write to that file)
p1.join()
p2.join()
I want to take the working code from below and put it into a function.
import multiprocessing as mp
def parameters(x,n):
for i in x:
yield (i,n)
def power(a):
x, n = a
return x**n
if __name__ == '__main__':
p = [i for i in range(1000)]
p = parameters(p,2)
agents = 4
chunk = 10
with mp.Pool(processes = agents) as pool:
o = pool.map(power,p,chunksize = chunk)
print(o)
So that I can call it whenever I want. I tried doing something like this:
import multiprocessing as mp
def parameters(x,n):
for i in x:
yield (i,n)
def power(a):
x, n = a
return x**n
def calculate(s,n):
p = [i for i in range(s)]
p = parameters(p,n)
agents = 4
chunk = 10
with mp.Pool(processes = agents) as pool:
o = pool.map(power,p,chunksize = chunk)
return o
print(calculate(1000,2))
However this does not work at all, It tells me That another process has started before one has ended. But the code above did work. Is there a way to properly take this code into a function? If not with this setup, then with what setup?
Make sure to protect code that should only run in the main process with if __name__ == '__main__':. This code works:
import multiprocessing as mp
def parameters(x,n):
for i in x:
yield (i,n)
def power(a):
x, n = a
return x**n
def calculate(s,n):
p = [i for i in range(s)]
p = parameters(p,n)
agents = 4
chunk = 10
with mp.Pool(processes = agents) as pool:
o = pool.map(power,p,chunksize = chunk)
return o
if __name__ == '__main__':
print(calculate(1000,2))
Without the if, the following error is raised:
RuntimeError:
An attempt has been made to start a new process before the
current process has finished its bootstrapping phase.
This probably means that you are not using fork to start your
child processes and you have forgotten to use the proper idiom
in the main module:
if __name__ == '__main__':
freeze_support()
...
The "freeze_support()" line can be omitted if the program
is not going to be frozen to produce an executable.
I need to run a function in parallel with different processes. I am using the code below. My function arguments types are (nested list of binary numbers, int). The problem is when I run the script processes start running but they finish in the middle of the function (check where I put "Here" in the code) without no error.
def CreateP(pattern):
print("start")
import pandas as pd
P=[]
for j in range( n1):
for i in range(n):
for i1 in range(n):
if i1!=i:
#Here (my processes end here)
if pattern[i][j]==pattern[i1][j]:
if (i,i1,j) not in P:
P.append((i,i1,j))
if (i1,i,j) not in P:
P.append((i1,i,j))
else:
if pattern[i][j]==1:
if (i1,i,j) not in P:
P.append((i1,i,j))
else:
if (i,i1,j) not in P:
P.append((i,i1,j))
pd.DataFrame(P).to_csv("test1.csv" ,index=False)
pd.DataFrame(pattern).to_csv("test2.csv" ,index=False)
return P
def main():
import concurrent.futures
for i in range(a1):
with concurrent.futures.ProcessPoolExecutor(max_workers=61) as executor:
f1=executor.submit(CreateP,( Pbig1[i],1))
f2=executor.submit(CreateP,( Pbig2[i],2))
if __name__ == "__main__":
main()
Does any one know how to call a def form a thread.
Clock Program:
import sys
from tkinter import *
from tkinter import messagebox
from tkinter import filedialog
from time import sleep
import threading
class MyThread ( threading.Thread ):
def mclock(): # function that it can't call
x = 1
z = 0
while x != -1:
Label(mGui,text = str(x) + "second(s)").pack()
x = x+1
sleep(1)
if x == 60:
x = 1
z = z+1
Label(mGui, text= str(z) + " minute(s) has past.").pack()
return
return
MyThread().start()
mGui = Tk()
mGui.geometry("300x200+100+100")
mGui.title("Jono's Clock")
menubar = Menu(mGui)
filemenu = Menu(menubar, tearoff = 0)
filemenu.add_command(label = "Clock",command = mclock) # can't use function
menubar.add_cascade(label = "File",menu = filemenu)
mGui.config(menu = menubar)
mGui.mainloop()
If any one sees any other errors please state. I am also using windows 7 and python 3.3.
There are several syntax errors in the code you've posted, and I'm not sure exactly what you intended with them, so here's an overview of how to run stuff from threads.
If you want your thread to run your own code from a custom thread class, the usual way to do that is to put the code in a method named run, which will be executed automatically when the thread is started:
import threading
class MyThread(threading.Thread):
def run(self):
# do your stuff here
print("Hello World")
MyThread().start()
Alternatively, if you don't need a class, you can create your function at the top level of your module, and pass it as an argument to threading.Thread's constructor:
def my_function():
print("Hello World")
threading.Thread(target=my_function).start()
Note that you often want to keep a reference to the thread object, rather than letting it go as the code above does. This requires you to use two lines to create and then start the thread:
thread = MyThread() # or the alternative version
thread.start()
This lets you later do:
thread.join()
Which ensures that the thread has finished its work.
I am using pythons multiprocessing module in some of my code. I have a controller class that controls a class and performs some action.
import multiprocessing
from multiprocessing import Queue, Process, Manager
class dosomething(multiprocessing.Process):
def __init__(self,managerList):
self.mlist = managerList
print self.mlist
def run(self):
self.mlist.append((4,5,6))
class doController:
def __init__(self):
mgr = Manager()
self.mlist = mgr.list()
self.mlist.append((1,2,3,4))
t = dosomething(self.mlist)
#t.daemon = True
#t.start()
def printer(self):
return self.mlist
gd = doController()
print gd.printer()
Pring mlist in the init part of dosomething prints [(1, 2, 3, 4)] as expected but the list in the dosomething part does not work giving out IOError 11. Can anyone help if it's right or wrong?
The call to the Process.__init__ is missing.
You don't necessarely need to create a Process subclass you could use functions:
from multiprocessing import Process, Manager
def dosomething(mlist):
mlist.append((4,5,6))
def main():
manager = Manager()
L = manager.list((1,2,3,4))
p = Process(target=dosomething, args=(L,))
p.start()
p.join()
print L
if __name__ == '__main__':
main()