I read this python documentation on multiprocessing.
But I had a requirement like following:
from multiprocessing import Pool
import time
def f(x):
print(x)
time.sleep(100000000000000);
return x*x
def f2(x):
print('**' + str(x) + '**')
time.sleep(100000000000000);
return x*x*x;
if __name__ == '__main__':
p = Pool(5)
print(p.map(f, [1, 2, 3]))
print(p.map(f2,[1,2,3]))
I have a use case where the function f is blocking(in my case it listens to a rabbitmq queue, so it should be blocking. I used a long time delay-100000000000000 to imply the infinite time blocking).
I have simulated blocking with the time.delay here.
but the need is i want the f2 (which is different from f) to start and run parallel. At present it does not even go to f2 as f itself blocks.
Can someone please give some guidance on how to make f and f2 start parallely despite the fact being f and f2 are both blocking.
Update:
Seems like i found a way, but puzzled
from multiprocessing import Pool
import time
def f(x):
print(x)
time.sleep(100000000000000);
return x*x
def f2(x):
print('**' + str(x) + '**')
time.sleep(100000000000000);
return x*x*x;
if __name__ == '__main__':
p = Pool(5)
res = p.apply_async(f, [2])
res = p.apply_async(f2,[4])
res.get()
how even after reassigning res1 it works and it invokes even f ?
Your approach doesn't work since first pool must complete to be able to print the result before the second pool. So the print kind of blocks the second pool from starting.
I used a thread pool to start both wrappers simultaneously, and I get the results asynchronously. I had to make a loop instead of the big sleep because python complained that sleep was too large. But amounts to the same thing.
In my case, there are printouts of 1,2,3 1,2,3 at once. And if I set the wait loop to a reasonable value I even get some correct results.
from multiprocessing import Pool
import concurrent.futures
import time
sleep_times = 1000000
def f(x):
print(x)
for i in range(sleep_times):
time.sleep(10)
return x*x
def f2(x):
print('**' + str(x) + '**')
for i in range(sleep_times):
time.sleep(10)
return x*x*x;
def fwrap(l):
p = Pool(len(l))
return(p.map(f, l))
def fwrap2(l):
p = Pool(len(l))
return(p.map(f2, l))
if __name__ == '__main__':
jr = dict()
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
jr[executor.submit(fwrap, (1, 2, 3))] = "f"
jr[executor.submit(fwrap2, (1, 2, 3))] = "f2"
for future in concurrent.futures.as_completed(jr):
ident = jr[future]
print(ident,future.result())
output (when sleep loop reduced to 1 to avoid waiting for too long)
1
**1**
2
**2**
**3**
3
there it waits a while (blocked by "processing time") then:
('f', [1, 4, 9])
('f2', [1, 8, 27])
Related
My program uses threading, but upon calling this certain function of another program (in a thread), the main program waits for the thread to end even though I haven't joined it, just started it.
main.py:
from constant_loops import Constant_loops
import threading
numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
z = threading.Thread(target=Constant_loops().add_up(numbers))
z.start()
print("Thread has been finished")
constant_loops.py:
import random
class Constant_loops:
def __init__(self):
global counter
counter = 0
def generate_number(self):
while True:
result = random.randint(1,10)
if result == 1 or result == 10:
print("Desired number obtained")
break
else:
print("Desired number hasn't been obtained yet")
self.add_up(self.numbers)
def add_up(self, numbers):
global counter
self.numbers = numbers
try:
self.numbers[counter] += 1
counter += 1
self.generate_number()
except:
print("Finished one number")
I am aware that there are much simpler ways of doing given task. This is just a simple example of the problem Im having with my main program.
I have a piece of code as shown below:
#!/bin/python3
import math
import os
import random
import re
import sys
import logging
def consumer():
while True:
x = yield
print(x)
def producer(n):
for _ in range(n):
x = int(input())
yield x
def rooter():
logging.info("Running the rooter")
while True:
value = (yield)
yield math.sqrt(value)
def squarer():
logging.info("Running the squarer")
while True:
value = (yield)
print("from squarer: {}".format(value))
yield value * value
def accumulator():
logging.info("Running the accumulator.")
running_total = 0
while True:
value = (yield)
running_total += value
yield running_total
def pipeline(prod, workers, cons):
logging.info("workers: {}".format(workers))
for num in prod:
for i, w in enumerate(workers):
num = w.send(num)
cons.send(num)
for worker in workers:
worker.close()
cons.close()
if __name__ == '__main__':
order = input().strip()
m = int(input())
prod = producer(m)
cons = consumer()
next(cons)
root = rooter()
next(root)
accumulate = accumulator()
next(accumulate)
square = squarer()
next(square)
pipeline(prod, eval(order), cons)
Sample input
[square, accumulate]
3 <- Number of inputs coming further
1 <- actual inputs
2
3
Sample Output
*The output should be as below:*
1
5
14
but comes to
10(sum of the squares of 1 and 3) when it should actually be 14 (sum of the squares of 1, 2, 3)
So essentially the input 2 is missed (It's second in the line of inputs).
On debugging further I found that this is the case for every alternate iteration, not just for the provided inputs here.
I am not able to decipher what's happening. If it's of any help, the co-routine squarer is the one returning None in the second iteration.
I'd appreciate any help.
I found a solution to this.
It's that we prime the co-routine after use in the pipeline function so the code becomes as follows: I have marked the next(w) line within asterix for reference.
#!/bin/python3
import math
import os
import random
import re
import sys
import logging
def consumer():
while True:
x = yield
print(x)
def producer(n):
for _ in range(n):
x = int(input())
yield x
def rooter():
logging.info("Running the rooter")
while True:
value = (yield)
yield math.sqrt(value)
def squarer():
logging.info("Running the squarer")
while True:
value = (yield)
print("from squarer: {}".format(value))
yield value * value
def accumulator():
logging.info("Running the accumulator.")
running_total = 0
while True:
value = (yield)
running_total += value
yield running_total
def pipeline(prod, workers, cons):
logging.info("workers: {}".format(workers))
for num in prod:
for i, w in enumerate(workers):
num = w.send(num)
**next(w)**
cons.send(num)
for worker in workers:
worker.close()
cons.close()
if __name__ == '__main__':
order = input().strip()
m = int(input())
prod = producer(m)
cons = consumer()
next(cons)
root = rooter()
next(root)
accumulate = accumulator()
next(accumulate)
square = squarer()
next(square)
pipeline(prod, eval(order), cons)
As mentioned in PEP specification it says that a generator function's yield
is always None when resumed by a normal next call. So when explicitly made to yield, it'll be ready to handle the next input immediately in this case.
I am trying to implement bitonic with the python multiprocessing library and a shared resource array that will be sorted at the end of the program.
The problem I am running into is that when I run the program, I get an prompt that asks "Your program is still running! Are you sure you want to cancel it?" and then when I click cancel N - 1 times (where N is the amount of processes I am trying to spawn) then it just hangs.
When this is run from the command line, it just outputs the unsorted array. Of course, I expect it to be sorted at the program's finish.
I've been using this resource to try and get a firm grasp on how I can mitigate my errors but I haven't had any luck, and now I am here.
ANY help would be appreciated, as I really don't have anywhere else to turn to.
I wrote this using Python 3.6 and here is the program in its entirety:
from multiprocessing import Process, Array
import sys
from random import randint
# remember to move this to separate file
def createInputFile(n):
input_file = open("input.txt","w+")
input_file.write(str(n)+ "\n")
for i in range(n):
input_file.write(str(randint(0, 1000000)) + "\n")
def main():
# createInputFile(1024) # uncomment this to create 'input.txt'
fp = open("input.txt","r") # remember to read from sys.argv
length = int(fp.readline()) # guaranteed to be power of 2 by instructor
arr = Array('i', range(length))
nums = fp.read().split()
for i in range(len(nums)):
arr[i]= int(nums[i]) # overwrite shared resource values
num_processes = 8 # remember to read from sys.argv
process_dict = dict()
change_in_bounds = len(arr)//num_processes
low_b = 0 # lower bound
upp_b = change_in_bounds # upper bound
for i in range(num_processes):
print("Process num: " + str(i)) # are all processes being generated?
process_dict[i] = Process(target=bitonic_sort, args=(True, arr[low_b:upp_b]) )
process_dict[i].start()
low_b += change_in_bounds
upp_b += change_in_bounds
for i in range(num_processes):
process_arr[i].join()
print(arr[:]) # Print our sorted array (hopefully)
def bitonic_sort(up, x):
if len(x) <= 1:
return x
else:
first = bitonic_sort(True, x[:len(x) // 2])
second = bitonic_sort(False, x[len(x) // 2:])
return bitonic_merge(up, first + second)
def bitonic_merge(up, x):
# assume input x is bitonic, and sorted list is returned
if len(x) == 1:
return x
else:
bitonic_compare(up, x)
first = bitonic_merge(up, x[:len(x) // 2])
second = bitonic_merge(up, x[len(x) // 2:])
return first + second
def bitonic_compare(up, x):
dist = len(x) // 2
for i in range(dist):
if (x[i] > x[i + dist]) == up:
x[i], x[i + dist] = x[i + dist], x[i] #swap
main()
I won't go into all the syntax errors in your code since I am sure your IDE tells you about those. The problem that you have is that you are missing an if name==main. I changed your def main() to def sort() and wrote this:
if __name__ == '__main__':
sort()
And it worked (after solving all the syntax errors)
The following code gets 2 numbers from the user and calculates how many primes are there between those numbers.
Running the code (between 1 and 1000000) without threads got me 2.8 secs and with 2 threads (my CPU has 2 cores) I got 4.8 secs.
Is it a problem with the PC, the code.
If you could try and run it on your PC and share the results i would thank you for that.
If you see a problem with the code or have any idea why that happens share your toughts.
import math
from threading import Thread
from timeit import timeit
first_num = input()
last_num = input()
primes = []
def is_prime(num):
if num == 2:
return True
if num == 1 or num % 2 == 0:
return False
for i in xrange(3, int(math.sqrt(num)) + 1, 2):
if (num % i == 0):
return False
return True
def find_primes(numbers):
for num in numbers:
if is_prime(num):
primes.append(num)
def targil_a():
find_primes(xrange(first_num, last_num))
print len(primes)
def targil_b():
numbers = range(first_num, last_num)
threads = []
for i in xrange(2):
part_len = int(len(numbers)/2)
t = Thread(target=find_primes, args=(numbers[part_len*i: part_len*(i+1) + 1],))
t.start()
threads.append(t)
for trd in threads:
trd.join()
print len(primes)
print timeit(targil_a, number=1)
primes = []
print timeit(targil_b, number=1)
i am trying to have a loop that keeps functioning until something is inputted which will break the loop. However if i put in 'stop = input()' in the loop then it has to go through that first before doing anything else. my code is like this: (it uses some minecraft commands. basically im trying to make a block move down a 20X20 square and have it be able to stop in the middle on command)
from mcpi import minecraft
mc=minecraft.Minecraft.create()
from time import sleep
pos=mc.player.getTilePos
x=pos.x
y=pos.y
z=pos.z
mc.setBlocks(x-10,y,z+30,x+10,y+20,z+30,49)
BB=0
while BB<20:
BB=BB+1
sleep(.7)
mc.setBlock(x,y+(20-BB),z+30,35)
mc.setBlock(x,y+(21-BB),z+30,49)
stop=input()
if stop=='stp':
break
how do i keep the loop going until someone inputs 'stp'? because currently it will move one block then stop and wait until i input something. The loop works if i take out the last three lines.
Whenever you run into input() in your code, Python will stop until it receives an input. If you're running the script in your console, then pressing Ctrl+C will stop execution of the program. (I'll assume you are, because how else would you be able to input 'stp'?)
You can run your logic in a different thread and signal this thread whenever you get an input.
from mcpi import minecraft
import threading
mc=minecraft.Minecraft.create()
from time import sleep
pos=mc.player.getTilePos
stop = False
def play():
x=pos.x
y=pos.y
z=pos.z
mc.setBlocks(x-10,y,z+30,x+10,y+20,z+30,49)
BB=0
while BB<20:
BB=BB+1
sleep(.7)
mc.setBlock(x,y+(20-BB),z+30,35)
mc.setBlock(x,y+(21-BB),z+30,49)
if stop:
break
t = threading.Thread(target=play)
t.start()
while True:
s = input()
if s == 'stp':
stop = True # the thread will see that and act appropriately
I know this is kind of old but I programed something similar earlier so here is a quick adaptation to your needs, hope you don't need it though xD
from mcpi.minecraft import Minecraft
from threading import Thread
from time import sleep
mc = Minecraft.create()
class Placer(Thread):
def __init__(self):
self.stop = False
Thread.__init__(self)
def run(self):
x, y, z = mc.player.getPos()
mc.setBlocks(x - 10, y, z + 30, x + 10, y + 20, z + 30, 49)
for i in range(20, -1, -1):
mc.setBlock(x, y + i, z + 30, 35)
mc.setBlock(x, y + i + 1 - 1, z + 30, 49)
sleep(0.7)
if self.stop:
break
placer = Placer()
placer.start()
while True:
text = input(": ")
if text == "stp":
placer.stop = True