I have this simple script:
import concurrent.futures
import time
executor= concurrent.futures.ThreadPoolExecutor(max_workers=3)
def pow():
time.sleep(1)
return 5
def main():
future = executor.submit(pow)
print(future.result())
if __name__ == '__main__':
main()
it will print 5
my question is - I would think the pow job would execute asynchronously - why I am able to print the future.result() value seemingly synchronously?
Related
I need an asynchronous parent process to handover function calls to a process pool.
The imports are to time consuming to spawn a new worker/process every time. So I thought to put tasks in an asyncio.queue and have a consumer listen to it and hand them off to the workers. (Sort of like how Gunicorn works but I don't want to run a webserver in order to make the calls).
However the function call seems to only be executed if I call res.get() on the response of pool.apply_async() but then it just runs as if I would call a normal synchronous for-loop.
This is my code:
#!/usr/bin/env python
import os
import time
import multiprocessing as mp
import asyncio
def f(x: list) -> int:
print(f'the pid of this process is: {os.getpid()}')
time.sleep(1)
return len(x)
def callback_func(x):
print(f'this is the callback function')
print(x)
async def consumer(queue):
with mp.Pool(processes=4) as pool:
while True:
x = await queue.get()
if x == 'stop':
break
# this makes it seem to run synchronous:
res = pool.apply_async(f, (x,))
print(res.get(), x, os.getpid())
# if I run this instead, both f() and callback_func
# are not executed.
#res = pool.apply_async(f, (x,), callback_func)
#print(x, os.getpid())
queue.task_done()
print(f'consumed')
async def producer(queue):
for i in range(20):
await queue.put([i,i+1,i+2])
# await asyncio.sleep(0.5)
await queue.put('stop')
async def main():
queue = asyncio.Queue()
input_coroutines = [consumer(queue), producer(queue)]
for f in asyncio.as_completed(input_coroutines):
try:
result = await f
print(result)
except Exception as e:
print('caught exception')
print(e)
if __name__ == "__main__":
asyncio.run(main())
What am I doing wrong?
please help me to find out where is the mistake, I want to run two functions in parallel. THNX
import asyncio
import time
async def test1():
for i in range(10):
print(f"First function {i}")
async def test2():
for i in range(10):
print(f"Second function {i}")
async def main():
print(f"started at {time.strftime('%X')}")
F = asyncio.create_task(test1())
S = asyncio.create_task(test2())
tasks = (F, S)
await asyncio.gather(*tasks)
print(f"finished at {time.strftime('%X')}")
asyncio.run(main())
Instead I get such an output that doesn't have anything relative to the parallel execution :
started at 10:42:53
First function 0
First function 1
First function 2
First function 3
First function 4
First function 5
First function 6
First function 7
First function 8
First function 9
Second function 0
Second function 1
Second function 2
Second function 3
Second function 4
Second function 5
Second function 6
Second function 7
Second function 8
Second function 9
finished at 10:42:53
Ok, finally I've rewrite my code to work with threads, and it works perfectly, but I couldn't figure out how to catch return values from threaded functions, so at first my code looked like this:
import threading
import time
def test1():
for i in range(10):
print(f"First function {i}")
return 11
def test2():
for i in range(10,20):
print(f"Second function {i}")
def main():
t = time.time()
t1 = threading.Thread(target=test1)
t2 = threading.Thread(target=test2)
t1.start()
t2.start()
print(t1.join())
t2.join()
print("Woaahh!! My work is finished..")
print("I took " + str(time.time() - t))
main()
And wwith help of this thread I've changed a bit to handle return values, here
So now my code look like this:
import time
import concurrent.futures
def test1():
for i in range(10):
print(f"First function {i}")
return 11
def test2():
for i in range(10,20):
print(f"Second function {i}")
return 12
def main():
t = time.time()
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(test1)
future2 = executor.submit(test2)
return_value = future.result()
rv= future2.result()
print(return_value)
print(rv)
print("Woaahh!! My work is finished..")
print("I took " + str(time.time() - t))
main()
I do not know what is your problem in your code but if you wan to run in parallel i think you can try something like this, by using thread ?
threading.Thread(target=test1).start()
threading.Thread(target=test2).start()
https://docs.python.org/fr/3/library/threading.html
Here is the other example of handling functions output code using Queues
import threading
import time
import queue
def test1(q):
for i in range(10):
print(f"First function {i}")
q.put(1)
def test2(q):
for i in range(10,20):
print(f"Second function {i}")
q.put(2)
def main():
q = queue.Queue()
t = time.time()
t1 = threading.Thread(target=test1, args=(q, ))
t2 = threading.Thread(target=test2, args=(q, ))
t1.start()
t2.start()
print(t1.join())
t2.join()
print("Woaahh!! My work is finished..")
print("I took " + str(time.time() - t))
while not q.empty():
print(q.get())
main()
Figured it out, i think? Runs as expected. I'm not able to view the original code so I wrote this new one up. Is there a better way to do this?
import time
import threading
def threadee():
f = open(r'log.txt')
for line in f:
print(line)
time.sleep(0.2)
def threader():
while True:
threadee()
def main():
thread = threading.Thread(target=threader)
thread.start()
while True:
print('main thread running')
print(threading.enumerate())
time.sleep(1)
if __name__ == '__main__':
main()
I want to achieve same effect as
# Code 1
from multiprocessing.pool import ThreadPool as Pool
from time import sleep, time
def square(a):
print('start', a)
sleep(a)
print('end', a)
return a * a
def main():
p = Pool(2)
queue = list(range(4))
start = time()
results = p.map(square, queue)
print(results)
print(time() - start)
if __name__ == "__main__":
main()
with async functions like
# Code 2
from multiprocessing.pool import ThreadPool as Pool
from time import sleep, time
import asyncio
async def square(a):
print('start', a)
sleep(a) # await asyncio.sleep same effect
print('end', a)
return a * a
async def main():
p = Pool(2)
queue = list(range(4))
start = time()
results = p.map_async(square, queue)
results = results.get()
results = [await result for result in results]
print(results)
print(time() - start)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
Currently Code 1 takes 4 seconds and Code 2 takes 6 seconds which means it is not running in parallel. What is the correct and cleanest way to run multiple async functions in parallel?
Better to be python 3.6 compatible. Thank you!
map_async() is not the same "async" as in async def - if it is fed with an async def method, it won't actually run it but return a coroutine instance immediately (try calling such a method without await). Then you awaited on the 4 coroutines one by one, that equals to sequential execution, and ended up with 6 seconds.
Please see following example:
from time import time
import asyncio
from asyncio.locks import Semaphore
semaphore = Semaphore(2)
async def square(a):
async with semaphore:
print('start', a)
await asyncio.sleep(a)
print('end', a)
return a * a
async def main():
start = time()
tasks = []
for a in range(4):
tasks.append(asyncio.ensure_future(square(a)))
await asyncio.wait(tasks)
print([t.result() for t in tasks])
print(time() - start)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
The Semaphore acts similarly like the ThreadPool - it allows only 2 concurrent coroutines entering the async with semaphore: block.
I am trying to create a simple script for python3.5 that can execute heavy computer vision algorithms in parallel. I have created a process by multiprocessing.Process in main process.
Inside that process I create concurrent.futures.ProcessPoolExecutor. Spawned process submits tasks to processPoolExecutor and it works perfectly fine. But when I try to stop and join spawned process it hangs on join.
Also if replace processPoolExecuter to threadPoolExecuter everything works perfectly. What did I miss?
Here is main file:
import multiprocessing as mp
import queue as Queue
import numpy as np
import cv2
from time import sleep
import executer_debug
def worker(queue):
pExecutor = executer_debug.Worker()
pExecutor.set()
while True:
print("-->{}<--".format(pExecutor.get()))
sleep(1)
try:
income = queue.get_nowait()
break
except Queue.Empty:
pass
pExecutor.set()
print("<1>{}<1>".format(pExecutor.get()))
print("<2>{}<2>".format(pExecutor.get()))
def main():
queue = mp.Queue()
currProcess = mp.Process(target = worker, args=(queue,))
currProcess.start()
frame = np.zeros((480,640), dtype=np.uint8)
while True:
cv2.imshow('frame',frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
print("stopped")
queue.put("stop")
currProcess.join()
if __name__ == "__main__":
main()
And here is the second file. Code is very simple just enough to demonstrate the issue.
import collections
from concurrent.futures import ProcessPoolExecutor
from time import sleep
import multiprocessing as mp
def worker():
return 1
class Worker():
def __init__(self):
self.workers_count = 4
self.poolExecutor = ProcessPoolExecutor(max_workers = self.workers_count)
self.executors = collections.deque()
def set(self):
self.executors.append(self.poolExecutor.submit(worker))
def get(self):
if len(self.executors) > 0:
if self.executors[0].done():
return self.executors.popleft().result()
else:
return 0
else:
return -1
Thank you!