python3 code not running concurrently Asyncio - python-3.x

please help me to find out where is the mistake, I want to run two functions in parallel. THNX
import asyncio
import time
async def test1():
for i in range(10):
print(f"First function {i}")
async def test2():
for i in range(10):
print(f"Second function {i}")
async def main():
print(f"started at {time.strftime('%X')}")
F = asyncio.create_task(test1())
S = asyncio.create_task(test2())
tasks = (F, S)
await asyncio.gather(*tasks)
print(f"finished at {time.strftime('%X')}")
asyncio.run(main())
Instead I get such an output that doesn't have anything relative to the parallel execution :
started at 10:42:53
First function 0
First function 1
First function 2
First function 3
First function 4
First function 5
First function 6
First function 7
First function 8
First function 9
Second function 0
Second function 1
Second function 2
Second function 3
Second function 4
Second function 5
Second function 6
Second function 7
Second function 8
Second function 9
finished at 10:42:53

Ok, finally I've rewrite my code to work with threads, and it works perfectly, but I couldn't figure out how to catch return values from threaded functions, so at first my code looked like this:
import threading
import time
def test1():
for i in range(10):
print(f"First function {i}")
return 11
def test2():
for i in range(10,20):
print(f"Second function {i}")
def main():
t = time.time()
t1 = threading.Thread(target=test1)
t2 = threading.Thread(target=test2)
t1.start()
t2.start()
print(t1.join())
t2.join()
print("Woaahh!! My work is finished..")
print("I took " + str(time.time() - t))
main()
And wwith help of this thread I've changed a bit to handle return values, here
So now my code look like this:
import time
import concurrent.futures
def test1():
for i in range(10):
print(f"First function {i}")
return 11
def test2():
for i in range(10,20):
print(f"Second function {i}")
return 12
def main():
t = time.time()
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(test1)
future2 = executor.submit(test2)
return_value = future.result()
rv= future2.result()
print(return_value)
print(rv)
print("Woaahh!! My work is finished..")
print("I took " + str(time.time() - t))
main()

I do not know what is your problem in your code but if you wan to run in parallel i think you can try something like this, by using thread ?
threading.Thread(target=test1).start()
threading.Thread(target=test2).start()
https://docs.python.org/fr/3/library/threading.html

Here is the other example of handling functions output code using Queues
import threading
import time
import queue
def test1(q):
for i in range(10):
print(f"First function {i}")
q.put(1)
def test2(q):
for i in range(10,20):
print(f"Second function {i}")
q.put(2)
def main():
q = queue.Queue()
t = time.time()
t1 = threading.Thread(target=test1, args=(q, ))
t2 = threading.Thread(target=test2, args=(q, ))
t1.start()
t2.start()
print(t1.join())
t2.join()
print("Woaahh!! My work is finished..")
print("I took " + str(time.time() - t))
while not q.empty():
print(q.get())
main()

Related

Multiprocessing pool executing synchronous

I need an asynchronous parent process to handover function calls to a process pool.
The imports are to time consuming to spawn a new worker/process every time. So I thought to put tasks in an asyncio.queue and have a consumer listen to it and hand them off to the workers. (Sort of like how Gunicorn works but I don't want to run a webserver in order to make the calls).
However the function call seems to only be executed if I call res.get() on the response of pool.apply_async() but then it just runs as if I would call a normal synchronous for-loop.
This is my code:
#!/usr/bin/env python
import os
import time
import multiprocessing as mp
import asyncio
def f(x: list) -> int:
print(f'the pid of this process is: {os.getpid()}')
time.sleep(1)
return len(x)
def callback_func(x):
print(f'this is the callback function')
print(x)
async def consumer(queue):
with mp.Pool(processes=4) as pool:
while True:
x = await queue.get()
if x == 'stop':
break
# this makes it seem to run synchronous:
res = pool.apply_async(f, (x,))
print(res.get(), x, os.getpid())
# if I run this instead, both f() and callback_func
# are not executed.
#res = pool.apply_async(f, (x,), callback_func)
#print(x, os.getpid())
queue.task_done()
print(f'consumed')
async def producer(queue):
for i in range(20):
await queue.put([i,i+1,i+2])
# await asyncio.sleep(0.5)
await queue.put('stop')
async def main():
queue = asyncio.Queue()
input_coroutines = [consumer(queue), producer(queue)]
for f in asyncio.as_completed(input_coroutines):
try:
result = await f
print(result)
except Exception as e:
print('caught exception')
print(e)
if __name__ == "__main__":
asyncio.run(main())
What am I doing wrong?

How to correctly use async-await with thread pool in Python 3

I want to achieve same effect as
# Code 1
from multiprocessing.pool import ThreadPool as Pool
from time import sleep, time
def square(a):
print('start', a)
sleep(a)
print('end', a)
return a * a
def main():
p = Pool(2)
queue = list(range(4))
start = time()
results = p.map(square, queue)
print(results)
print(time() - start)
if __name__ == "__main__":
main()
with async functions like
# Code 2
from multiprocessing.pool import ThreadPool as Pool
from time import sleep, time
import asyncio
async def square(a):
print('start', a)
sleep(a) # await asyncio.sleep same effect
print('end', a)
return a * a
async def main():
p = Pool(2)
queue = list(range(4))
start = time()
results = p.map_async(square, queue)
results = results.get()
results = [await result for result in results]
print(results)
print(time() - start)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
Currently Code 1 takes 4 seconds and Code 2 takes 6 seconds which means it is not running in parallel. What is the correct and cleanest way to run multiple async functions in parallel?
Better to be python 3.6 compatible. Thank you!
map_async() is not the same "async" as in async def - if it is fed with an async def method, it won't actually run it but return a coroutine instance immediately (try calling such a method without await). Then you awaited on the 4 coroutines one by one, that equals to sequential execution, and ended up with 6 seconds.
Please see following example:
from time import time
import asyncio
from asyncio.locks import Semaphore
semaphore = Semaphore(2)
async def square(a):
async with semaphore:
print('start', a)
await asyncio.sleep(a)
print('end', a)
return a * a
async def main():
start = time()
tasks = []
for a in range(4):
tasks.append(asyncio.ensure_future(square(a)))
await asyncio.wait(tasks)
print([t.result() for t in tasks])
print(time() - start)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
The Semaphore acts similarly like the ThreadPool - it allows only 2 concurrent coroutines entering the async with semaphore: block.

Appending to merged async generators in Python

I'm trying to merge a bunch of asynchronous generators in Python 3.7 while still adding new async generators on iteration. I'm currently using aiostream to merge my generators:
from asyncio import sleep, run
from aiostream.stream import merge
async def go():
yield 0
await sleep(1)
yield 50
await sleep(1)
yield 100
async def main():
tasks = merge(go(), go(), go())
async for v in tasks:
print(v)
if __name__ == '__main__':
run(main())
However, I need to be able to continue to add to the running tasks once the loop has begun. Something like.
from asyncio import sleep, run
from aiostream.stream import merge
async def go():
yield 0
await sleep(1)
yield 50
await sleep(1)
yield 100
async def main():
tasks = merge(go(), go(), go())
async for v in tasks:
if v == 50:
tasks.merge(go())
print(v)
if __name__ == '__main__':
run(main())
The closest I've got to this is using the aiostream library but maybe this can also be written fairly neatly with just the native asyncio standard library.
Here is an implementation that should work efficiently even with a large number of async iterators:
class merge:
def __init__(self, *iterables):
self._iterables = list(iterables)
self._wakeup = asyncio.Event()
def _add_iters(self, next_futs, on_done):
for it in self._iterables:
it = it.__aiter__()
nfut = asyncio.ensure_future(it.__anext__())
nfut.add_done_callback(on_done)
next_futs[nfut] = it
del self._iterables[:]
return next_futs
async def __aiter__(self):
done = {}
next_futs = {}
def on_done(nfut):
done[nfut] = next_futs.pop(nfut)
self._wakeup.set()
self._add_iters(next_futs, on_done)
try:
while next_futs:
await self._wakeup.wait()
self._wakeup.clear()
for nfut, it in done.items():
try:
ret = nfut.result()
except StopAsyncIteration:
continue
self._iterables.append(it)
yield ret
done.clear()
if self._iterables:
self._add_iters(next_futs, on_done)
finally:
# if the generator exits with an exception, or if the caller stops
# iterating, make sure our callbacks are removed
for nfut in next_futs:
nfut.remove_done_callback(on_done)
def append_iter(self, new_iter):
self._iterables.append(new_iter)
self._wakeup.set()
The only change required for your sample code is that the method is named append_iter, not merge.
This can be done using stream.flatten with an asyncio queue to store the new generators.
import asyncio
from aiostream import stream, pipe
async def main():
queue = asyncio.Queue()
await queue.put(go())
await queue.put(go())
await queue.put(go())
xs = stream.call(queue.get)
ys = stream.cycle(xs)
zs = stream.flatten(ys, task_limit=5)
async with zs.stream() as streamer:
async for item in streamer:
if item == 50:
await queue.put(go())
print(item)
Notice that you may tune the number of tasks that can run at the same time using the task_limit argument. Also note that zs can be elegantly defined using the pipe syntax:
zs = stream.call(queue.get) | pipe.cycle() | pipe.flatten(task_limit=5)
Disclaimer: I am the project maintainer.

Timer runs only once in python

The following program print hello world only once instead it has to print the string for every 5 seconds.
from threading import Timer;
class TestTimer:
def __init__(self):
self.t1 = Timer(5.0, self.foo);
def startTimer(self):
self.t1.start();
def foo(self):
print("Hello, World!!!");
timer = TestTimer();
timer.startTimer();
(program - 1)
But the following program prints the string for every 5 seconds.
def foo():
print("World");
Timer(5.0, foo).start();
foo();
(program - 2)
Why (program - 1) not printing the string for every 5 seconds ?. And how to make the (program - 1) to print the string for every 5 seconds continuously.
(program - 2) prints a string every 5 seconds because it is calling itself recursively. As you can see, you call foo() function inside itself and this is the reason because it works.
If you want to print a string every 5 secs in (program - 1) using a class you could (but it's not really a good practice!):
from threading import Timer
class TestTimer:
def boo(self):
print("World")
Timer(1.0, self.boo).start()
timer = TestTimer()
timer.boo()
As has been pointed out, you're calling the foo() recursively:
def foo():
print("World");
Timer(5.0, foo).start(); # Calls foo() again after 5s and so on
foo();
In your question, you've created a wrapper around threading.Timer - I suggest you simply subclass it:
from threading import Timer
class TestTimer(Timer):
def __init__(self, i):
self.running = False
super(TestTimer, self).__init__(i, self.boo)
def boo(self):
print("Hello World")
def stop():
self.running = False
super(TestTimer, self).stop()
def start():
self.running = True
while self.running:
super(TestTimer, self).start()
t = TestTimer(5)
t.start()

Use of asyncio event loop recursively

I am new to asyncio ( used with python3.4 ) and I am not sure I use it as one should. I have seen in this thread that it can be use to execute a function every n seconds (in my case ms) without having to dive into threading.
I use it to get data from laser sensors through a basic serial protocol every n ms until I get m samples.
Here is the definition of my functions :
def countDown( self,
loop,
funcToDo,
*args,
counter = [ 1 ],
**kwargs ):
""" At every call, it executes funcToDo ( pass it args and kwargs )
and count down from counter to 0. Then, it stop loop """
if counter[ 0 ] == 0:
loop.stop()
else:
funcToDo( *args, **kwargs )
counter[ 0 ] -= 1
def _frangeGen( self, start = 0, stop = None, step = 1 ):
""" use to generate a time frange from start to stop by step step """
while stop is None or start < stop:
yield start
start += step
def callEvery( self,
loop,
interval,
funcToCall,
*args,
now = True,
**kwargs ):
""" repeat funcToCall every interval sec in loop object """
nb = kwargs.get( 'counter', [ 1000 ] )
def repeat( now = True,
times = self._frangeGen( start = loop.time(),
stop=loop.time()+nb[0]*interval,
step = interval ) ):
if now:
funcToCall( *args, **kwargs )
loop.call_at( next( times ), repeat )
repeat( now = now )
And this is how I use it (getAllData is the function that manage serial communication) :
ts = 0.01
nbOfSamples = 1000
loop = asyncio.get_event_loop()
callEvery( loop, ts, countDown, loop, getAllData, counter = [nbOfSamples] )
loop.run_forever()
I want to put that bloc into a function and call it as often as I want, something like this :
for i in range( nbOfMeasures ):
myFunction()
processData()
But the second test does not call getAllData 1000 times, only twice, sometimes thrice. The interesting fact is one time in two I get as much data as I want. I don't really understand, and I can't find anything in the docs, so I am asking for your help. Any explanation or an easier way to do it is gladly welcome :)
You are complicating things too much and, generally speaking, doing recursion when you have an event loop is bad design.
asyncio is fun only when you make use of coroutines. Here's one way of doing it:
import asyncio as aio
def get_laser_data():
"""
get data from the laser using blocking IO
"""
...
#aio.coroutine
def get_samples(loop, m, n):
"""
loop = asyncio event loop
m = number of samples
n = time between samples
"""
samples = []
while len(samples) < m:
sample = yield from loop.run_in_executor(None, get_laser_data)
samples.append(sample)
yield from aio.sleep(n)
return samples
#aio.coroutine
def main(loop):
for i in range(nbOfMeasures):
samples = yield from get_samples(loop, 1000, 0.01)
...
loop = aio.get_event_loop()
loop.run_until_complete(main(loop))
loop.close()
If you are completely confused by this, consider reading some tutorials/documentation about asyncio.
But I would like to point out that you must use a thread to get the data from the laser sensor. Doing any blocking IO in the same thread that the event loop is running will block the loop and throw off aio.sleep. This is what yield from loop.run_in_executor(None, get_laser_data) is doing. It's running the get_laser_data function in a separate thread.
In python 3.5, you can make use of the async for syntax and create an asynchronous iterator to control your time frames. It has to implement the __aiter__ and __anext__ methods:
class timeframes(collections.AsyncIterator):
def __init__(self, steps, delay=1.0, *, loop=None):
self.loop = asyncio.get_event_loop() if loop is None else loop
self.ref = self.loop.time()
self.delay = delay
self.steps = steps
self.iter = iter(range(steps))
async def __anext__(self):
try:
when = self.ref + next(self.iter) * self.delay
except StopIteration:
raise StopAsyncIteration
else:
future = asyncio.Future()
self.loop.call_at(when, future.set_result, None)
await future
return self.loop.time()
async def __aiter__(self):
return self
Here's a coroutine that simulates an execution:
async def simulate(steps, delay, execution):
# Prepare timing
start = loop.time()
expected = steps * delay - delay + execution
# Run simulation
async for t in timeframes(steps, delay):
await loop.run_in_executor(None, time.sleep, execution)
# Return error
result = loop.time() - start
return result - expected
And this is the kind of result you'll get on a linux OS:
>>> loop = asyncio.get_event_loop()
>>> simulation = simulate(steps=1000, delay=0.020, execution=0.014)
>>> error = loop.run_until_complete(simulation)
>>> print("Overall error = {:.3f} ms".format(error * 1000))
Overall error = 1.199 ms
It is different on a windows OS (see this answer) but the event loop will catch up and the overall error should never exceed 15 ms.

Resources