I want to write a test case for a method in my code mocking the multiprocessing part using pytest.
This the code and I need to write a test for simulation method.
from multiprocessing import Pool
class Simulation:
def __init__(self, num):
self.num = num
def simulation(self):
pool = Pool(processes=self.num)
val = [1, 2, 3, 4, 5]
res = pool.map(self.sim, val)
return res
def sim(self, val):
return val * val
Here is the test case.
import pytest
from multi import Simulation
#pytest.fixture
def sim():
return Simulation(2)
#pytest.fixture
def mock_map():
def map(self, val1=1, val2=2):
return [val1, val2]
return map
def test_sim(sim, mock_map, monkeypatch):
monkeypatch.setattr('multiprocessing.Pool.map', mock_map)
res = sim.simulation()
assert res == [1,2]
When running the test I get the out put as [1,4,6,16,25] where I need to mock the out put as [1,2].
Related
The following returns "NameError: name 'times_2' is not defined", and I can't figure out why:
def pass_data(data): return times_2(data)
def times_2(data): return data*2
import multiprocessing
multiprocessing.pool = Pool()
pool.ncpus = 2
res = pool.map(pass_data, range(5))
print(res)
What I'm actually trying to do is apply a function to a pandas dataframe. However, because I can't use a lambda function to do this:
pool.map(lambda x: x.apply(get_weather, axis=1), df_split)
instead I have this with the following helper methods, but it throws a "NameError: name 'get_weather' is not defined":
def get_weather(df):
*do stuff*
return weather_df
def pass_dataframe(df):
return df.apply(get_weather, axis=1)
results = pool.map(pass_dataframe, df_split)
Try using Pool like this:
from multiprocessing import Pool
def pass_data(data): return times_2(data)
def times_2(data): return data*2
with Pool(processes=4) as pool:
res = pool.map(pass_data, range(5))
print(res)
On Windows:
from multiprocessing import Pool
def pass_data(data): return times_2(data)
def times_2(data): return data*2
if __name__ == '__main__':
with Pool(processes=4) as pool:
res = pool.map(pass_data, range(5))
print(res)
See docs https://docs.python.org/3/library/multiprocessing.html#multiprocessing-programming
I am trying to understand how to test Context managert with pytests.
I created some Class and need to count how much times was called static method do_dome_stuff
class Iterator():
def __init__(self):
pass
#staticmethod
def do_some_stuff():
pass
def __enter__(self):
return [i for i in range(10)]
def __exit__(self, *args):
return True
iterator = Iterator()
def f(iterator):
with iterator as i:
for _ in i:
iterator.do_some_stuff()
I have created py.test file and need to check if function was called 10 times. But my solution isn't working:
#pytest.fixture
def iterator():
return MagicMock(spec=Iterator)
def test_f(iterator):
f(iterator)
assert (iterator.do_some_stuff.call_count == 10)
Thanks in advance
The reason your code doesn't work, is that MagicMock(spec=Iterator) replaces the __enter__ method of your Iterator class by a MagicMock object, see the MagicMock documentation. This means that in your test, the value of i in function f is a MagicMock object instead of list(range(10)), so the code inside the for loop is never executed.
To make it work, you will probably only want to mock the do_some_stuff method:
#pytest.fixture
def iterator():
it = Iterator()
it.do_some_stuff = Mock()
return it
def test_f(iterator):
f(iterator)
assert (iterator.do_some_stuff.call_count == 10)
base_class.py
import threading
import websocket
from abc import ABCMeta, abstractmethod
class A:
__metaclass__ = ABCMeta
def __init__(self):
self.some_var = 0
#abstractmethod
def failed_method(self):
pass
def on_websocket_message(self, ws, msg):
var = self.failed_method()
print(var)
...
def open_websocket(self):
ws = websocket.WebSocketApp('http://some_url.com',
on_message=self.on_websocket_message, ...)
ws.run_forever()
def callback_method(self):
websocket_thread = threading.Thread(target=self.open_websocket, name='some_websocket_name')
another_var = self.failed_method()
print('Another variable\'s value is [{}]'.format(another_var))
child_class.py
from base_class import A
class B(A):
def failed_method(self):
return 3
other_class.py
import threading
from child_class import B
def main():
child_class_instance = B()
some_thread = threadings.Thread(target=child_class_instance.callback_method, name='some_name')
some_thread.start()
The result of main() is printed None, not 3, i.e., abstract class' method is called instead of child's one. (Assume all the modules are in the one place.)
Can anyone explain this behaviour? Or what is the thing I do not understand in inheritance in Python combined with threadings.Thread?
P.S. With the similar code I've met error from callback <bound method BaseClass... of < ...ChildClass...> > from websocket.WebSocketApp._callback().
P.P.S. Important to note that I use websocket-client, not websockets.
I have a code which is listening to messages on WebSocket using aiohttp.
It looks like:
async for msg in ws:
await self._ws_msg_handler.handle_message(ws, msg, _services)
Where ws is an instance of aiohttp.web.WebSocketResponse() (original code)
In my test I mock WebSocketResponse() and its __aiter__ method:
def coro_mock(**kwargs):
return asyncio.coroutine(mock.Mock(**kwargs))
#pytest.mark.asyncio
#mock.patch('aiojsonrpc.request_handler.WebSocketMessageHandler')
async def test_rpc_websocket_handler(
MockWebSocketMessageHandler,
rpc_websocket_handler
):
ws_response = 'aiojsonrpc.request_handler.WebSocketResponse'
with mock.patch(ws_response) as MockWebSocketResponse:
MockRequest = mock.MagicMock()
req = MockRequest()
ws_instance = MockWebSocketResponse.return_value
ws_instance.prepare = coro_mock()
ws_instance.__aiter__ = coro_mock(return_value=iter(range(5)))
ws_instance.__anext__ = coro_mock()
handle_msg_result = 'Message processed'
MockWebSocketMessageHandler.handle_message.side_effect = Exception(
handle_msg_result)
msg_handler = MockWebSocketMessageHandler()
with pytest.raises(Exception) as e:
await request_handler.RpcWebsocketHandler(msg_handler)(req)
assert str(e.value) == handle_msg_result
Though when I run the test it fails with the error message saying:
'async for' requires an object with __aiter__ method, got MagicMock
=================================================================================== FAILURES ===================================================================================
__________________________________________________________________________ test_rpc_websocket_handler __________________________________________________________________________
MockWebSocketMessageHandler = <MagicMock name='WebSocketMessageHandler' id='140687969989632'>
rpc_websocket_handler = <aiojsonrpc.request_handler.RpcWebsocketHandler object at 0x7ff47879b0f0>
#pytest.mark.asyncio
#mock.patch('aiojsonrpc.request_handler.WebSocketMessageHandler')
async def test_rpc_websocket_handler(
MockWebSocketMessageHandler,
rpc_websocket_handler
):
ws_response = 'aiojsonrpc.request_handler.WebSocketResponse'
with mock.patch(ws_response) as MockWebSocketResponse:
# MockRequest = mock.create_autospec(aiohttp.web_reqrep.Request)
# req = MockRequest(*[None] * 6)
MockRequest = mock.MagicMock()
req = MockRequest()
ws_instance = MockWebSocketResponse.return_value
ret = mock.Mock()
ws_instance.prepare = coro_mock()
ws_instance.__aiter__ = coro_mock(return_value=iter(range(5)))
ws_instance.__anext__ = coro_mock()
handle_msg_result = 'Message processed'
MockWebSocketMessageHandler.handle_message.side_effect = Exception(
handle_msg_result)
msg_handler = MockWebSocketMessageHandler()
with pytest.raises(Exception) as e:
await request_handler.RpcWebsocketHandler(msg_handler)(req)
> assert str(e.value) == handle_msg_result
E assert "'async for' ...got MagicMock" == 'Message processed'
E - 'async for' requires an object with __aiter__ method, got MagicMock
E + Message processed
tests/test_request_handler.py:252: AssertionError
So it behaves like __aiter__() was never mocked.
How I'm supposed to accomplish correct mocking in this case?
Update:
For now I've found a workaround to make the code testable though I would really appreciate if someone tell me how to deal with the issue described in the original question.
You can make the mocked class return an object implementing the expected interface:
class AsyncIterator:
def __init__(self, seq):
self.iter = iter(seq)
def __aiter__(self):
return self
async def __anext__(self):
try:
return next(self.iter)
except StopIteration:
raise StopAsyncIteration
MockWebSocketResponse.return_value = AsyncIterator(range(5))
I don't think there is a way (yet) to correctly mock an object implementing __aiter__, it may be a python bug, as async for rejects a MagicMock, even if hasattr(the_magic_mock, '__aiter__') is True.
EDIT (13/12/2017): the library asynctest supports asynchronous iterators and context managers since 0.11, asynctest.MagicMock provides this feature for free.
For posterity, I had the same problem of needing to test an async for loop, but the accepted solution doesn't seem to work for Python 3.7. The example below works for 3.6.x and 3.7.0, but not for 3.5.x:
import asyncio
class AsyncIter:
def __init__(self, items):
self.items = items
async def __aiter__(self):
for item in self.items:
yield item
async def print_iter(items):
async for item in items:
print(item)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
things = AsyncIter([1, 2, 3])
loop.run_until_complete(print_iter(things))
loop.close()
With the above, mocking it looks something like:
with mock.patch('some.async.iter', return_value=AsyncIter([1, 2, 3])):
# do test requiring mocked iter
Works for py38
from unittest.mock import MagicMock
async def test_iterable(self):
loop_iterations = 0
mock = MagicMock()
mock.__aiter__.return_value = range(5)
async for _ in mock:
loop_iterations += 1
self.assertEqual(5, loop_iterations)
I have a python version that supports AsyncMock and I also leverage pytest_mock. I came up with this solution to this problem combining the use of AsyncMock side_effect:
from typing import List
import pytest
import asyncio
from pytest_mock.plugin import MockerFixture
pytestmark = pytest.mark.asyncio
async def async_generator(numbers: List[int]):
for number in numbers:
yield number
await asyncio.sleep(0.1)
async def function_to_test(numbers: List[int]):
async for thing in async_generator(numbers):
yield thing * 3
await asyncio.sleep(0.1)
async def test_async_generator(mocker: MockerFixture):
mock_numbers = [1, 2, 3, 4, 5]
async def async_generator_side_effect(numbers: List[int]):
for number in numbers:
yield number
mock_async_generator = mocker.patch("tests.test_async_generator.async_generator")
mock_async_generator.side_effect = async_generator_side_effect
actual = []
async for result in function_to_test(mock_numbers):
actual.append(result)
assert actual == [3, 6, 9, 12, 15]
I am using pythons multiprocessing module in some of my code. I have a controller class that controls a class and performs some action.
import multiprocessing
from multiprocessing import Queue, Process, Manager
class dosomething(multiprocessing.Process):
def __init__(self,managerList):
self.mlist = managerList
print self.mlist
def run(self):
self.mlist.append((4,5,6))
class doController:
def __init__(self):
mgr = Manager()
self.mlist = mgr.list()
self.mlist.append((1,2,3,4))
t = dosomething(self.mlist)
#t.daemon = True
#t.start()
def printer(self):
return self.mlist
gd = doController()
print gd.printer()
Pring mlist in the init part of dosomething prints [(1, 2, 3, 4)] as expected but the list in the dosomething part does not work giving out IOError 11. Can anyone help if it's right or wrong?
The call to the Process.__init__ is missing.
You don't necessarely need to create a Process subclass you could use functions:
from multiprocessing import Process, Manager
def dosomething(mlist):
mlist.append((4,5,6))
def main():
manager = Manager()
L = manager.list((1,2,3,4))
p = Process(target=dosomething, args=(L,))
p.start()
p.join()
print L
if __name__ == '__main__':
main()