I am using pool.imap_unordered to apply a function over different txt files saved locally.
Is it possible to capture the exception and pass?
If my code runs into an exception, it blocks the entire loop.
pool = Pool(processes=15)
results = {}
files = glob.glob('{}/10K_files/*.txt'.format(path_input))
for key, output in tqdm(pool.imap_unordered(process_file, files),total=len(files)):
results[key] = output
I've tried something like this:
pool = Pool(processes=15)
results = {}
files = glob.glob('{}/10K_files/*.txt'.format(path_input))
try:
for key, output in tqdm(pool.imap_unordered(process_file, files), total=len(files)):
results[key] = output
except:
print("error")
but then I want to resume the loop from where I started.
Thanks!
You could catch the exception in process_file and return it. Then test for whether the return value is an exception. Here is an example:
import os
import traceback
import multiprocessing as mp
def main():
work_items = [i for i in range(20)]
pool = mp.Pool()
for result in pool.imap_unordered(process_file_exc, work_items):
if isinstance(result, Exception):
print("Got exception: {}".format(result))
else:
print("Got OK result: {}".format(result))
def process_file_exc(work_item):
try:
return process_file(work_item)
except Exception as ex:
return Exception("Err on item {}".format(work_item)
+ os.linesep + traceback.format_exc())
def process_file(work_item):
if work_item == 9:
# this will raise ZeroDivisionError exception
return work_item / 0
return "{} * 2 == {}".format(work_item, work_item * 2)
if __name__ == '__main__':
main()
Related
I am following the David Beazley Python Concurrency From the Ground Up: LIVE! - PyCon 2015 talk where I aim to understand basics of Python concurrency.
I would like to know why Python select is failing which is used for polling the operating system (OS) and check if there is some task that needs to be done.
from socket import *
from select import select
from collections import deque
tasks = deque()
recv_wait = {}
send_wait = {}
def fib(n):
if n <= 2:
return 1
else:
return fib(n-1)+fib(n-2)
def run():
while any([tasks, recv_wait, send_wait]):
while not tasks:
can_recv, can_send, _ = select(recv_wait, send_wait, [])
for s in can_recv:
tasks.append(recv_wait.pop(s))
for s in can_send:
tasks.append(send_wait.pop(s))
task = tasks.popleft()
try:
why, what = next(task)
if why == 'recv':
recv_wait[what] = task
elif why == 'send':
send_wait[what] = task
else:
raise RuntimeError("Arg!!")
except StopIteration:
print("task done")
def fib_server(address):
sock = socket(AF_INET, SOCK_STREAM)
sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
sock.bind(address)
sock.listen(5)
while True:
yield 'recv', sock
client, addr = sock.accept()
print("Connection", addr)
tasks.append(fib_handler(client))
def fib_handler(client):
while True:
yield 'recv', client
req = client.recv(100)
if not req:
break
n = int(req)
result = fib(n)
result = fib(n)
resp = str(result).encode('ascii') + b'\n'
yield 'send',resp
client.send(resp)
print("Closed")
tasks.append(fib_server(('', 25000)))
run()
# Separate terminal window
nc localhost 25000
12
# Running Python server
➜ python3 -i aserver.py
Connection ('127.0.0.1', 61098)
Traceback (most recent call last):
File "aserver.py", line 63, in <module>
run()
File "aserver.py", line 20, in run
can_recv, can_send, _ = select(recv_wait, send_wait,[])
TypeError: argument must be an int, or have a fileno() method.
def fib_handler(client):
while True:
yield 'recv', client
req = client.recv(100)
if not req:
break
n = int(req)
result = fib(n)
result = fib(n) # duplicate
resp = str(result).encode('ascii')+ b'\n'
yield 'send',resp # should be yielding fd (client)
client.send(resp)
print("Closed")
How can we capture the print statement within an except when the exception is mocked?
In the example below, I am mocking the make_request_and_get_response using side_effect.
Then, using the pytest.raises I assert that this exception was raised. But, when trying to assert that the corresponding print statement is being output by using capsys to capture it, nothing is captured.
Am I correct to expect that since I am mocking the exception the corresponding print statement will be executed?
import pytest
from unittest import mock
from foo.main import get_response_from_external_api
def test_url_error(capsys):
mock_query = mock.Mock()
mock_query.make_request_and_get_response.side_effect = Exception('HTTPError')
with pytest.raises(Exception) as excinfo:
get_response_from_external_api(mock_query)
assert str(excinfo.value) == 'HTTPError '
out, err = capsys.readouterr()
assert "Got a HTTPError" in out # AssertionError
query.py
from foo.query import *
def get_response_from_external_api(query):
response = 0
try:
response = query.make_request_and_get_response()
except urllib.error.HTTPError as e:
print('Got a HTTPError: ', e)
return response
if __name__ == "__main__":
query = Query('input A', 'input B')
result = get_response_from_external_api(query)
return result
main.py
from foo.query import *
def get_response_from_external_api(query):
response = 0
try:
response = query.make_request_and_get_response()
except urllib.error.HTTPError as e:
print('Got a HTTPError: ', e)
return response
if __name__ == "__main__":
query = Query('input A', 'input B')
result = get_response_from_external_api(query)
return resul
I am running Python ib-api to receive the realtime market data from Interactive Brokers. It can provide the data I expected but it ends with "unhandled exception in EReader thread".
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract as IBcontract
from threading import Thread
import queue
import pandas as pd
from ibapi.ticktype import TickTypeEnum`
`DEFAULT_PRICE_DATA_ID = 1001`
`FINISHED = object()
STARTED = object()
TIME_OUT = object()`
class finishableQueue(object):
def __init__(self, queue_to_finish):
self._queue = queue_to_finish
self.status = STARTED
def get(self, timeout):
contents_of_queue=[]
finished=False
while not finished:
try:
current_element = self._queue.get(timeout=timeout)
if current_element is FINISHED:
finished = True
self.status = FINISHED
else:
contents_of_queue.append(current_element)
except queue.Empty:
finished = True
self.status = TIME_OUT
return contents_of_queue
def timed_out(self):
return self.status is TIME_OUT
class TestWrapper(EWrapper):
def __init__(self):
self._my_price_data_dict = {}
def get_error(self, timeout=5):
if self.is_error():
try:
return self._my_errors.get(timeout=timeout)
except queue.Empty:
return None
return None
def is_error(self):
an_error_if=not self._my_errors.empty()
return an_error_if
def init_error(self):
error_queue=queue.Queue()
self._my_errors = error_queue
def error(self, id, errorCode, errorString):
## Overriden method
errormsg = "IB error id %d errorcode %d string %s" % (id, errorCode, errorString)
self._my_errors.put(errormsg)
def init_ibprices(self, tickerid):
ibprice_data_queue = self._my_price_data_dict[tickerid] = queue.Queue()
return ibprice_data_queue
def tickPrice(self, reqId, tickType, price, attrib):
tickdata = (TickTypeEnum.to_str(tickType), price)
price_data_dict = self._my_price_data_dict
if reqId not in price_data_dict.keys():
self.init_ibprices(reqId)
price_data_dict[reqId].put(tickdata)
class TestClient(EClient):
def __init__(self, wrapper):
EClient.__init__(self, wrapper)
def error(self, reqId, errorCode, errorString):
print("Error: ", reqId, " ", errorCode, " ", errorString)
def getIBrealtimedata(self, ibcontract, tickerid=DEFAULT_PRICE_DATA_ID):
ib_data_queue = finishableQueue(self.init_ibprices(tickerid))
self.reqMktData(
tickerid,
ibcontract,
"",
False,
False,
[]
)
MAX_WAIT_SECONDS = 5
print("Getting data from the server... could take %d seconds to complete " % MAX_WAIT_SECONDS)
price_data = ib_data_queue.get(timeout = MAX_WAIT_SECONDS)
while self.wrapper.is_error():
print(self.get_error())
if ib_data_queue.timed_out():
print("Exceeded maximum wait for wrapper to confirm finished - seems to be normal behaviour")
self.cancelMktData(tickerid)
return price_data
class TestApp(TestWrapper, TestClient):
def __init__(self, ipaddress, portid, clientid):
TestWrapper.__init__(self)
TestClient.__init__(self, wrapper=self)
self.connect(ipaddress, portid, clientid)
thread = Thread(target = self.run)
thread.start()
setattr(self, "_thread", thread)
self.init_error()
def main(slist):
app = TestApp("127.0.0.1", 7497, 1)
for i in slist:
ibcontract = IBcontract()
ibcontract.secType = "STK"
ibcontract.symbol = i
ibcontract.exchange ="SEHK"
Lastprice = app.getIBrealtimedata(ibcontract)
df = pd.DataFrame(Lastprice)
print(ibcontract.symbol, df.head())
app.disconnect()
if __name__ == "__main__":
seclist = [700,2318,5,12]
main(seclist)
Here are the error messages:
unhandled exception in EReader thread
Traceback (most recent call last):
File "D:\Anaconda3\envs\myweb\lib\site-packages\ibapi\reader.py", line 34, >in run
data = self.conn.recvMsg()
File "D:\Anaconda3\envs\myweb\lib\site-packages\ibapi\connection.py", line >99, in recvMsg
buf = self._recvAllMsg()
File "D:\Anaconda3\envs\myweb\lib\site-packages\ibapi\connection.py", line >119, in _recvAllMsg
buf = self.socket.recv(4096)
OSError: [WinError 10038] An operation was attempted on something that is >not a socket
A separate thread is started to read incoming messages from the socket:
thread = Thread(target = self.run)
thread.start()
But this thread is never stopped, and is still running when you call disconnect(). As a result, it tries to access the socket object which is now None, triggering the error. Try stopping the EReader thread prior to disconnecting by setting done=True.
As a side note, since this error happens at the very end of the program at the disconnection it shouldn't interfere with receiving the expected data.
A workaround to avoid the warning.
Implement this at your EClient / EWrapper subclass:
Create a socket shutdown function:
import socket, time
[...]
def _socketShutdown(self):
self.conn.lock.acquire()
try:
if self.conn.socket is not None:
self.conn.socket.shutdown(socket.SHUT_WR)
finally:
self.conn.lock.release()
Use it before closing the connection:
self._socketShutdown()
time.sleep(1)
self.disconnect()
I got distracted and ended up writing a test framework in python. I'm struggling with a particular problem that this throws up.
During my assertion, I want to throw an exception as a way of bubbling a problem up to the run_test() method without requiring the user to have any knowledge of the framework. The problem is that when I do that, it seems that the try/catch block is not honoured.
Here is a cut down version of my fledgling framework:
# test_framework.py
import inspect
import module_containing_tests
class AssertionException(Exception):
def __init__(self, message):
self.message = message
def __str__(self):
return self.message
def run_test(test_name, test_method):
try:
print(">", test_name)
test_method()
print("Passed")
except AssertionException as error:
print("Failed")
print(str(error))
def assert_true(conditional):
if not conditional:
raise AssertionException("Expected True. Was False")
def test(func):
func.is_test = True
return func
members = inspect.getmembers(module_containing_tests)
for member in members:
if "is_test" in dir(member[1]) and not member[0] == "module_containing_tests":
run_test(member[0], member[1])
The module containing the tests will look like this:
# module_containing_tests.py
from test_framework import *
#test
def passing_test():
assert_true(1 + 2 == 3)
#test
def failing_test():
assert_true(1 + 2 == 5)
The output has all the exception stack tracing in it and it also halts the execution
λ python test_framework.py
> failing_test
Traceback (most recent call last):
File "test_framework.py", line 29, in <module>
run_test(member[0], member[1])
File "test_framework.py", line 13, in run_test
test_method()
File "C:\Git\simpy-test\module_containing_tests.py", line 9, in failing_test
assert_true(1 + 2 == 5)
File "C:\Git\simpy-test\test_framework.py", line 20, in assert_true
raise AssertionException("Expected True. Was False")
test_framework.AssertionException: Expected True. Was False
What I want is something like this:
λ python test_framework.py
> failing_test
Expected True. Was False
Failed
> passing_test
Passed
I think the issue is partly in the circular reference between the two files that might mess up with the visibility of the methods (as somehow explained here) and partly, maybe, in the approach. If you think about how many other testing framework work, you often have 3 elements, the unit to test, the testing framework and a test runner.
So if we try to split everythig folllowing that logic you end up having:
test_framework.py
# test_framework.py
class AssertionException(Exception):
pass
def test(f):
f.is_test = True
return f
def assert_true(conditional):
if not conditional:
raise AssertionException("Expected True. Was False")
test_runner.py
# test_runner.py
import inspect
import unit_test
from test_framework import AssertionException
def run_test(test_name, test_method):
try:
print(">", test_name)
test_method()
print("Passed")
except AssertionException as error:
print("Failed with AssertionException: " + str(error))
except Exception as error:
print("Failed with Exception: " + str(error))
if __name__ == "__main__":
members = inspect.getmembers(unit_test)
for member in members:
if "is_test" in dir(member[1]):
run_test(member[0], member[1])
unit_test.py
# unit_test.py
from test_framework import *
#test
def a_passing_test():
assert_true(1 + 2 == 3)
#test
def z_failing_test():
assert_true(1 + 2 == 5)
With this setup the circular dependency is removed and all the visibility context are respected and the output/behaviour is the expected one.
I hope it helps.
Not sure this is what you want but this works.
Copied from here Hide traceback unless a debug flag is set
Output:
$ ./test_framework.py
> a_passing_test
Passed
> z_failing_test
test_framework.AssertionException: Expected True. Was False
First file:
#!/usr/bin/env python3
#test_framework.py
import inspect
import module_containing_tests
import sys
class AssertionException(Exception):
def __init__(self, message):
self.message = message
def __str__(self):
return self.message
def run_test(test_name, test_method):
try:
print(">", test_name)
test_method()
print("Passed")
except AssertionException as error:
print("Failed")
print(str(error))
def assert_true(conditional):
if not conditional:
raise AssertionException("Expected True. Was False")
def test(func):
func.is_test = True
return func
sys.tracebacklimit=0
members = inspect.getmembers(module_containing_tests)
for member in members:
if "is_test" in dir(member[1]) and not member[0] == "module_containing_tests":
run_test(member[0], member[1])
second file:
#!/usr/bin/env python3
#module_containing_tests.py
from test_framework import *
#test
def a_passing_test():
assert_true(1 + 2 == 3)
#test
def z_failing_test():
assert_true(1 + 2 == 5)
I am running a small unittest to check the roman number converter. Here is my code :-
class RomConverter(object):
def __init__(self):
self.digital_mapping = {"M":1000, "D":500, "C":100, "L":50, "X":10, "V":5, "I":1}
def convert(self, rom_num):
value = 0
for char in rom_num:
val += self.digital_mapping[char]
return value
import unittest
class RomConverterTest(unittest.TestCase):
def settingUp(self):
print ("Creating a new RomConverter...")
self.cvt = RomConverter()
def tearDown(self):
print ("Destroying the RomConverter...")
self.cvt = None
def test_empty_num(self):
self.assertTrue(self.cvt.convert("") == 0)
self.assertFalse(self.cvt.convert("") > 0)
def test_no_rom_num(self):
self.assertRaises(TypeError,self.cvt.convert, None)
if __name__ == "__main__":
unittest.main()
But I am getting this message when I run the code :-
Traceback (most recent call last):
File "receipe2 - Copy.py", line 31, in test_empty_roman_numeral
self.assertTrue(self.cvt.convert_to_decimal("") == 0)
AssertionError: False is not true
I see two problems in your code.
First def settingUp(self): should be def setUp(self):
And the return of def convert(self, rom_num): is indented to much. In result the method does not return 0 incase of an empty string.
Here is a working version:
class RomConverter(object):
def __init__(self):
self.digital_mapping = {"M":1000, "D":500, "C":100, "L":50, "X":10, "V":5, "I":1}
def convert(self, rom_num):
value = 0
for char in rom_num:
value += self.digital_mapping[char]
return value
import unittest
class RomConverterTest(unittest.TestCase):
def setUp(self):
print ("Creating a new RomConverter...")
self.cvt = RomConverter()
def tearDown(self):
print ("Destroying the RomConverter...")
self.cvt = None
def test_empty_num(self):
self.assertTrue(self.cvt.convert("") == 0)
self.assertFalse(self.cvt.convert("") > 0)
def test_no_rom_num(self):
self.assertRaises(TypeError,self.cvt.convert, None)
if __name__ == "__main__":
unittest.main()