I tried to catch TweepError exception in while - try - except loop but unsuccessful. The following code keeps stop running when TweepError/ RateLimitError occuring.
import tweepy
import time
name_set = ('name1','name2','name3')
result = []
for screen_name in name_set:
while True:
profile = api.get_user(screen_name = screen_name)
try:
print('collecting user %s'%screen_name)
result.append(profile)
break
except tweepy.RateLimitError:
print('sleep 15 minutes')
sleep(900)
continue
except tweepy.TweepError as e:
print(e)
print('Account %s'%screen_name)
break
else:
print('Account %s'%screen_name)
break
TweepError
TweepError: [{'message': 'User not found.', 'code': 50}]
You should put a API call statement in the try block to catch a exception:
try:
profile = api.get_user(screen_name = screen_name)
print('collecting user %s'%screen_name)
...
Related
I am trying to have try and except block in my code to catch an error, then put it to sleep for 5 seconds and then I want to continue where it left off. Following is my code and currently as soon as it catches exception, it does not continue and stops after exception.
from botocore.exceptions import ClientError
tries = 0
try:
for pag_num, page in enumerate(one_submitted_jobs):
if 'NextToken' in page:
print("Token:",pag_num)
else:
print("No Token in page:", pag_num)
except ClientError as exception_obj:
if exception_obj.response['Error']['Code'] == 'ThrottlingException':
print("Throttling Exception Occured.")
print("Retrying.....")
print("Attempt No.: " + str(tries))
time.sleep(5)
tries +=1
else:
raise
How can I make it to continue after exception? Any help would be great.
Note - I am trying to catch AWS's ThrottlingException error in my code.
Following code is for demonstration to #Selcuk to show what I have currently from his answer. Following will be deleted as soon as we agree if I am doing it correct or not.
tries = 1
pag_num = 0
# Only needed if one_submitted_jobs is not an iterator:
one_submitted_jobs = iter(one_submitted_jobs)
while True:
try:
page = next(one_submitted_jobs)
# do things
if 'NextToken' in page:
print("Token: ", pag_num)
else:
print("No Token in page:", pag_num)
pag_num += 1
except StopIteration:
break
except ClientError as exception_obj:
# Sleep if we are being throttled
if exception_obj.response['Error']['Code'] == 'ThrottlingException':
print("Throttling Exception Occured.")
print("Retrying.....")
print("Attempt No.: " + str(tries))
time.sleep(3)
tries +=1
You are not able to keep running because the exception occurs in your for line. This is a bit tricky because in this case the for statement has no way of knowing if there are more items to process or not.
A workaround could be to use a while loop instead:
pag_num = 0
# Only needed if one_submitted_jobs is not an iterator:
one_submitted_jobs = iter(one_submitted_jobs)
while True:
try:
page = next(one_submitted_jobs)
# do things
pag_num += 1
except StopIteration:
break
except ClientError as exception_obj:
# Sleep if we are being throttled
Is it possible to capture the errors generated and return them to the caller script?
p1.py
def fun1():
try:
...
except Exception as err:
print('ERROR from p1.py')
sys.exit(1)
p2.py
import p1
def fun2():
try:
...
except Exception as err:
print('ERROR from p2.py')
sys.exit(1)
p3.py
import p2
#want to catch the errors here
errors1 = 'ERROR from p1.py'
errors2 = 'ERROR from p2.py'
Is there a way to get the errors from either p1.py ('ERROR from p1.py') or p2.py ('ERROR from p2.py') in p3.py?
One way could be chaging where you handle the exception when you execute the functions of the p1.py and p2.py file in the p3.py file, you can do a try-except in p3.py(instead of catching the exception inside of every function):
p3.py
from p1 import fun1
from p2 import fun2
# calling the function from p1.py
try:
fun1()
except Exception as err:
print('You have the error from fun1 in p1.py, you can make your logic here')
sys.exit(1)
# calling function from p2.py
try:
fun2()
except Exception as err:
print('You have the error from fun2 in p2.py, you can make your logic here')
sys.exit(1)
How can we capture the print statement within an except when the exception is mocked?
In the example below, I am mocking the make_request_and_get_response using side_effect.
Then, using the pytest.raises I assert that this exception was raised. But, when trying to assert that the corresponding print statement is being output by using capsys to capture it, nothing is captured.
Am I correct to expect that since I am mocking the exception the corresponding print statement will be executed?
import pytest
from unittest import mock
from foo.main import get_response_from_external_api
def test_url_error(capsys):
mock_query = mock.Mock()
mock_query.make_request_and_get_response.side_effect = Exception('HTTPError')
with pytest.raises(Exception) as excinfo:
get_response_from_external_api(mock_query)
assert str(excinfo.value) == 'HTTPError '
out, err = capsys.readouterr()
assert "Got a HTTPError" in out # AssertionError
query.py
from foo.query import *
def get_response_from_external_api(query):
response = 0
try:
response = query.make_request_and_get_response()
except urllib.error.HTTPError as e:
print('Got a HTTPError: ', e)
return response
if __name__ == "__main__":
query = Query('input A', 'input B')
result = get_response_from_external_api(query)
return result
main.py
from foo.query import *
def get_response_from_external_api(query):
response = 0
try:
response = query.make_request_and_get_response()
except urllib.error.HTTPError as e:
print('Got a HTTPError: ', e)
return response
if __name__ == "__main__":
query = Query('input A', 'input B')
result = get_response_from_external_api(query)
return resul
I am using pool.imap_unordered to apply a function over different txt files saved locally.
Is it possible to capture the exception and pass?
If my code runs into an exception, it blocks the entire loop.
pool = Pool(processes=15)
results = {}
files = glob.glob('{}/10K_files/*.txt'.format(path_input))
for key, output in tqdm(pool.imap_unordered(process_file, files),total=len(files)):
results[key] = output
I've tried something like this:
pool = Pool(processes=15)
results = {}
files = glob.glob('{}/10K_files/*.txt'.format(path_input))
try:
for key, output in tqdm(pool.imap_unordered(process_file, files), total=len(files)):
results[key] = output
except:
print("error")
but then I want to resume the loop from where I started.
Thanks!
You could catch the exception in process_file and return it. Then test for whether the return value is an exception. Here is an example:
import os
import traceback
import multiprocessing as mp
def main():
work_items = [i for i in range(20)]
pool = mp.Pool()
for result in pool.imap_unordered(process_file_exc, work_items):
if isinstance(result, Exception):
print("Got exception: {}".format(result))
else:
print("Got OK result: {}".format(result))
def process_file_exc(work_item):
try:
return process_file(work_item)
except Exception as ex:
return Exception("Err on item {}".format(work_item)
+ os.linesep + traceback.format_exc())
def process_file(work_item):
if work_item == 9:
# this will raise ZeroDivisionError exception
return work_item / 0
return "{} * 2 == {}".format(work_item, work_item * 2)
if __name__ == '__main__':
main()
I have a list of IDs that I am passing into a URL within a for loop:
L = [1,2,3]
lst=[]
for i in L:
url = 'URL.Id={}'.format(i)
xml_data1 = requests.get(url).text
lst.append(xml_data1)
time.sleep(1)
print(xml_data1)
I am trying to create a try/catch where regardless of the error, the request library keeps trying to connect to the URL from the ID it left off on from the list (L), how would I do this?
I setup this try/catch from this answer (Correct way to try/except using Python requests module?)
However this forces the system to exit.
try:
for i in L:
url = 'URL.Id={}'.format(i)
xml_data1 = requests.get(url).text
lst.append(xml_data1)
time.sleep(1)
print(xml_data1)
except requests.exceptions.RequestException as e:
print (e)
sys.exit(1)
You can put the try-except block in a loop and only break the loop when the request does not raise an exception:
L = [1,2,3]
lst=[]
for i in L:
url = 'URL.Id={}'.format(i)
while True:
try:
xml_data1 = requests.get(url).text
break
except requests.exceptions.RequestException as e:
print(e)
lst.append(xml_data1)
time.sleep(1)
print(xml_data1)