The Python3 fetch_token method in this library does not check the response status before consuming the response. If the API call it makes fails, then the response will be invalid and the script crashes. Is there something I can set so that an exception will be raised on a non-success response before the library can read the response?
import requests
from requests.auth import HTTPBasicAuth
from requests_oauthlib import OAuth2Session
from oauthlib.oauth2 import BackendApplicationClient
from oauthlib.oauth2 import OAuth2Error
AUTH_TOKEN_URL = "https://httpstat.us/500" # For testing
AUTH = HTTPBasicAuth("anID", "aSecret")
CLIENT = BackendApplicationClient(client_id="anID")
SCOPES = "retailer.orders.write"
MAX_API_RETRIES = 4
class MyApp:
def __init__(self):
"""Initialize ... and obtain initial auth token for request"""
self.client = OAuth2Session(client=CLIENT)
self.client.headers.update(
{
"Content-Type": "application/json"
}
)
self.__authenticate()
def __authenticate(self):
"""Obtain auth token."""
server_errors = 0
# This needs more work. fetch_token is not raising errors but failing
# instead.
while True:
try:
self.token = self.client.fetch_token(
token_url=AUTH_TOKEN_URL, auth=AUTH, scope=SCOPES
)
break
except (OAuth2Error, requests.exceptions.RequestException) as e:
server_errors = MyApp.__process_retry(
server_errors, e, None, MAX_API_RETRIES
)
#staticmethod
def __process_retry(errors, exception, resp, max_retries):
# Log and process retries
# ...
return errors + 1
MyApp() # Try it out
You can add a "compliance hook" that will be passed the Response object from requests before the library attempts to parse it, like so:
def raise_on_error(response):
response.raise_for_status()
return response
self.client.register_compliance_hook('access_token_response', raise_on_error)
Depending on exactly when you may get errors, you might want to do this with 'refresh_token_response' and/or 'protected_request' as well. See the docstring for the register_compliance_hook method for more info.
Related
I made such a minimal example that completely repeats the behavior of my code. I make requests from firefox and chrome. I noticed that after making a request from chrome, firefox ceases to receive answers. After some research, I realized that the server response with error on localhost:8000/favicon.ico path request. After receiving error 404 once, chrome after each response from server creates another connection to the server, but does not send data, which causes a lock on the recv function.
File "/usr/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
I found that I can set the connection timeout for the handler class, it is taken into account in the StreamRequestHandler:r:
if self.timeout is not None:
self.connection.settimeout(self.timeout)
But I am embarrassed that there is no information about this in the documentation
https://docs.python.org/3/library/socketserver.html#socketserver.BaseRequestHandler.handle
import logging
import json
import http.server
from http import HTTPStatus
from typing import Optional
from urllib.parse import urlparse, parse_qs
import socketserver
from threading import Thread
import traceback
from functools import wraps
import sys, os
project_dir = os.path.abspath(os.curdir)
sys.path.append(project_dir)
logging.getLogger().setLevel("DEBUG")
class RESTHandler(http.server.BaseHTTPRequestHandler):
"""
Rest router for api methods
"""
def __init__(self, *args, **kwargs):
logging.info(f"Creating RESTHandler obj. Args: {args}, kwargs: {kwargs}")
super().__init__(*args, **kwargs)
def end_headers(self) -> None:
self.send_header('Access-Control-Allow-Origin', '*')
http.server.BaseHTTPRequestHandler.end_headers(self)
# noinspection PyPep8Naming
def do_GET(self):
logging.info(self.path)
url = urlparse(self.path)
if "favicon.ico" in url.path:
self.send_error(HTTPStatus.NOT_FOUND, message='Unknown api path.')
return
self.send_response(HTTPStatus.OK)
self.send_header('Content-Type', 'application/json')
self.end_headers()
self.wfile.write(json.dumps({"resp":"I am OK", "int": 5}, ensure_ascii=False).encode('utf-8'))
class ApiService():
DEFAULT_API_PORT = 8000
DEFAULT_API_HOST = ''
def __init__(self, ui_service = None, host: Optional[str] = None, port: Optional[int] = None):
self.ui_service = ui_service
self.host = host or self.DEFAULT_API_HOST
self.port = port or self.DEFAULT_API_PORT
def _run(self):
while True:
try:
with socketserver.TCPServer((self.host, self.port), RESTHandler, bind_and_activate=False) as httpd:
logging.info("Starting server....")
httpd.allow_reuse_address = True
httpd.server_bind()
httpd.server_activate()
logging.info(f"Serving API at {self.host}:{self.port}")
httpd.serve_forever()
break
except Exception as e:
tb_list = traceback.format_exception( type(e), e, tb=e.__traceback__)
tb_list = [ s.replace("\n", "") for s in tb_list ]
tb_str = "; ".join(tb_list)
logging.error(f"Unexpected exception while http-server was working: {tb_str}")
def run(self, in_thread=True):
if in_thread:
t = Thread(target=self._run)
t.start()
else:
self._run()
if __name__ == '__main__':
ApiService().run(in_thread=False)
I guess, Chrome uses web browsers pre-opening sockets, on which TCPServer would wait indefinitely in my case. But I am still interesting why only after 404 and what about legitimacy of using timeout of request handler.
I am trying to achieve fire_and_forget functionality and here are the details.
Async decorater i am trying to use:
import asyncio
import time
def fire_and_forget(f):
def wrapped(*args, **kwargs):
return asyncio.get_event_loop().run_in_executor(None, f, *args, *kwargs)
return wrapped
My Async call using above decorator:
#fire_and_forget
def call_async(self, req, body, headers_params):
logger.info("Calling ASYNC")
try:
f = urllib.request.urlopen(req)
response = f.read()
f.close()
except Exception as e:
logger.exception("api exception %s" % e)
return None
# Parse and return the response
try:
res = self._parse_response(response)
except Exception as e:
logger.exception("Exception in parsing response of %s" % e)
res = None
logger.debug("clevertap response: {}".format(res))
My Flask app calling test_xyz which in turns fire the above fire and forget call_async:
from flask import Flask, jsonify, request
from call_xyz import test_xyz
app = Flask(__name__)
#app.route('/events_dm')
def events_dm():
session_id = request.args.get('sessionId', "11111111")
test_obj = test_xyz(session_id)
test_obj.track_test()
return jsonify({"success": True})
app.run(
host='0.0.0.0',
port=8091,
debug=True,
threaded=True
)
I am not able to understand where to set my event loop correctly so that i don't get the error: "Error in xyz There is no current event loop in thread 'Thread-7'" and my events get fired correctly.
Flask is not asyncio-compatible, so you shouldn't attempt to use asyncio within it.
Besides, you're not actually using asyncio's functionality, but run_in_executor which calls into concurrent.futures, i.e. uses threads under the hood. If that's what you need, you can create an executor directly and just submit your function to it. For example:
import concurrent.futures
_pool = concurrent.futures.ThreadPoolExecutor()
def fire_and_forget(f):
def wrapped(*args, **kwargs):
return _pool.submit(lambda: f(*args, **kwargs))
return wrapped
You must probably go through once asyncio usage and its main heart event loop understanding.
Similar issue which might help you to understand.
Here is the tutorial with some good explanation
Just give some flow for how use coroutine with normal Flask app here is the sample one.
import asyncio
import datetime
from flask import Flask, jsonify, request
app = Flask(__name__)
def print_now():
print(datetime.datetime.now())
async def keep_printing(name: str="") -> None:
print(name, end=" ")
print_now()
await asyncio.sleep(1.50)
async def main(num_times):
result = await asyncio.gather(
keep_printing("first"),
keep_printing("second"),
keep_printing("third"),
)
return result
def execute_statement():
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(main(2))
loop.close()
#app.route('/events_dm')
def events_dm():
execute_statement()
return jsonify({"success": True})
app.run(
host='0.0.0.0',
port=8091,
debug=True,
threaded=True
)
Output when you hit /events_dm
first 2020-07-18 00:46:26.452921
second 2020-07-18 00:46:26.452921
third 2020-07-18 00:46:26.452921
127.0.0.1 - - [18/Jul/2020 00:46:27] "GET /events_dm HTTP/1.1" 200 -
I am trying to create a middleware in flask to store request and response inside database. I am able to access data present inside request but unable to figure out how to access data present inside response. Response header showing content-length 1000 & if i am changing it then showing differences in the content-length.
import pprint
from werkzeug.wrappers import Request, Response
class LoggerMiddleware:
def __init__(self, application):
self.__application = application
def __call__(self, environ, start_response):
errors = environ['wsgi.errors']
length = int(environ.get('CONTENT_LENGTH', '0'))
data = environ['wsgi.input'].read(min(4096, length))
if data:
print(data)
pprint.pprint(('REQUEST', environ), stream=errors)
def _start_response(status, headers, *args):
pprint.pprint(('RESPONSE', status, headers), stream=errors)
return start_response(status, headers, *args)
return self.__application(environ, _start_response)
import tornado.web
import tornado.ioloop
from apiApplicationModel import userData
from cleanArray import Label_Correction
import json
import requests
colName=['age', 'resting_blood_pressure', 'cholesterol', 'max_heart_rate_achieved', 'st_depression', 'num_major_vessels', 'st_slope_downsloping', 'st_slope_flat', 'st_slope_upsloping', 'sex_male', 'chest_pain_type_atypical angina', 'chest_pain_type_non-anginal pain', 'chest_pain_type_typical angina', 'fasting_blood_sugar_lower than 120mg/ml', 'rest_ecg_left ventricular hypertrophy', 'rest_ecg_normal', 'exercise_induced_angina_yes', 'thalassemia_fixed defect', 'thalassemia_normal',
'thalassemia_reversable defect']
class processRequestHandler(tornado.web.RequestHandler):
def post(self):
data_input_array = []
for name in colName:
x = self.get_body_argument(name, default=0)
data_input_array.append(int(x))
label = Label_Correction(data_input_array)
finalResult = int(userData(label))
output = json.dumps({"Giveput":finalResult})
self.write(output)
class basicRequestHandler(tornado.web.RequestHandler):
def get(self):
self.render('report.html')
class staticRequestHandler(tornado.web.RequestHandler):
def post(self):
data_input_array = []
for name in colName:
x = self.get_body_argument(name, default=0)
data_input_array.append(str(x))
send_data = dict(zip(colName, data_input_array))
print(send_data)
print(type(send_data))
url = "http://localhost:8887/output"
headers={}
response = requests.request('POST',url,headers=headers,data=send_data)
print(response.text.encode('utf8'))
print("DONE")
if __name__=='__main__':
app = tornado.web.Application([(r"/",basicRequestHandler),
(r"/result",staticRequestHandler),
(r"/output",processRequestHandler)])
print("API IS RUNNING.......")
app.listen(8887)
tornado.ioloop.IOLoop.current().start()
Actually I am trying to create an API and the result of the API can be used but
The page keeps on loading, and no response is shown.
Response should be a python dictionary send by post function of class processRequestHandler
After using a debugger the lines after response = requests.request('POST',url,headers=headers,data=send_data) are not executed.
The class processRequestHandler is working fine when checked with POSTMAN.
requests.request is a blocking method. This blocks the event loop and prevents any other handlers from running. In a Tornado handler, you need to use Tornado's AsyncHTTPClient (or another non-blocking HTTP client such as aiohttp) instead.
async def post(self):
...
response = await AsyncHTTPClient().fetch(url, method='POST', headers=headers, body=send_data)
See the Tornado users's guide for more information.
I have the following function and it is a generic function which will make API call based on the input hostname and data. It will construct http request to make API and will return the response. This function will throw four types of exception(invalid URL, timeout, auth error and status check). How can I Mcok and Test the exception raised in API call using pytest? Which will be the best method to test the exceptions raised from API call?
import ssl
import urllib
import urllib.request
import urllib.error
import xml
import xml.etree.ElementTree as ET
def call_api(hostname, data):
'''Function to make API call
'''
# Todo:
# Context to separate function?
# check response for status codes and return reponse.read() if success
# Else throw exception and catch it in calling function
error_codes = {
"1": "Unknown command",
"6": "Bad Xpath",
"7": "Object not present",
"8": "Object not unique"
}
url = "http://" + hostname + "/api"
encoded_data = urllib.parse.urlencode(data).encode('utf-8')
try:
response = urllib.request.urlopen(url, data=encoded_data,
timeout=10).read()
root = ET.fromstring(response)
if root.attrib.get('status') != "success":
Errorcode = root.attrib.get('code')
raise Exception(pan_error_codes.get(Errorcode, "UnknownError"),
response)
else:
return response
except urllib.error.HTTPError as e:
raise Exception(f"HttpError: {e.code} {e.reason} at {e.url}", None)
except urllib.error.URLError as e:
raise Exception(f"Urlerror: {e.reason}", None)
If i call this function
def create_key(hostname, username, password):
hostname = 'myhost ip'
data = {
'type': 'keygen',
'username': username,
'password': password
}
username = 'myuser'
password = 'password'
response = call_api(hostname, data)
return response
i will get a response like following
b"<response status = 'success'><result><key>mykey</key></result></response>"
You can mock error raising via side_effect parameter:
Alternatively side_effect can be an exception class or instance. In this case the exception will be raised when the mock is called.
In your case, this can be used like this (assuming call_api is defined in module foo):
import pytest
from unittest.mock import patch
def test_api():
with patch('foo.call_api', side_effect=Exception('mocked error')):
with pytest.raises(Exception) as excinfo:
create_key('localhost:8080', 'spam', 'eggs')
assert excinfo.value.message == 'mocked error'