I'm trying to pull some api data and save it for later use
How would I properly handle errors, with this code block:
# import modules
import requests
import json
#test api data
url='https://pipl.ir/v1/getPerson'
#error handling
try:
url_response = requests.get(url,timeout=3)
url_response.raise_for_status()
except requests.exceptions.HTTPError as errh:
print ("Http Error:",errh)
except requests.exceptions.ConnectionError as errc:
print ("Error Connecting:",errc)
except requests.exceptions.Timeout as errt:
print ("Timeout Error:",errt)
except requests.exceptions.RequestException as err:
print ("OOps: Something Else",err)
#json dictionary
json_data = url_response.json()
#print api json response
print(json.dumps(json_data, indent=4, sort_keys=True))
This works if I get a valid json response, if not, I get a return like
Http Error: 404 Client Error: Not Found for url: https://google.com/fakesite
Traceback (most recent call last):
File "/home/telendrith/python/blapi.py", line 19, in <module>
json_data = url_response.json()
File "/usr/lib/python3/dist-packages/requests/models.py", line 897, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/lib/python3/dist-packages/simplejson/__init__.py", line 518, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3/dist-packages/simplejson/decoder.py", line 370, in decode
obj, end = self.raw_decode(s)
File "/usr/lib/python3/dist-packages/simplejson/decoder.py", line 400, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
And .. I'm back at square nothing.
Related
I have a TornadoAPI for keras model, is it possible to check concurrency using python multiprocessing module, I tried with below code, but it throws error
from multiprocessing import Pool
import requests, json
url = 'http://localhost:8888/index/predict'
payload = { "colA":"some1", "colB":"some2",...….)
pl = json.dumps(payload)
def callAPI(x):
session = requests.Session()
r = session.post(url, json=json.loads(pl))
response = r.json()
return response
if __name__ == '__main__':
Pool(processes=15).map(callAPI, range(5))
Error
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 44, in mapstar
return list(map(*args))
File "C:\Models\CallThreadsCheck.py", line 40, in callAPI
response = r.json()
File "C:\Users\HZ\Anaconda3\lib\site-packages\requests\models.py", line 897, in json
return complexjson.loads(self.text, **kwargs)
File "C:\Users\HZ\Anaconda3\lib\site-packages\simplejson\__init__.py", line 525, in loads
return _default_decoder.decode(s)
File "C:\Users\HZ\Anaconda3\lib\site-packages\simplejson\decoder.py", line 370, in decode
obj, end = self.raw_decode(s)
File "C:\Users\HZ\Anaconda3\lib\site-packages\simplejson\decoder.py", line 400, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "CallThreadsCheck.py", line 49, in <module>
p.map(callAPI, range(calls))
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 657, in get
raise self._value
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
May I know what's this error about pls.
I was following a tutorial and when using request.urlopen(url) I get an error, I have tried checking the URL
(https://www.wsj.com/market-data/quotes/PH/XPHS/JFC/historical-prices/download?MOD_VIEW=page&num_rows=150&range_days=150&startDate=06/01/2020&endDate=07/05/2020)
and it's fine.
Here is my code:
from urllib import request
import datetime
def download_stock_from_day_until_today(stock_code, start_date):
current_day = datetime.date.today()
formatted_current_day = datetime.date.strftime(current_day, "%m/%d/%Y") #formats today's date for links
#formatted url
url = "https://www.wsj.com/market-data/quotes/PH/XPHS/"+ stock_code +"/historical-prices/download?MOD_VIEW=page&num_rows=150&range_days=150&startDate="+ start_date +"&endDate=" + formatted_current_day
print(url)
response = request.urlopen(url) #requests the csv file
csv = response.read() #reads the csv file
csv_str = str(csv)
lines = csv_str.split("\\n")
dest_url = r'asd.csv'
fx = open(dest_url, "w")
for line in lines:
fx.write(line + "\n")
fx.close()
download_stock_from_day_until_today("JFC", "06/01/2020")
and the error I get in the console is:
Traceback (most recent call last):
File "C:/Users/Lathrix/PycharmProject/StockExcelDownloader/main.py", line 23, in <module>
download_stock_from_day_until_today("JFC", "06/01/2020")
File "C:/Users/Lathrix/PycharmProject/StockExcelDownloader/main.py", line 12, in download_stock_from_day_until_today
response = request.urlopen(url) #requests the csv file
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 531, in open
response = meth(req, response)
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 640, in http_response
response = self.parent.error(
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 569, in error
return self._call_chain(*args)
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 502, in _call_chain
result = func(*args)
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 649, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found
Looks like wsj.com does not like urllib's User-Agent.
With the line
response = request.urlopen(request.Request(url,headers={'User-Agent': 'Mozilla/5.0'}))
your code works correctly
It seems the task is easy but...
I have simple PIL.Image object. How to make Cherrypy response with this image dynamically?
def get_image(self, data_id):
cherrypy.response.headers['Content-Type'] = 'image/png'
img = PIL.Image.frombytes(...)
buffer = io.StringIO()
img.save(buffer, 'PNG')
return buffer.getvalue()
This code gives me:
500 Internal Server Error
The server encountered an unexpected condition which prevented it from fulfilling the request.
Traceback (most recent call last):
File "C:\Users\Serge\AppData\Local\Programs\Python\Python36\lib\site-packages\cherrypy\_cprequest.py", line 631, in respond
self._do_respond(path_info)
File "C:\Users\Serge\AppData\Local\Programs\Python\Python36\lib\site-packages\cherrypy\_cprequest.py", line 690, in _do_respond
response.body = self.handler()
File "C:\Users\Serge\AppData\Local\Programs\Python\Python36\lib\site-packages\cherrypy\_cpdispatch.py", line 60, in __call__
return self.callable(*self.args, **self.kwargs)
File "D:\Dev\Bf\webapp\controllers\calculation.py", line 69, in get_image
img.save(buffer, 'PNG')
File "C:\Users\Serge\AppData\Local\Programs\Python\Python36\lib\site-packages\PIL\Image.py", line 1930, in save
save_handler(self, fp, filename)
File "C:\Users\Serge\AppData\Local\Programs\Python\Python36\lib\site-packages\PIL\PngImagePlugin.py", line 731, in _save
fp.write(_MAGIC)
TypeError: string argument expected, got 'bytes'
Can someone help me please?
Use io.BytesIO() instead of io.StringIO(). (From this answer.)
I am not a python guru I am just writing a code to check my api authentication and URL access status. I just want to assure that my api and domain url is accessible to users.
For the above reason I am writing a python script which can check and a cron can send an alert to me.
Here is my Code:
def check(argv):
# I'm going to use argpase.It makes
# command-line args a breeze.
parser = argparse.ArgumentParser()
parser.add_argument('-H', '--hostname', dest='hostname', required=True)
parser.add_argument('-a', '--auth_id', dest='authenticationid')
parser.add_argument('-t', '--auth_token', dest='authenticationtoken')
parser.add_argument('-r', '--dest_url', dest='dest_url',help="""Path to report relative to root, like /v1/ OR /""", required=True)
parser.add_argument("-q", "--quiet", action="store_false", dest="verbose", default=True,
help="don't print status messages to stdout")
args = vars(parser.parse_args())
if args['authenticationid'] and args['authenticationtoken'] and not len(sys.argv) == 7:
authurl = urllib.request.Request('https://{%s}:{%s}#%s%s/%s/' %(args['authenticationid'],args['authenticationtoken'],args['hostname'], args['dest_url'],args['authenticationid']))
return (getAuthResponseCode(authurl))
else:
url = urllib.request.Request("https://%s%s" %(args['hostname'], args['dest_url']))
return(getResponseCode(url))
def getResponseCode(url):
try:
conn = urllib.request.urlopen(url,timeout=10)
code = conn.getcode()
return (status['OK'], code)
except timeout:
return (status['WARNING'], logging.error('socket timed out - URL %s', url))
except urllib.error.URLError as e:
return (status['CRITICAL'], e.reason)
else:
return (status['UNKNOWN'])
def getAuthResponseCode(authurl):
try:
authconn = urllib.request.urlopen(authurl, timeout=10)
authcode = authconn.getcode()
return (status['OK'], authcode)
except timeout:
return (status['WARNING'], logging.error('socket timed out - URL %s'))
except urllib.error.URLError as err:
return (status['CRITICAL'], err.reason)
else:
return (status['UNKNOWN'])
ERROR Message:
G:\Python>python check_http.py -H api.mydomain.com -r /API/Function/ -a 'MAMZMZZGVLMG
FMNTHIYTREETBESSS' -t 'DafniisfnsifnsifsbANBBDSDNBISDExODZlODAwMmZm'
Traceback (most recent call last):
File "C:\Python33\lib\http\client.py", line 770, in _set_hostport
port = int(host[i+1:])
ValueError: invalid literal for int() with base 10: "{'DafniisfnsifnsifsbANBBDSDNBISDExODZlODAw
MmZm'}#api.mydomain.com"
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "check_http.py", line 76, in <module>
print (check(sys.argv[1:]))
File "check_http.py", line 41, in check
return (getAuthResponseCode(authurl))
File "check_http.py", line 61, in getAuthResponseCode
authconn = urllib.request.urlopen(authurl, timeout=10)
File "C:\Python33\lib\urllib\request.py", line 156, in urlopen
return opener.open(url, data, timeout)
File "C:\Python33\lib\urllib\request.py", line 469, in open
response = self._open(req, data)
File "C:\Python33\lib\urllib\request.py", line 487, in _open
'_open', req)
File "C:\Python33\lib\urllib\request.py", line 447, in _call_chain
result = func(*args)
File "C:\Python33\lib\urllib\request.py", line 1283, in https_open
context=self._context, check_hostname=self._check_hostname)
File "C:\Python33\lib\urllib\request.py", line 1219, in do_open
h = http_class(host, timeout=req.timeout, **http_conn_args)
File "C:\Python33\lib\http\client.py", line 1172, in __init__
source_address)
File "C:\Python33\lib\http\client.py", line 749, in __init__
self._set_hostport(host, port)
File "C:\Python33\lib\http\client.py", line 775, in _set_hostport
raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
http.client.InvalidURL: nonnumeric port: '{'DafniisfnsifnsifsbANBBDSDNBISDExODZlODAw
MmZm'}#api.mydomain.com'
I know this is not write my code forum but I am helpless and looking for some help.If some one can tell where exactly I went wrong so that I can fix it.
I am using python3.
You're passing ʹhttps://user:pass#whateverʹ as the url.
Python doesn't understand you're trying to authenticate, and thinks you're passing ʹhttps://domain:port...ʹ
To do basic auth with urllib, you need to use a urllib.request.HTTPBasicAuthHandler
Sorry I'm not posting links and/or example code, but I'm typing this on my phone, and it makes those a pain.
I tried to write a simple server-client program using webob.
1. The client send data using 'Transfer-Encoding', 'chunked'
2. the received data is then print in the server side.
The Server.py received the data correctly.
However, I got error a bunch of message from webob.
However, anyone kindly tell me what had happened or just
give a simple guide line for write such a simple program(sending chunk) with webob.
thanks!
the codes and error are as below:
server.py
from webob import Request
from webob import Response
class ChunkApp(object):
def __init__(self):
pass
def __call__(self, environ, start_response):
req = Request(environ)
for buf in self.chunkreadable(req.body_file, 65535):
print buf
resp = Response('chunk received')
return resp(environ, start_response)
def chunkreadable(self, iter, chunk_size=65536):
return self.chunkiter(iter, chunk_size) if \
hasattr(iter, 'read') else iter
def chunkiter(self, fp, chunk_size=65536):
while True:
chunk = fp.read(chunk_size)
if chunk:
yield chunk
else:
break
if __name__ == '__main__':
app = ChunkApp()
from wsgiref.simple_server import make_server
httpd = make_server('localhost', 8080, app)
print 'Serving on http://localhost:%s' % '8080'
try:
httpd.serve_forever()
except KeyboardInterrupt:
print '^C'
client.py
from httplib import HTTPConnection
conn = HTTPConnection("localhost:8080")
conn.putrequest('PUT', "/")
body = "1234567890"
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
for chunk in body:
#print '%x\r\n%s\r\n' % (len(chunk), chunk)
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
conn.send('0\r\n\r\n')
Error
Traceback (most recent call last):
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 86, in run
self.finish_response()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 127, in finish_response
self.write(data)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 210, in write
self.send_headers()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 268, in send_headers
self.send_preamble()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 192, in send_preamble
'Date: %s\r\n' % format_date_time(time.time())
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 324, in write
self.flush()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 303, in flush
self._sock.sendall(view[write_offset:write_offset+buffer_size])
error: [Errno 32] Broken pipe
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 52575)
Traceback (most recent call last):
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 284, in _handle_request_noblock
self.process_request(request, client_address)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 310, in process_request
self.finish_request(request, client_address)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 323, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 641, in __init__
self.finish()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 694, in finish
self.wfile.flush()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 303, in flush
self._sock.sendall(view[write_offset:write_offset+buffer_size])
error: [Errno 32] Broken pipe