I have an application (written in PyQt5) that returns x, y, and elevation of a location. When the user fills up the x, y, and hits getz button, the app calls the function below:
def getz(self, i):
"""calculates the elevation"""
import urllib
url = "https://api.open-elevation.com/api/v1/lookup"
x = self.lineEditX.text()
y = self.lineEditY.text()
url = url + "\?locations\={},{}".format(x, y)
print(url)
if i is "pushButtonSiteZ":
response = urllib.request.Request(url)
fp= urllib.request.urlopen(response)
print('response is '+ response)
self.lineEditSiteZ.setText(fp)
according to Open Elevation guide, it says that you have to make requests in the form of:
curl https://api.open-elevation.com/api/v1/lookup\?locations\=50.3354,10.4567
in order to get elevation data as a JSON object. But in my case it returns an error saying:
raise RemoteDisconnected("Remote end closed connection without"
RemoteDisconnected: Remote end closed connection without response
and nothing happens. How can I fix this?
There is no other way than to create a loop (try until the response is ok). Because the Open Elevation API's handling of so many responses is still problematic. But the following piece of code works after a possibly long delay:
def getz(self, i):
import json
import requests
url = "https://api.open-elevation.com/api/v1/lookup"
"""calculates the elevation"""
if i is 'pushButtonSiteZ':
x = self.lineEditSiteX.text()
y = self.lineEditSiteY.text()
param = url + '?locations={},{}'.format(x,y)
print(param)
while True:
try:
response = requests.get(param)
print(response.status_code)
if str(response.status_code) == '200':
r = response.text
r = json.loads(r)
out = r['results'][0]['elevation']
print(out)
self.lineEditSiteZ.setText(str(out))
cal_rng(self)
break
except ConnectionError:
continue
except json.decoder.JSONDecodeError:
continue
except KeyboardInterrupt:
continue
except requests.exceptions.SSLError:
continue
except requests.exceptions.ConnectionError:
continue
Related
I was assisted with below code for a webscraper by one of the very helpful chaps on here however - it has all of a sudden stopped returning results. It either returns blank set() or nothing at all.
Does the below work for you ? Need to know if it's an issue with my IDE as it doesn't make any sense for it to be working one minute then giving random results the next when no amends was made to the code.
from requests_html import HTMLSession
import requests
def get_source(url):
try:
session = HTMLSession()
response = session.get(url)
return response
except requests.exceptions.RequestException as e:
print(e)
def scrape_google(query, start):
response = get_source(f"https://www.google.co.uk/search?q={query}&start={start}")
links = list(response.html.absolute_links)
google_domains = ('https://www.google.',
'https://google.',
'https://webcache.googleusercontent.',
'http://webcache.googleusercontent.',
'https://policies.google.',
'https://support.google.',
'https://maps.google.')
for url in links[:]:
if url.startswith(google_domains):
links.remove(url)
return links
data = []
for i in range(3):
data.extend(scrape_google('best place', i * 10))
print(set(data))
I have a list of IDs that I am passing into a URL within a for loop:
L = [1,2,3]
lst=[]
for i in L:
url = 'URL.Id={}'.format(i)
xml_data1 = requests.get(url).text
lst.append(xml_data1)
time.sleep(1)
print(xml_data1)
I am trying to create a try/catch where regardless of the error, the request library keeps trying to connect to the URL from the ID it left off on from the list (L), how would I do this?
I setup this try/catch from this answer (Correct way to try/except using Python requests module?)
However this forces the system to exit.
try:
for i in L:
url = 'URL.Id={}'.format(i)
xml_data1 = requests.get(url).text
lst.append(xml_data1)
time.sleep(1)
print(xml_data1)
except requests.exceptions.RequestException as e:
print (e)
sys.exit(1)
You can put the try-except block in a loop and only break the loop when the request does not raise an exception:
L = [1,2,3]
lst=[]
for i in L:
url = 'URL.Id={}'.format(i)
while True:
try:
xml_data1 = requests.get(url).text
break
except requests.exceptions.RequestException as e:
print(e)
lst.append(xml_data1)
time.sleep(1)
print(xml_data1)
I want to check a URL for the existence of robots.txt file. I found out about urllib.robotparser in python 3 and tried getting the response. But I can't find a way to return the status code (or just true/false existance) of robotss.txt
from urllib import parse
from urllib import robotparser
def get_url_status_code():
URL_BASE = 'https://google.com/'
parser = robotparser.RobotFileParser()
parser.set_url(parse.urljoin(URL_BASE, 'robots.txt'))
parser.read()
# I want to return the status code
print(get_url_status_code())
This isn't too hard to do if you're okay using the requests module which is highly recommended
import requests
def status_code(url):
r = requests.get(url)
return r.status_code
print(status_code('https://github.com/robots.txt'))
print(status_code('https://doesnotexist.com/robots.txt'))
Otherwise, if you want to avoid using a GET request, you could use a HEAD.
def does_url_exist(url):
return requests.head(url).status_code < 400
Better yet,
def does_url_exist(url):
try:
r = requests.head(url)
if r.status_code < 400:
return True
else:
return False
except requests.exceptions.RequestException as e:
print(e)
# handle your exception
Ok so I have two questions about the same code:
Question 1)
Say that I make a request. When that request goes past .5 second, I want it to end the current iteration and retry the connection. Basically, if it fails to get a request, I want it to either try again or exit the iteration so that it can try again. How do I do this? So with the code below, after a while, it gives me a "read time out" error. How do I fix this?
import requests
import json
import time
import sys
def calcProfits():
url = 'https://api.gdax.com/products/BTC-USD/trades'
t0 = time.time() #timer i made to see if the "timeout" works
res = requests.get(url, timeout = .5) #i want it to retry after 1 second of failing to get a response, is this correct?
t1 = time.time()
total_time = round((t1-t0),3)
json_res = json.loads(res.text)
currDollarBit = float(json_res[0]['price'])
sys.stdout.write(str(currDollarBit) + " ")
sys.stdout.write(str(total_time) + "\n")
time.sleep(1)
while True:
calcProfits()
Question 2)
So I came across this question and tried to use except. but I still see that the request is passing the 0.5 seconds i set up in the timeout and nothing is happening. eventually, it does a timeout and it says "UnboundLocalError: local variable 'res' referenced before assignment". i guessing that if it timeout, the res variable is completely ignored? Heres the code:
import requests
import json
import time
import sys
def calcProfits():
url = 'https://api.gdax.com/products/BTC-USD/trades'
try:
t0 = time.time() #timer i made to see if the "timeout" works
res = requests.get(url, timeout = .5)
t1 = time.time()
total_time = round((t1-t0),3)
except Exception as e:
print("\nTIMEOUT\n")
json_res = json.loads(res.text)
currDollarBit = float(json_res[0]['price'])
sys.stdout.write(str(currDollarBit) + " ")
sys.stdout.write(str(total_time) + "\n")
time.sleep(1)
while True:
calcProfits()
Anyways how do i fix it so that it just retires after that 0.5 seconds?
I'm writing an script that reads in the differences between two CSV files. Once it is read out I am supposed to use a WebHook to contact a slack page with the results of the comparison. I am having difficulty in sending the Post Method.
The link supplied by slack generates a response of 400
with either /post or :8080 at the end you get a 200, but nothing pops up in the slack page.
Any thoughts or suggestions?
def main():
csvDiff()
#print(l)
post()
def csvDiff():
f = open("new.csv")
csv_f = csv.reader(f)
old=set(pd.read_csv("old.csv", index_col=False, header=None)[0]) #reads the csv, takes only the first column and creates a set out of it.
new=set(pd.read_csv("new.csv", index_col=False, header=None)[0]) #same here
diff = new - old
#Convert the diff set into a list
diff=list(diff)
#print(diff)
#print(newConnections)
for row in csv_f:
if row[0] in diff:
l.append(row)
def makeCsv():
l = pd.to_csv
def post():
url = 'whatever'
payload={"text": "A very important thing has occurred! <https://alert-system.com/alerts/1234|Click here> for details!"}
r = requests.post(url, data=json.dumps(l).encode('utf8'))
print(r)
if __name__ == "__main__":
main()
Try this line instead:
r = requests.post(url, json=payload)