Reading a "special" URL - python-3.x

The task is simple- I want to tranfer a html file from an URL to a variable and read the feed below:
How can I read the contents of an URL with Python?
All that works well except with the url = "https://www.goyax.de/"
with
import urllib
#fp = urllib.request.urlopen("https://www.spiegel.de/")
fp = urllib.request.urlopen("https://www.goyax.de/")
print("Result code: " + str(fp.getcode()))
print("Returned data: -----------------")
data = fp.read().decode("utf-8")
print(data)
I get only "403" and "Forbidden". Also with
import requests
url = 'https://www.goyax.de/'
#url = 'https://www.spiegel.de'
r = requests.get(url)
tt = r.text
print(tt)
I don't get an improvement. With other URLs both solutions work well so far.
Until now I was using an Autohotkey script (UrlDownloadToFile) (Windows only) and tried it also with Octave (s = urlread("https://www.goyax.de/")) where I get the right result and no error message. the scripts ae running sicne years on a PC but I want to move this task to a Raspberry Pi. Because of that I was learning Python
The output / error messages:
fp = urllib.request.urlopen("http://www.goyax.de/")
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 531, in open
response = meth(req, response)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 640, in http_response
response = self.parent.error(
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 563, in error
result = self._call_chain(*args)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 502, in _call_chain
result = func(*args)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 755, in http_error_302
return self.parent.open(new, timeout=req.timeout)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 531, in open
response = meth(req, response)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 640, in http_response
response = self.parent.error(
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 569, in error
return self._call_chain(*args)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 502, in _call_chain
result = func(*args)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 649, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
HTTPError: Forbidden

Well, I found the answer myself or with some help of a friend: The key is to the soluition is to set the user agent.
Solution 1) (with "Requests")
import requests
r = requests.get(url2, headers={"User-Agent":'Mozilla/5.0'}, timeout=25)
mystr=r.text # mybytes=r.content
print(mystr)
Solution 2) (with "urllib.request" + "CookieJar")
import urllib.request
from http.cookiejar import CookieJar
req = urllib.request.Request(url2, None, {"User-Agent":'Mozilla/5.0'})
# instead of #req = urllib.request.Request(url2)
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(CookieJar()))
response = opener.open(req)
content = response.read()
mystr2 = content.decode("utf8")
print(mystr2)
Usually the user agent 'Mozilla/5.0' is sufficient. Or check
https://manytools.org/http-html-text/user-agent-string/
and
https://www.scrapehero.com/how-to-fake-and-rotate-user-agents-using-python-3/
for a real user agent string.

Related

Can't download video using pytube

I am trying to make a python program which downloads youtube video when link given using Pytube module but when i try to run it, its giving a huge error which is as follows-
Traceback (most recent call last):
File "c:\Users\Sumit\vs code python\projects\proj's\yt.py", line 9, in <module>
yt = YouTube(link)
File "C:\Python39\lib\site-packages\pytube\__main__.py", line 91, in __init__
self.prefetch()
File "C:\Python39\lib\site-packages\pytube\__main__.py", line 181, in prefetch
self.vid_info_raw = request.get(self.vid_info_url)
File "C:\Python39\lib\site-packages\pytube\request.py", line 36, in get
return _execute_request(url).read().decode("utf-8")
File "C:\Python39\lib\site-packages\pytube\request.py", line 24, in _execute_request
return urlopen(request) # nosec
File "C:\Python39\lib\urllib\request.py", line 214, in urlopen
return opener.open(url, data, timeout)
File "C:\Python39\lib\urllib\request.py", line 523, in open
response = meth(req, response)
File "C:\Python39\lib\urllib\request.py", line 632, in http_response
response = self.parent.error(
File "C:\Python39\lib\urllib\request.py", line 555, in error
result = self._call_chain(*args)
File "C:\Python39\lib\urllib\request.py", line 494, in _call_chain
result = func(*args)
File "C:\Python39\lib\urllib\request.py", line 747, in http_error_302
return self.parent.open(new, timeout=req.timeout)
File "C:\Python39\lib\urllib\request.py", line 523, in open
response = meth(req, response)
File "C:\Python39\lib\urllib\request.py", line 632, in http_response
response = self.parent.error(
File "C:\Python39\lib\urllib\request.py", line 561, in error
return self._call_chain(*args)
File "C:\Python39\lib\urllib\request.py", line 494, in _call_chain
result = func(*args)
File "C:\Python39\lib\urllib\request.py", line 641, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 410: Gone
I dont know whats the problem and I am not understanding what should I do, Please help.
Also, I am using python 3.9,the code is given below-
from pytube import YouTube
SAVE_PATH = r"C:\Users\Sumit\vs code python\projects"
link="https://www.youtube.com/watch?v=JfVOs4VSpmA&t=3s"
yt = YouTube(link)
mp4files = yt.filter('mp4')
yt.set_filename('Video')
Avideo = yt.get(mp4files[-1].extension,mp4files[-1].resolution)
try:
# downloading the video
Avideo.download(SAVE_PATH)
except:
print("Some Error!")
print('Task Completed!')
https://pytube.io/en/latest/api.html#pytube.Stream.download
from pytube import YouTube
SAVE_PATH = r"C:\Users\Sumit\vs code python\projects"
link="https://www.youtube.com/watch?v=JfVOs4VSpmA&t=3s"
yt = YouTube('http://youtube.com/watch?v=9bZkp7q19f0')
yt.streams
.filter(progressive=True, file_extension='mp4')
.order_by('resolution')
.desc()
.first()
.download(SAVE_PATH, 'videoFilename', 'mp4')

HTTP 403 error when trying to download a file from URL using Python

I'm trying to download a file from the URL -> https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519
I can manually download the file by accessing the URL via a browser and the file gets automatically saved onto the local machine in the Downloads folder. (The file is in JSON format)
However, I need to achieve this using a Python script. I tried using urllib.request & wget, but in both cases I keep getting the error -
urllib.request.urlretrieve(url, path)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py", line 247, in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py", line 531, in open
response = meth(req, response)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py", line 641, in http_response
'http', request, response, code, msg, hdrs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py", line 569, in error
return self._call_chain(*args)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py", line 503, in _call_chain
result = func(*args)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/urllib/request.py", line 649, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 403: Forbidden
Is there a workaround to this? Dealing with dynamic changes ?
You could try the following script to get the download url and download the json file:
import requests
import re
import urllib.request
rq= requests.get("https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519")
t = re.search("https://download.microsoft.com/download/.*?\.json", rq.text )
a= t.group()
print(a)
path = r"$(Build.sourcesdirectory)\agent.json"
urllib.request.urlretrieve(a, path)
Result:
Python 3
import urllib.request, json
with urllib.request.urlopen("https://download.microsoft.com/download/7/1/D/71D86715-5596-4529-9B13-DA13A5DE5B63/ServiceTags_Public_20210329.json") as url:
data = json.loads(url.read().decode())
print(data)

urllib.error.HTTPError: HTTP Error 404: Not Found when using request.urlopen()

I was following a tutorial and when using request.urlopen(url) I get an error, I have tried checking the URL
(https://www.wsj.com/market-data/quotes/PH/XPHS/JFC/historical-prices/download?MOD_VIEW=page&num_rows=150&range_days=150&startDate=06/01/2020&endDate=07/05/2020)
and it's fine.
Here is my code:
from urllib import request
import datetime
def download_stock_from_day_until_today(stock_code, start_date):
current_day = datetime.date.today()
formatted_current_day = datetime.date.strftime(current_day, "%m/%d/%Y") #formats today's date for links
#formatted url
url = "https://www.wsj.com/market-data/quotes/PH/XPHS/"+ stock_code +"/historical-prices/download?MOD_VIEW=page&num_rows=150&range_days=150&startDate="+ start_date +"&endDate=" + formatted_current_day
print(url)
response = request.urlopen(url) #requests the csv file
csv = response.read() #reads the csv file
csv_str = str(csv)
lines = csv_str.split("\\n")
dest_url = r'asd.csv'
fx = open(dest_url, "w")
for line in lines:
fx.write(line + "\n")
fx.close()
download_stock_from_day_until_today("JFC", "06/01/2020")
and the error I get in the console is:
Traceback (most recent call last):
File "C:/Users/Lathrix/PycharmProject/StockExcelDownloader/main.py", line 23, in <module>
download_stock_from_day_until_today("JFC", "06/01/2020")
File "C:/Users/Lathrix/PycharmProject/StockExcelDownloader/main.py", line 12, in download_stock_from_day_until_today
response = request.urlopen(url) #requests the csv file
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 531, in open
response = meth(req, response)
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 640, in http_response
response = self.parent.error(
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 569, in error
return self._call_chain(*args)
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 502, in _call_chain
result = func(*args)
File "C:\Users\Lathrix\AppData\Local\Programs\Python\Python38-32\lib\urllib\request.py", line 649, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found
Looks like wsj.com does not like urllib's User-Agent.
With the line   
response = request.urlopen(request.Request(url,headers={'User-Agent': 'Mozilla/5.0'}))
your code works correctly

Downloading a csv file with python

I'm trying to download historical stock prices from Yahoo Finance using Python using the following code:
import urllib.request
import ssl
import os
url = 'https://query1.finance.yahoo.com/v7/finance/download/%5ENSEI?period1=1537097203&period2=1568633203&interval=1d&events=history&crumb=0PVssBOEZBk'
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
connection = urllib.request.urlopen(url,context = ctx)
data = connection.read()
with urllib.request.urlopen(url) as testfile, open('data.csv', 'w') as f:
f.write(testfile.read().decode())
however, I'm getting a traceback as mentioned below:
Traceback (most recent call last):
File "C:/Users/jmirand/Desktop/test.py", line 11, in <module>
connection = urllib.request.urlopen(url,context = ctx)
File "C:\Users\jmirand\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\jmirand\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 531, in open
response = meth(req, response)
File "C:\Users\jmirand\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 641, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Users\jmirand\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 569, in error
return self._call_chain(*args)
File "C:\Users\jmirand\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 503, in _call_chain
result = func(*args)
File "C:\Users\jmirand\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 649, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 401: Unauthorized
I assume this has to do with the fact that it's because of the HTTPS and Python doesn't have enough certificates to put into it by default.
The webpage im on is here Yahoo Finance NSEI historical prices and its on the 'Download Data' tab that you click on where the data automatically gets downloaded through a csv file.
Can you please help in rectifying the code?
The yahoo api expects cookies from your browser to authenticate. I have copied the cookies from my browser and passed them through python requests
import requests
import csv
url = "https://query1.finance.yahoo.com/v7/finance/download/%5ENSEI?period1=1537099135&period2=1568635135&interval=1d&events=history&crumb=MMDwV5mvf2J"
cookies = {
"APID":"UP26c2bef4-bc0b-11e9-936a-066776ea83e8",
"APIDTS":"1568635136",
"B":"d10v5dhekvhhg&b=3&s=m6",
"GUC":"AQEBAQFda2VeMUIeqgS6&s=AQAAAICdZvpJ&g=XWoXdg",
"PRF":"t%3D%255ENSEI",
"cmp":"t=1568635133&j=0",
"thamba":"2"
}
with requests.Session() as s:
download = s.get(url,cookies=cookies)
decoded_content = download.content.decode('utf-8')
cr = csv.reader(decoded_content.splitlines(), delimiter=',')
my_list = list(cr)
for row in my_list:
print(row)

WinError 10060 when trying to save scraped data to csv file

I am trying to save a buch of scraped data to a csv file with python, but I am keep getting time out errors, don't really know how to go about it, what Should I do?
what I have tried
import pandas as pd
import urllib.request
from bs4 import BeautifulSoup as b
npo_codici ={}
codici_no = 0
df = pd.read_excel("led-italia_lampadine.xlsx","foglio1")
sku = df["codice SKU"].tolist()
rounded_sku = [round(x) for x in sku]
request_length = len(rounded_sku)
for e in range(request_length):
request = str(rounded_sku[e])
e+=1
base_url = "https://v-tac.it/led-products-results-page/?q="
url = base_url + request
html = urllib.request.urlopen(url).read()
soup = b(html, "html.parser")
these are the errors that I am getting
File "C:\Users\antonella\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\antonella\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 525, in open
response = self._open(req, data)
File "C:\Users\antonella\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 543, in _open
'_open', req)
File "C:\Users\antonella\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 503, in _call_chain
result = func(*args)
File "C:\Users\antonella\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 1360, in https_open
context=self._context, check_hostname=self._check_hostname)
File "C:\Users\antonella\AppData\Local\Programs\Python\Python37-32\lib\urllib\request.py", line 1319, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [WinError 10060]
because of network problem , maybe try with headers or check out this Why can't I get Python's urlopen() method to work on Windows?

Resources