Better Web Scraping Methodology? - python-3.x

Question

I was just curious if someone could help me out with a bit of webscraping. I am currently starting my scrape at this link -- https://www.basketball-reference.com/leagues/NBA_1981_games-october.html. I am scraping all the table "schedule" for each month and then moving onto the next year. I am able to successfully scrape from 1989 to 2001 (every month) and put into the format I desire. But my code is so fragile.... I was curious if there is a better methodology that can be explained to me rather than just pulling in the schedule table as one massive piece of text and then splicing it to fit my needs. For example, here is my code:
from selenium import webdriver as wd
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
import pandas as pd
import os
chromeDriverPath = r'path of chromeDriver used by Selenium'
browser = wd.Chrome(executable_path= chromeDriverPath)
#Create the links needed
link ="https://www.basketball-reference.com/leagues/NBA_"
years = range(1989,2018,1)
months = ['october', 'november', 'december', 'january', 'february', 'march',
'april', 'may', 'june', 'july', 'august', 'september']
hour_list = ['1:00','1:30', '1:40','2:00','2:30','3:00','3:30','4:00','4:30','5:00',
'5:30','6:00','6:30','7:00','7:30','8:00','8:30','9:00','9:30',
'10:00','10:30','11:00','11:30','12:00', '12:30','12:40']
ampm = ['pm', 'am']
def scrape(url):
try:
browser.get(url)
schedule = WebDriverWait(browser,5).until(EC.presence_of_all_elements_located((By.ID, "schedule")))
except TimeoutException:
print(str(url) + ' does not exist!')
return
o_players = [schedule[i].text for i in range(0, len(schedule))]
o_players = ''.join(o_players)
o_players = o_players.splitlines()
o_players = o_players[1:]
o_players = [x.replace(',','') for x in o_players]
o_players = [x.split(' ') for x in o_players]
l0 = []
l1 = []
l2 = []
for x in o_players:
if "at" in x:
l1.append(x[:x.index("at")])
elif 'Game' in x:
l0.append(x[:x.index("Game")])
else:
l2.append(x)
l3 = l1 + l2 + l0
for x in l3:
for y in x:
if y in hour_list:
x.remove(y)
for t in x:
if t in ampm:
x.remove(t)
ot = ['OT','2OT', '3OT', '4OT','5OT']
for x in l3:
x.insert(0,'N/A')
if x[-1] != 'Score' and x[-1] not in ot:
x.insert(1,x[-1])
else:
x.insert(1,'N/A')
for y in ot:
if y in x:
x.remove('N/A')
x.remove(y)
x.insert(0,y)
l3 = [t for t in l3 if 'Playoffs' not in t]
for x in l3:
if len(x) == 17:
x.insert(0,' '.join(x[6:9]))
x.insert(1,' '.join(x[11:14]))
x.insert(1, x[11])
x.insert(3, x[16])
if len(x) == 16 and x[-1] != 'Score':
if x[8].isdigit():
x.insert(0,' '.join(x[6:8]))
x.insert(1,' '.join(x[10:13]))
x.insert(1, x[10])
x.insert(3, x[15])
else:
x.insert(0,' '.join(x[6:9]))
x.insert(1,' '.join(x[11:13]))
x.insert(1, x[11])
x.insert(3, x[15])
if len(x) == 16 and x[-1] == 'Score':
x.insert(0,' '.join(x[6:9]))
x.insert(1, ' '.join(x[11:14]))
x.insert(1, x[11])
x.insert(3, x[16])
if len(x) == 15 and x[-1] != 'Score':
x.insert(0,' '.join(x[6:8]))
x.insert(1,' '.join(x[10:12]))
x.insert(1, x[10])
x.insert(3, x[14])
if len(x) == 15 and x[-1] == 'Score':
if x[8].isdigit():
x.insert(0,' '.join(x[6:8]))
x.insert(1,' '.join(x[10:13]))
x.insert(1, x[10])
x.insert(3, x[15])
else:
x.insert(0,' '.join(x[6:9]))
x.insert(1,' '.join(x[11:13]))
x.insert(1, x[11])
x.insert(3, x[15])
if len(x) == 14:
x.insert(0,' '.join(x[6:8]))
x.insert(1,' '.join(x[10:12]))
x.insert(1, x[10])
x.insert(3, x[14])
l4 = []
for x in l3:
x = x[:10]
l4.append(x)
#Working With Pandas to Standardize Data
df = pd.DataFrame(l4)
df['Date'] = df[7] + ' '+ df[8] + ', ' + df[9]
df['Date'] = pd.to_datetime(df['Date'])
df = df.sort_values(by=['Date'])
headers = ['Visitor', 'Visitor Points', 'Home', 'Home Points', 'OT',
'Attendance','Weekday', 'Month', 'Day', 'Year', 'Date' ]
headers_order = ['Date', 'Weekday', 'Day', 'Month', 'Year', 'Visitor', 'Visitor Points',
'Home', 'Home Points', 'OT', 'Attendance']
df.columns = headers
df = df[headers_order]
file_exists = os.path.isfile("NBA_Scrape.csv")
if not file_exists:
df.to_csv('NBA_Scrape.csv', mode='a', header=True, index=False)
else:
df.to_csv('NBA_Scrape.csv', mode='a', header=False, index=False)
for x in years:
link0 = link+str(x)+'_games-'
for y in months:
final_links = link0+str(y)+'.html'
scrape(final_links)
My code starts to return errors at year 2001 I believe. I would like to scrape through the present. Please help me scrape better. I imagine there is a much more proficient way, like looping through each element in the table "schedule" and appending each one to a different list or different column in pandas? Please lend me a hand.
Thanks,
Joe

Answer 1

Your target is perfectly static, so no necessary to run selenium. I would suggest using Scrapy python library. It was designed to fit all web scraping needs. It incredibly fast and flexible tool. You can use xpath to pull all elements from page separately, instead of considering it as a huge piece of text.