How would I loop through each page and scrape the specific parameters? - python-3.x

Question

#Import Needed Libraries
import requests
from bs4 import BeautifulSoup
import pprint
res = requests.get('https://news.ycombinator.com/news')
soup = BeautifulSoup(res.text, 'html.parser')
links = soup.select('.titlelink')
subtext = soup.select('.subtext')
def sort_stories_by_votes(hnlist): #Sorting your create_custom_hn dict by votes(if)
return sorted(hnlist, key= lambda k:k['votes'], reverse=True)
def create_custom_hn(links, subtext): #Creates a list of links and subtext
hn = []
for idx, item in enumerate(links): #Need to use this because not every link has a lot of votes
title = links[idx].getText()
href = links[idx].get('href', None)
vote = subtext[idx].select('.score')
if len(vote):
points = int(vote[0].getText().replace(' points', ''))
if points > 99: #Only appends stories that are over 100 points
hn.append({'title': title, 'link': href, 'votes': points})
return sort_stories_by_votes(hn)
pprint.pprint(create_custom_hn(links, subtext))
My question is that this is only for the first page, which has only 30 stories.
How would I apply my web scraping method by going through each page.... let's say the next 10 pages and keeping the formatted code above?

Answer 1

The URL for each page is like this
https://news.ycombinator.com/news?p=<page_number>
Use a for-loop to scrape content from each page. See the code below.
Here is the code that prints the contents from the first two pages. You can change the page_no depending on your need.
import requests
from bs4 import BeautifulSoup
import pprint
def sort_stories_by_votes(hnlist): #Sorting your create_custom_hn dict by votes(if)
return sorted(hnlist, key= lambda k:k['votes'], reverse=True)
def create_custom_hn(links, subtext, page_no): #Creates a list of links and subtext
hn = []
for idx, item in enumerate(links): #Need to use this because not every link has a lot of votes
title = links[idx].getText()
href = links[idx].get('href', None)
vote = subtext[idx].select('.score')
if len(vote):
points = int(vote[0].getText().replace(' points', ''))
if points > 99: #Only appends stories that are over 100 points
hn.append({'title': title, 'link': href, 'votes': points})
return sort_stories_by_votes(hn)
for page_no in range(1,3):
print(f'Page: {page_no}')
url = f'https://news.ycombinator.com/news?p={page_no}'
res = requests.get(url)
soup = BeautifulSoup(res.text, 'html.parser')
links = soup.select('.titlelink')
subtext = soup.select('.subtext')
pprint.pprint(create_custom_hn(links, subtext, page_no))
Page: 1
[{'link': 'https://www.thisworddoesnotexist.com/',
'title': 'This word does not exist',
'votes': 904},
{'link': 'https://www.sparkfun.com/news/3970',
'title': 'A patent troll backs off',
'votes': 662},
.
.
Page: 2
[{'link': 'https://www.vice.com/en/article/m7vqkv/how-fbi-gets-phone-data-att-tmobile-verizon',
'title': "The FBI's internal guide for getting data from AT&T, T-Mobile, "
'Verizon',
'votes': 802},
{'link': 'https://www.dailymail.co.uk/news/article-10063665/Government-orders-Google-track-searching-certain-names-addresses-phone-numbers.html',
'title': 'Feds order Google to track people searching certain names or '
'details',
'votes': 733},
.
.