Best approach for webscraping two websites Python 3 - multithreading

Question

I want to get information from two websites and display it on 'real-time' in the console.
To get the information from the website I am using BeautifulSoup 4. I have read, that the bottleneck of scraping websites is the connection or the website itself. So I wanted to use multithreading, so the read can be done 'simultaneously'. Below is my code:
import urllib
from bs4 import BeautifulSoup
import threading
import time
link_website_one = 'http://www.website1.com'
link_website_one = 'http://www.website2.com'
def request_url(url):
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36'
header = { 'User-Agent' : user_agent }
req = urllib.request.Request(url, headers=header)
try:
response = urllib.request.urlopen(req)
return response
except urllib.error.HTTPError as e:
print('The server couldn\'t fulfill the request.')
print(e.code,': ', e.reason)
return None
except urllib.error.URLError as e:
print('We failed to reach a server.')
print(e.code,': ', e.reason)
return None
def new_entry(website, lis, arr, a_id):
if website == 0 :
info1 = arr.find()
info2 = arr.find('div', {'class' : '1'}).find('span', {'class' : '7'}).get_text().strip()
info3 = arr.find('span', {'class' : '2'}).get_text().strip()
info4 = arr.find('span', {'class' : '3'}).get_text().strip()
info5 = arr.find('span', {'class' : '4'}).get_text().strip()
info6 = arr.find('div', {'class' : '5'}).get_text().strip()
lis.append({'info1' : info1, 'info2' : info2, 'info3' : info3, 'info4' : info4, 'info5' : info5, 'info6' : info6})
elif website == 1 :
info1 = a_id
info2 = arr.find('span', {'class' : '8'}).get_text()
info3 = arr.find('div', {'class' : '9'}).get_text()
info4 = arr.find_all('div', {'class' : '10'})[0].get_text()
info5 = arr.find_all('div', {'class' : '10'})[1].get_text()
info6 = arr.a["href"]
lis.append({'info1' : info1, 'info2' : info2, 'info3' : info3, 'info4' : info4, 'info5' : info5, 'info6' : info6})
class AsyncSearch(threading.Thread):
def __init__(self, website, iter_rounds, delay_time):
threading.Thread.__init__(self)
self.website = website
self.iter_rounds = iter_rounds
self.delay_time = delay_time
def run(self):
if self.website == 0:
for z in range(self.iter_rounds):
req_1 = request_url(link_1)
content_1 =req_1.read()
soup_1 = BeautifulSoup(content_1, 'lxml')
arr_1 = soup_1.find_all('div', {'class' : 'special_class'})
for x in range(len(arr_1)):
id_as = int(arr_1[x].find('class123')['class2345'].split(',')[0].split('"')[3].strip())
if id_as not in found_ids_1:
found_ids_1.add(id_as)
new_entry(0, all_entries_1, arr_1[x], id_as)
#else:
# break
req_1.close()
time.sleep(self.delay_time)
elif self.website == 1:
for z in range(self.iter_rounds):
req_2 = request_url(link)
content_2 =req_2.read()
soup_2 = BeautifulSoup(content_2, 'lxml')
arr_2 = soup_2.find_all('div', {'class' : 'class445'})
for x in range(len(arr_2)):
if arr_2[x].a['oid'] not in found_ids_2:
#print('Mobile: ',test[x].a[Inserat_ID])
found_ids_2.add(arr_2[x].a['oid'])
new_entry(1, all_entries_1, arr_2[x], arr_2[x].a['oid'])
#else:
# break
req.close()
time.sleep(self.delay_time)
all_entries_1=[]
all_entries_2=[]
found_ids_1 = set()
found_ids_2 = set()
website1 = AsyncSearch(0,10,1)
website2 = AsyncSearch(1,10,1)
website1.start()
website2.start()
website1.join()
website2.join()
First of two lists (all_entries 1/2) and two sets (found_ids 1/2) are created.
The websites I am scraping are offering 20 ads per page with unique ids. With the new_entry method you can say which website you want, in which list it should append the new entry, which array contains the beautifulsoup and the id of the ad you want to append.
Now for the mutlithreading I created a class AsyncSearch, where you can choose the website, choose the number of iterations of requests and how long to wait for the next request.
The two sets found_ids 1/2 are there so you don't append an ad more than once in the all_entries list.
Now to the problem. This code works.
But: If delay_time = 1 and iterations = 10 it needs 20 seconds to finish. Is there a faster approach to solve this problem? the .read() duration is between 0.12 and 0.17 seconds for website 1 and between 0.03 and 0.10 seconds for website 2.
If I said anything not understandable please ask.