Multithreading optimization in python and selenium - multithreading

Question

I've written a script for automating the process of logging in to a website and taking screenshots of a particular page using Selenium and the undetected_chromedriver library. The script reads a list of usernames and passwords from an Excel file (users.xlsx), and then creates a queue of tasks for each user to log in and take a screenshot of a specific web page. The script uses multiple threads to handle the queue, with a maximum number of workers that can be set to the number of users. Once a task is completed, the driver is quit and the next task is started. The output of the script is a set of PDF files with the usernames as the filename.
Issue with the script is that it takes too long to (5 seconds) from finishing the task and closing the driver to start a new one.
I'm not sure the issue is with the run_threads function or the get_ss_hbd function.
I'm looking for ways to optimize this code and reduce runtime.
import undetected_chromedriver as uc
import threading
import concurrent.futures
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException, TimeoutException
import pandas as pd
import time
import base64
import os
import queue
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Safari/605.1.15'
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--headless')
chrome_options.add_argument('--start-maximized')
chrome_options.add_argument('--disable-infobars')
chrome_options.add_argument('--disable-extensions')
chrome_options.add_argument('--enable-javascript')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('User-Agent={0}'.format(user_agent))
chrome_options.add_argument('--window-size=1920,1080')
chrome_options.add_argument('--kiosk-printing')
chrome_options.add_argument("--window-size=2000x2000")
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', True)
logout = "***********"
# Test with example username and password lists
df = pd.read_excel("users.xlsx")
username_list = df['username'].tolist()
password_list = df['password'].tolist()
width = 500
height = 600
def login_hbd(driver, username, password):
driver.get("******")
WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.NAME, "username")))
username_field = driver.find_element(By.NAME, "username")
username_field.send_keys(username)
password_field = driver.find_element(By.NAME, "password")
password_field.send_keys(password)
submit_btn = WebDriverWait(driver, 2).until(EC.element_to_be_clickable((By.ID, "submitBtn")))
driver.execute_script("arguments[0].click();", submit_btn)
def get_ss_hbd(username, password, x, y, width, height, start_time):
with uc.Chrome() as driver:
driver.set_window_position(x, y)
driver.set_window_size(width, height)
while True:
try:
login_hbd(driver, username, password)
WebDriverWait(driver, 1).until(EC.presence_of_element_located((By.XPATH, "//a[text()='s']")))
driver.get("******")
time.sleep(1.5)
pdf = driver.execute_cdp_cmd(
"Page.printToPDF", {
"printBackground": True,
"landscape": False,
"displayHeaderFooter": False,
"scale": 1,
})
with open(f'{username}.pdf', "wb") as f:
f.write(base64.b64decode(pdf['data']))
# Return the path to the PDF file
end_time = time.monotonic()
elapsed_time = end_time - start_time
print(f"Thread for {username} finished in {elapsed_time:.2f} seconds.")
return driver
break
except (NoSuchElementException, ElementClickInterceptedException, TimeoutException):
continue
def run_threads(username_list, password_list, width, height):
num_users = len(username_list)
x_positions = range(0, width*num_users, width)
max_workers = 2 # Set max_workers to the number of users
task_queue = queue.Queue()
# Fill the task queue with the user credentials
for i in range(num_users):
task_queue.put((i, username_list[i], password_list[i], x_positions[i], 0, width, height))
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {}
completed = {}
# Submit the tasks to the thread pool executor
while not task_queue.empty() or len(futures) > 0:
# Check if there are any completed tasks
for task_id, future in futures.items():
if future.done():
try:
result = future.result()
result.quit() # Quit the driver when the task is done
except Exception as exc:
print(f"generated an exception: {exc}")
else:
print(f"Thread for {result} finished.")
completed[task_id] = future
# Remove the completed tasks from the futures dictionary
for task_id in completed:
del futures[task_id]
completed.clear()
# Submit new tasks if there are any available
while not task_queue.empty() and len(futures) < max_workers:
task_id, username, password, x, y, width, height = task_queue.get()
start_time = time.monotonic() # Start the timer
future = executor.submit(get_ss_hbd, username, password, x, y, width, height, start_time)
futures[task_id] = future
# Wait for a short while to avoid spinning the loop too fast
time.sleep(0.1)
run_threads(username_list, password_list, width, height)