Python Seleniumdriver get NoneType - python-3.x

I would like to ask if anyone knows why they tell me that the driver is a NoneType, below is my code:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
def get_site(endpoint):
driver = webdriver.Firefox()
driver = driver.get(endpoint)
return driver
def get_logged(driver):
try:
inputUser = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "username")))
finally:
driver.quit()
if __name__ == "__main__":
url = "https://logon.fastweb.it/oam/server/obrareq.cgi?encquery%3D1%2FGW5uB%2Bh4bKAKj%2FPYv8lCcoHT7AvYWgpoiTqOAaltkzLYb0%2BCJWw7yAeIR4JSsd9lNz4QgZbaGY%2B0We0nqu0pzODI8foO7KuGl9XoXnEEPpiOt2pgZLIy%2FsuuZT8sG6mYgWEFJ3SjumdY0Vxy27q8cab8%2BgapLT85otLP3Zv5bKdIkJ6aRxZLMvg8JU5ZpXtPYn2icoJSpGBDDmUZaxewYunYeMJ0%2Bhgu9%2FdieDYP44N4QIV642dgpsZPs%2Bv3kkEYZDasbOJkQt0ZXQnQWqM%2BZnKMS7EMF2ddeYwBDqaKbg4qEqp5RIASYNNijg7D%2F8%20agentid%3DFront-End-OAM%20ver%3D1%20crmethod%3D2"
driver = get_site(url)
get_logged(driver)
and the error:
AttributeError: 'NoneType' object has no attribute 'find_element'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 23, in <module>
get_logged(driver)
File "main.py", line 16, in get_logged
driver.quit()
AttributeError: 'NoneType' object has no attribute 'quit'
Anyone have an idea?

This has to do with the way you're handling Selenium.
Selenium should not be passed around your functions like a typical HTTP request, it is a standalone driver that you are operating with code.
For example, driver = driver.get(url) does not return a driver, in fact it returns nothing, because all you're doing with driver.get() is telling your webdriver to navigate to a specific link. This is the root of your 'NoneType' object has no attribute 'find_element' because the new driver variable you've defined is the result of driver.get(), which is nothing.
I've gone ahead and refactored your code (untested) to a class approach, this should be a much cleaner way to tackle your problem.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
class DriverController():
def __init__(self):
self.driver = webdriver.Firefox()
def get_site(self, url):
self.driver.get(url)
def get_logged(self):
try:
inputUser = WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.ID, "username")))
finally:
self.driver.quit()
url = "https://logon.fastweb.it/oam/server/obrareq.cgi?encquery%3D1%2FGW5uB%2Bh4bKAKj%2FPYv8lCcoHT7AvYWgpoiTqOAaltkzLYb0%2BCJWw7yAeIR4JSsd9lNz4QgZbaGY%2B0We0nqu0pzODI8foO7KuGl9XoXnEEPpiOt2pgZLIy%2FsuuZT8sG6mYgWEFJ3SjumdY0Vxy27q8cab8%2BgapLT85otLP3Zv5bKdIkJ6aRxZLMvg8JU5ZpXtPYn2icoJSpGBDDmUZaxewYunYeMJ0%2Bhgu9%2FdieDYP44N4QIV642dgpsZPs%2Bv3kkEYZDasbOJkQt0ZXQnQWqM%2BZnKMS7EMF2ddeYwBDqaKbg4qEqp5RIASYNNijg7D%2F8%20agentid%3DFront-End-OAM%20ver%3D1%20crmethod%3D2"
dc = DriverController()
dc.get_site(url)
dc.get_logged()

Related

Python3 web automation error - ElementNotInteractableException: Message: element not interactable [duplicate]

I understand this question has been asked but I need some solution for this error:
Traceback (most recent call last):
File "goeventz_automation.py", line 405, in <module>
if login(driver) is not None:
File "goeventz_automation.py", line 149, in login
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/support/wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
This is the code where its getting error:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import TimeoutException
import urllib.request as request
import urllib.error as error
from PIL import Image
from selenium.webdriver.chrome.options import Options
import datetime as dt
import time
from common_file import *
from login_credentials import *
def login(driver):
global _email, _password
if waiter(driver, "//a[#track-element='header-login']") is not None:
#login = driver.find_element_by_xpath("//a[#track-element='header-login']")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
#login.click()
if waiter(driver,"//input[#id='user_email']") is not None:
email = driver.find_element_by_xpath("//input[#id='user_email']")
password = driver.find_element_by_xpath("//input[#id='password']")
email.send_keys(_email)
password.send_keys(_password)
driver.find_element_by_xpath("//button[#track-element='click-for-login']").click()
return driver
else:
print("There was an error in selecting the email input field. It may be the page has not loaded properly.")
return None
else:
print("There was an error in selecting the header-login attribute on the page.")
return None
if __name__ == '__main__':
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('/usr/bin/chromium/chromedriver',chrome_options=chrome_options)
#d.get('https://www.google.nl/')
#driver = webdriver.Chrome()
driver.maximize_window()
driver.get('https://www.goeventz.com/')
if login(driver) is not None:
print(create_event(driver))
I think there is some problem with Keys.ENTER, but I don't know how to solve this. I have tried every possible solution.............
This error message...
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
...implies that the desired element was not interactable when you tried to invoke click() on it.
A couple of facts:
When you initialize the Chrome browser always in maximized mode.
You can disable-extensions.
You need to disable-infobars as well.
I have used the same xpath which you have constructed and you can use the following solution:
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
options = webdriver.ChromeOptions()
options.add_argument("start-maximized");
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(chrome_options=options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get("https://www.goeventz.com/")
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
Browser Snapshot:
copy full xpath instead of copying only xpath. It will work
Instead of using login.send_keys(Keys.ENTER) you should use selenium click() method which would work fine for you.
You can check first if the element is clickable first and then you can click on it.
Like:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
Overview
It seems like you're having an XPATH problem finding the "Submit" button or your Submit button is not clickable, or your Submit button has some client side events attached to it (javascript/etc) that are required in order to effectively submit the page.
Calling the pw.submit() method in most cases should get rid of the need to wait for the submit button to become clickable and avoid any issues in locating the button in most cases. On many other websites, some of the necessary back-end processes are primed by client-side activities that are performed after the "submit" button is actually clicked (although on a side-note this is not considered best-practice because it makes the site less accessible, etc, I digress). Above all, it's important to watch your script execute and make sure that you're not getting any noticeable errors displayed on the webpage about the credentials that you're submitting.
Also, however, some websites require that you add a certain minimum amount of time between the entry of the username, password, and submitting the page in order for it to be considered a valid submitting process. I've even run in to websites that require you to use send_keys 1 at a time for usernames and passwords to avoid some anti-scraping technologies they employ. In these cases, I usually use the following between the calls:
from random import random, randint
def sleepyTime(first=5, second=10):
# returns the value of the time slept (as float)
# sleeps a random amount of time between the number variable in first
# and the number variable second (in seconds)
sleepy_time = round(random() * randint(first, second), 2)
sleepy_time = sleepy_time if sleepy_time > first else (first + random())
sleep(sleepy_time)
return sleepy_time
I don't see what use you have for making the _email and _password variables global, unless they are being changed somewhere in the login function and you want that change to be precipitated out to the other scopes.
How I would try to solve it
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException, TimeoutException
TIME_TIMEOUT = 20 # Twenty-second timeout default
def eprint(*args, **kwargs):
""" Prints an error message to the user in the console (prints to sys.stderr), passes
all provided args and kwargs along to the function as usual. Be aware that the 'file' argument
to print can be overridden if supplied again in kwargs.
"""
print(*args, file=sys.stderr, **kwargs)
def login(driver):
global _email, _password
try:
email = WebDriverWait(driver, TIME_TIMEOUT).until(EC.presence_of_element_located((By.XPATH, "//input[#id='user_email']")))
pw = WebDriverWait(driver, TIME_TIMEOUT).until(EC.presence_of_element_located((By.XPATH, "//input[#id='password']"))
pw.submit()
# if this doesn't work try the following:
# btn_submit = WebDriverWait(driver, TIME_TIMEOUT).until(EC.element_to_be_clickable((By.XPATH, "//button[#track-element='click-for-login']"))
# btn_submit.click()
# if that doesn't work, try to add some random wait times using the
# sleepyTime() example from above to add some artificial waiting to your email entry, your password entry, and the attempt to submit the form.
except NoSuchElementException as ex:
eprint(ex.msg())
except TimeoutException as toex:
eprint(toex.msg)
if __name__ == '__main__':
driver = webdriver.Chrome('/usr/bin/chromium/chromedriver',chrome_options=chrome_options)
#d.get('https://www.google.nl/')
#driver = webdriver.Chrome()
driver.maximize_window()
driver.get('https://www.goeventz.com/')
if login(driver) is not None:
print(create_event(driver))
For headless chrome browser you need to provide window size as well in chrome options.For headless browser selenium unable to know what your window size.Try that and let me know.
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('window-size=1920x1480')
I faced this error as well. Now check your browser if the element is inside the iframe. If so, use driver.find_element(By.CSS_SELECTOR, "#payment > div > div > iframe") and driver.switch_to.frame(iframe) Then you will be able to work out.

Selenium headless cannot interact with element but non-headless can [duplicate]

I understand this question has been asked but I need some solution for this error:
Traceback (most recent call last):
File "goeventz_automation.py", line 405, in <module>
if login(driver) is not None:
File "goeventz_automation.py", line 149, in login
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/support/wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
This is the code where its getting error:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import TimeoutException
import urllib.request as request
import urllib.error as error
from PIL import Image
from selenium.webdriver.chrome.options import Options
import datetime as dt
import time
from common_file import *
from login_credentials import *
def login(driver):
global _email, _password
if waiter(driver, "//a[#track-element='header-login']") is not None:
#login = driver.find_element_by_xpath("//a[#track-element='header-login']")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
#login.click()
if waiter(driver,"//input[#id='user_email']") is not None:
email = driver.find_element_by_xpath("//input[#id='user_email']")
password = driver.find_element_by_xpath("//input[#id='password']")
email.send_keys(_email)
password.send_keys(_password)
driver.find_element_by_xpath("//button[#track-element='click-for-login']").click()
return driver
else:
print("There was an error in selecting the email input field. It may be the page has not loaded properly.")
return None
else:
print("There was an error in selecting the header-login attribute on the page.")
return None
if __name__ == '__main__':
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('/usr/bin/chromium/chromedriver',chrome_options=chrome_options)
#d.get('https://www.google.nl/')
#driver = webdriver.Chrome()
driver.maximize_window()
driver.get('https://www.goeventz.com/')
if login(driver) is not None:
print(create_event(driver))
I think there is some problem with Keys.ENTER, but I don't know how to solve this. I have tried every possible solution.............
This error message...
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
...implies that the desired element was not interactable when you tried to invoke click() on it.
A couple of facts:
When you initialize the Chrome browser always in maximized mode.
You can disable-extensions.
You need to disable-infobars as well.
I have used the same xpath which you have constructed and you can use the following solution:
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
options = webdriver.ChromeOptions()
options.add_argument("start-maximized");
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(chrome_options=options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get("https://www.goeventz.com/")
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
Browser Snapshot:
copy full xpath instead of copying only xpath. It will work
Instead of using login.send_keys(Keys.ENTER) you should use selenium click() method which would work fine for you.
You can check first if the element is clickable first and then you can click on it.
Like:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
Overview
It seems like you're having an XPATH problem finding the "Submit" button or your Submit button is not clickable, or your Submit button has some client side events attached to it (javascript/etc) that are required in order to effectively submit the page.
Calling the pw.submit() method in most cases should get rid of the need to wait for the submit button to become clickable and avoid any issues in locating the button in most cases. On many other websites, some of the necessary back-end processes are primed by client-side activities that are performed after the "submit" button is actually clicked (although on a side-note this is not considered best-practice because it makes the site less accessible, etc, I digress). Above all, it's important to watch your script execute and make sure that you're not getting any noticeable errors displayed on the webpage about the credentials that you're submitting.
Also, however, some websites require that you add a certain minimum amount of time between the entry of the username, password, and submitting the page in order for it to be considered a valid submitting process. I've even run in to websites that require you to use send_keys 1 at a time for usernames and passwords to avoid some anti-scraping technologies they employ. In these cases, I usually use the following between the calls:
from random import random, randint
def sleepyTime(first=5, second=10):
# returns the value of the time slept (as float)
# sleeps a random amount of time between the number variable in first
# and the number variable second (in seconds)
sleepy_time = round(random() * randint(first, second), 2)
sleepy_time = sleepy_time if sleepy_time > first else (first + random())
sleep(sleepy_time)
return sleepy_time
I don't see what use you have for making the _email and _password variables global, unless they are being changed somewhere in the login function and you want that change to be precipitated out to the other scopes.
How I would try to solve it
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException, TimeoutException
TIME_TIMEOUT = 20 # Twenty-second timeout default
def eprint(*args, **kwargs):
""" Prints an error message to the user in the console (prints to sys.stderr), passes
all provided args and kwargs along to the function as usual. Be aware that the 'file' argument
to print can be overridden if supplied again in kwargs.
"""
print(*args, file=sys.stderr, **kwargs)
def login(driver):
global _email, _password
try:
email = WebDriverWait(driver, TIME_TIMEOUT).until(EC.presence_of_element_located((By.XPATH, "//input[#id='user_email']")))
pw = WebDriverWait(driver, TIME_TIMEOUT).until(EC.presence_of_element_located((By.XPATH, "//input[#id='password']"))
pw.submit()
# if this doesn't work try the following:
# btn_submit = WebDriverWait(driver, TIME_TIMEOUT).until(EC.element_to_be_clickable((By.XPATH, "//button[#track-element='click-for-login']"))
# btn_submit.click()
# if that doesn't work, try to add some random wait times using the
# sleepyTime() example from above to add some artificial waiting to your email entry, your password entry, and the attempt to submit the form.
except NoSuchElementException as ex:
eprint(ex.msg())
except TimeoutException as toex:
eprint(toex.msg)
if __name__ == '__main__':
driver = webdriver.Chrome('/usr/bin/chromium/chromedriver',chrome_options=chrome_options)
#d.get('https://www.google.nl/')
#driver = webdriver.Chrome()
driver.maximize_window()
driver.get('https://www.goeventz.com/')
if login(driver) is not None:
print(create_event(driver))
For headless chrome browser you need to provide window size as well in chrome options.For headless browser selenium unable to know what your window size.Try that and let me know.
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('window-size=1920x1480')
I faced this error as well. Now check your browser if the element is inside the iframe. If so, use driver.find_element(By.CSS_SELECTOR, "#payment > div > div > iframe") and driver.switch_to.frame(iframe) Then you will be able to work out.

Running this code gives me a timeout exception error, why?

I'm trying to build a script to automate filling in a textbox with selenium, but I cant seem to get it to work.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
import time
browser =
webdriver.Chrome('C:/Users/xiang/PycharmProjects/testo/chromedriver.exe')
browser.get('https://zbib.org/')
wait = WebDriverWait(browser, 10)
name = "form-control form-control form-control-lg id-input"
try:
input = wait.until(ec.presence_of_element_located((By.CLASS_NAME,
name)))
finally:
browser.quit()
I expected there to be no error and the browser/driver doesn't quit, but I get this error in the terminal and the browser/driver quits:
Traceback (most recent call last):
File "C:/Users/xiang/PycharmProjects/testo/bib.py", line 14, in
<module>
input = wait.until(ec.presence_of_element_located((By.CLASS_NAME,
name)))
File "C:\Users\xiang\PycharmProjects\testo\venv\lib\site-
packages\selenium\webdriver\support\wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
Please help, thanks!
Your code can definitely throw an Exception.
WebDriverWait will throw a TimeoutException if it does not meet the request after the timeout set time.
You can ignore the exception by adding this:
from selenium.common.exceptions import TimeoutException
try:
input = wait.until(ec.presence_of_element_located((By.CLASS_NAME,
name)))
except TimeoutException:
pass
finally:
browser.quit()
Since your className has spaces you should be using css selector. Your name variable would be:
name = ".form-control.form-control.form-control-lg.id-input"
Your code would then be:
from selenium.common.exceptions import TimeoutException
try:
input = wait.until(ec.presence_of_element_located((By.CSS_SELECTOR,
name)))
except TimeoutException:
pass
finally:
browser.quit()
To ensure your browser will be closed you can also use the context manager like this:
name = ".form-control.form-control.form-control-lg.id-input"
chromedriver = 'C:/Users/xiang/PycharmProjects/testo/chromedriver.exe'
with webdriver.Chrome(chromedriver) as browser:
browser.get('https://zbib.org/')
wait = WebDriverWait(browser, 10)
try:
input = wait.until(ec.presence_of_element_located((By.CSS_SELECTOR, name)))
except TimeoutException:
pass

selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable when clicking on an element using Selenium Python

I understand this question has been asked but I need some solution for this error:
Traceback (most recent call last):
File "goeventz_automation.py", line 405, in <module>
if login(driver) is not None:
File "goeventz_automation.py", line 149, in login
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/support/wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
This is the code where its getting error:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import TimeoutException
import urllib.request as request
import urllib.error as error
from PIL import Image
from selenium.webdriver.chrome.options import Options
import datetime as dt
import time
from common_file import *
from login_credentials import *
def login(driver):
global _email, _password
if waiter(driver, "//a[#track-element='header-login']") is not None:
#login = driver.find_element_by_xpath("//a[#track-element='header-login']")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
#login.click()
if waiter(driver,"//input[#id='user_email']") is not None:
email = driver.find_element_by_xpath("//input[#id='user_email']")
password = driver.find_element_by_xpath("//input[#id='password']")
email.send_keys(_email)
password.send_keys(_password)
driver.find_element_by_xpath("//button[#track-element='click-for-login']").click()
return driver
else:
print("There was an error in selecting the email input field. It may be the page has not loaded properly.")
return None
else:
print("There was an error in selecting the header-login attribute on the page.")
return None
if __name__ == '__main__':
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('/usr/bin/chromium/chromedriver',chrome_options=chrome_options)
#d.get('https://www.google.nl/')
#driver = webdriver.Chrome()
driver.maximize_window()
driver.get('https://www.goeventz.com/')
if login(driver) is not None:
print(create_event(driver))
I think there is some problem with Keys.ENTER, but I don't know how to solve this. I have tried every possible solution.............
This error message...
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
...implies that the desired element was not interactable when you tried to invoke click() on it.
A couple of facts:
When you initialize the Chrome browser always in maximized mode.
You can disable-extensions.
You need to disable-infobars as well.
I have used the same xpath which you have constructed and you can use the following solution:
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
options = webdriver.ChromeOptions()
options.add_argument("start-maximized");
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(chrome_options=options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get("https://www.goeventz.com/")
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
Browser Snapshot:
copy full xpath instead of copying only xpath. It will work
Instead of using login.send_keys(Keys.ENTER) you should use selenium click() method which would work fine for you.
You can check first if the element is clickable first and then you can click on it.
Like:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
Overview
It seems like you're having an XPATH problem finding the "Submit" button or your Submit button is not clickable, or your Submit button has some client side events attached to it (javascript/etc) that are required in order to effectively submit the page.
Calling the pw.submit() method in most cases should get rid of the need to wait for the submit button to become clickable and avoid any issues in locating the button in most cases. On many other websites, some of the necessary back-end processes are primed by client-side activities that are performed after the "submit" button is actually clicked (although on a side-note this is not considered best-practice because it makes the site less accessible, etc, I digress). Above all, it's important to watch your script execute and make sure that you're not getting any noticeable errors displayed on the webpage about the credentials that you're submitting.
Also, however, some websites require that you add a certain minimum amount of time between the entry of the username, password, and submitting the page in order for it to be considered a valid submitting process. I've even run in to websites that require you to use send_keys 1 at a time for usernames and passwords to avoid some anti-scraping technologies they employ. In these cases, I usually use the following between the calls:
from random import random, randint
def sleepyTime(first=5, second=10):
# returns the value of the time slept (as float)
# sleeps a random amount of time between the number variable in first
# and the number variable second (in seconds)
sleepy_time = round(random() * randint(first, second), 2)
sleepy_time = sleepy_time if sleepy_time > first else (first + random())
sleep(sleepy_time)
return sleepy_time
I don't see what use you have for making the _email and _password variables global, unless they are being changed somewhere in the login function and you want that change to be precipitated out to the other scopes.
How I would try to solve it
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException, TimeoutException
TIME_TIMEOUT = 20 # Twenty-second timeout default
def eprint(*args, **kwargs):
""" Prints an error message to the user in the console (prints to sys.stderr), passes
all provided args and kwargs along to the function as usual. Be aware that the 'file' argument
to print can be overridden if supplied again in kwargs.
"""
print(*args, file=sys.stderr, **kwargs)
def login(driver):
global _email, _password
try:
email = WebDriverWait(driver, TIME_TIMEOUT).until(EC.presence_of_element_located((By.XPATH, "//input[#id='user_email']")))
pw = WebDriverWait(driver, TIME_TIMEOUT).until(EC.presence_of_element_located((By.XPATH, "//input[#id='password']"))
pw.submit()
# if this doesn't work try the following:
# btn_submit = WebDriverWait(driver, TIME_TIMEOUT).until(EC.element_to_be_clickable((By.XPATH, "//button[#track-element='click-for-login']"))
# btn_submit.click()
# if that doesn't work, try to add some random wait times using the
# sleepyTime() example from above to add some artificial waiting to your email entry, your password entry, and the attempt to submit the form.
except NoSuchElementException as ex:
eprint(ex.msg())
except TimeoutException as toex:
eprint(toex.msg)
if __name__ == '__main__':
driver = webdriver.Chrome('/usr/bin/chromium/chromedriver',chrome_options=chrome_options)
#d.get('https://www.google.nl/')
#driver = webdriver.Chrome()
driver.maximize_window()
driver.get('https://www.goeventz.com/')
if login(driver) is not None:
print(create_event(driver))
For headless chrome browser you need to provide window size as well in chrome options.For headless browser selenium unable to know what your window size.Try that and let me know.
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('window-size=1920x1480')
I faced this error as well. Now check your browser if the element is inside the iframe. If so, use driver.find_element(By.CSS_SELECTOR, "#payment > div > div > iframe") and driver.switch_to.frame(iframe) Then you will be able to work out.

The web scraping issue in python, web page not loading in time

I am making a program for scrapping the Amazon websites mobile phones but my program is giving me timeout exception even after the page is loaded on time.
Here is my code
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import NoSuchElementException
from bs4 import BeautifulSoup
import urllib.request
class Amazon_all_mobile_scraper:
def __init__(self):
self.driver = webdriver.Firefox()
self.delay = 60
self.url = "https://www.amazon.in/mobile-phones/b/ref=sd_allcat_sbc_mobcomp_all_mobiles?ie=UTF8&node=1389401031"
def load_amazon(self):
self.driver.get(self.url)
try:
wait = WebDriverWait(self.driver,self.delay)
wait.until(EC.presence_of_element_located((By.CLASS_NAME,"acs-ln-link")))
print("Page is ready.")
except TimeoutException:
print("Took too much time to load!")
except:
print("Something went wrong in loading part!!")
def extract_list_of_mobiles(self):
try:
mobile_list = self.driver.find_element_by_xpath('//div[#class = "acs-ln-link"]')
print(mobile_list)
except NoSuchElementException:
print("Sorry, Unable to get the requested element")
scraper = Amazon_all_mobile_scraper()
scraper.load_amazon()
scraper.extract_list_of_mobiles()
Please help me to figure out whats wrong in this code.
Only changing from acs-ln-link to acs-ln-links will not do the trick. Your xpath should look more like '//div[contains(#class,"acs-ln-nav-expanded")]//*[#class="acs-ln-links"]//a'. This is, however, you can cope with to get the required output:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
class Amazon_all_mobile_scraper:
url = "https://www.amazon.in/mobile-phones/b/ref=sd_allcat_sbc_mobcomp_all_mobiles?ie=UTF8&node=1389401031"
def __init__(self):
self.driver = webdriver.Chrome()
self.wait = WebDriverWait(self.driver, 15)
def load_n_get_from_amazon(self):
self.driver.get(self.url)
mobile_list = self.wait.until(EC.presence_of_all_elements_located((By.XPATH,'//div[contains(#class,"acs-ln-nav-expanded")]//*[#class="acs-ln-links"]//a')))
return mobile_list
def __del__(self):
self.driver.close()
if __name__ == '__main__':
scraper = Amazon_all_mobile_scraper()
for item in scraper.load_n_get_from_amazon():
print(f'{item.text}\n{item.get_attribute("href")}\n')
The class wasn't matching "acs-ln-link" should be "acs-ln-links".

Resources