Python Selenium Multiple element checking - python-3.x

I'm using python with selenium to access a webpage. And I want to keep checking rather the element's text exist. Only if it exist then the loop stopped. I come up with some code like:
while True:
try:
myElem = WebDriverWait(driver, 0).until(EC.presence_of_element_located((By.XPATH, '//*[#id="EP"]/ol/li/li')))
if(myElem.text == "HELLO"):
print("Found!")
except TimeoutException:
print("Not Found!")
continue
break
Now, the main issue is instead of 1 element. I need to check 3 elements. If any one of the element was found. Then print the found element and stop the loop. How can I achieved this?
WebDriverWait(driver, 0).until(EC.presence_of_element_located((By.XPATH, '//*[#id="EP"]/ol/li/li')))
WebDriverWait(driver, 0).until(EC.presence_of_element_located((By.XPATH, '//*[#id="EP"]/div[2]/p[2]/p[2]')))
WebDriverWait(driver, 0).until(EC.presence_of_element_located((By.XPATH, '//*[#class="class20"]')))
python: 3.11.1, selenium: 4.8.0

Put the try...except... in a function which takes as parameters the element's xpath and text, and returns True if text is found, otherwise False. Then loop and check if any of the three calls returned True.
from selenium.common.exceptions import TimeoutException
def check_presence(xpath, txt):
try:
myElem = WebDriverWait(driver, 0).until(EC.presence_of_element_located((By.XPATH, xpath)))
if myElem.text == txt:
print("Found!",txt)
return True
else:
print(f'Element exists but text "{txt}" does not match')
except TimeoutException:
print("Not Found!",txt)
return False
while 1:
a = check_presence('//*[#id="EP"]/ol/li/li' , "HELLO")
b = check_presence('//*[#id="EP"]/div[2]/p[2]/p[2]' , "HI")
c = check_presence('//*[#class="class20"]' , "CIAO")
if any([a,b,c]):
break

The easiest way to do this is to combine the different XPaths into one and check for the existence of "HELLO" in the XPath itself.
//*[#id="EP"]/ol/li/li[text()="HELLO"] | //*[#class="class20"][text()="HELLO"] | //*[#id="EP"]/div[2]/p[2]/p[2][text()="HELLO"]
If any element is returned, you've found a desired element. You put this in a wait with a reasonable timeout (30s?) and you're done.
try:
myElem = WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.XPATH, '//*[#id="EP"]/ol/li/li[text()="HELLO"] | //*[#class="class20"][text()="HELLO"] | //*[#id="EP"]/div[2]/p[2]/p[2][text()="HELLO"]')))
print("Found!")
except TimeoutException:
print("Not Found!")
NOTE: I don't think //*[#id="EP"]/ol/li/li and //*[#id="EP"]/div[2]/p[2]/p[2] are valid locators. You can't have nested LI or P tags in valid HTML. You might want to check those again and update them.

Related

How to extract multiple text using selenium python

I am scrolling google job page to extract multiple company names, but getting only 2 records.
Can anyone suggest me how to tweak the below code to get all the companies name present next to word 'via' as showing in the below image.
driver.get("https://www.google.com/search?q=bank+jobs+in+india&rlz=1C1CHBF_enIN869IN869&oq=upsc+jo&aqs=chrome.1.69i57j0i433i512j0i131i433i512j0i512l3j0i131i433i512l2j0i512j0i433i512&sourceid=chrome&ie=UTF-8&ibp=htl;jobs&sa=X&sqi=2&ved=2ahUKEwjR27GN_qPzAhX4ppUCHb_0B_QQkd0GegQIORAB#fpstate=tldetail&sxsrf=AOaemvIxuJXh3if0tw7ezZfjkXRe5DSxsA:1632911697417&htivrt=jobs&htidocid=hr3yUBTZAssve05hAAAAAA%3D%3D")
name = []
cnt = 0
try:
while True:
element = driver.find_elements_by_xpath("//div[#role='treeitem']")
driver.execute_script("arguments[0].scrollIntoView(true);", element[cnt])
time.sleep(2)
try:
nam = driver.find_element_by_xpath("//div[contains(#class, 'oNwCmf')]").text
nam1 = nam.split("\nvia ")[1]
name.append(nam1.split("\n")[0])
except:
name.append("")
cnt=cnt+1
except:
pass
Try like this:
Get the name nam using WebElement element(instead of finding with driver). Since we are finding element within elements now, add a dot in the xpath. This will get the name of that particular Element.
try:
while True:
element = driver.find_elements_by_xpath("//div[#role='treeitem']")
driver.execute_script("arguments[0].scrollIntoView(true);", element[cnt])
time.sleep(2)
try:
nam = element[cnt].find_element_by_xpath(".//div[contains(#class, 'oNwCmf')]").text # finds the name of that particular element[cnt], add a dot to find element within element.
print(nam)
nam1 = nam.split("\nvia ")[1]
name.append(nam1.split("\n")[0])
except:
name.append("")
cnt=cnt+1
except:
pass

web scraping data from glassdoor using selenium

please I need some help to run this code (https://github.com/PlayingNumbers/ds_salary_proj/blob/master/glassdoor_scraper.py)
In order to scrape job offer data from Glassdoor
Here's the code snippet:
from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException
from selenium import webdriver
import time
import pandas as pd
options = webdriver.ChromeOptions()
#Uncomment the line below if you'd like to scrape without a new Chrome window every time.
#options.add_argument('headless')
#Change the path to where chromedriver is in your home folder.
driver = webdriver.Chrome(executable_path=path, options=options)
driver.set_window_size(1120, 1000)
url = "https://www.glassdoor.com/Job/jobs.htm?suggestCount=0&suggestChosen=false&clickSource=searchBtn&typedKeyword="+'data scientist'+"&sc.keyword="+'data scientist'+"&locT=&locId=&jobType="
#url = 'https://www.glassdoor.com/Job/jobs.htm?sc.keyword="' + keyword + '"&locT=C&locId=1147401&locKeyword=San%20Francisco,%20CA&jobType=all&fromAge=-1&minSalary=0&includeNoSalaryJobs=true&radius=100&cityId=-1&minRating=0.0&industryId=-1&sgocId=-1&seniorityType=all&companyId=-1&employerSizes=0&applicationType=0&remoteWorkType=0'
driver.get(url)
#Let the page load. Change this number based on your internet speed.
#Or, wait until the webpage is loaded, instead of hardcoding it.
time.sleep(5)
#Test for the "Sign Up" prompt and get rid of it.
try:
driver.find_element_by_class_name("selected").click()
except NoSuchElementException:
pass
time.sleep(.1)
try:
driver.find_element_by_css_selector('[alt="Close"]').click() #clicking to the X.
print(' x out worked')
except NoSuchElementException:
print(' x out failed')
pass
#Going through each job in this page
job_buttons = driver.find_elements_by_class_name("jl")
I'm getting an empty list
job_buttons
[]
Your problem is with wrong except argument.
With driver.find_element_by_class_name("selected").click() you are trying to click non-existing element. There is no element matching "selected" class name on that page. This causes NoSuchElementException exception as you can see yourself while you are trying to catch ElementClickInterceptedException exception.
To fix this you should use the correct locator or at least the correct argument in except.
Like this:
try:
driver.find_element_by_class_name("selected").click()
except NoSuchElementException:
pass
Or even
try:
driver.find_element_by_class_name("selected").click()
except:
pass
I'm not sure what elements do you want to get into job_buttons.
The search results containing all the details per each job can be found by this:
job_buttons = driver.find_elements_by_css_selector("li.react-job-listing")

Not able to click on link in <li> elements list python code

Please Help!!
I want to click on all main menu links for this i have written following code:
html_list = self.driver.find_element_by_xpath("//*[#id='main-navigation']/ul")
items = html_list.find_elements_by_tag_name("li")
for item in items:
item.click()
time.sleep(20)
but it clicked on first menu option Brands and then it failed and gave following error:
exception_class = ErrorInResponseException
if status in ErrorCode.NO_SUCH_ELEMENT:
exception_class = NoSuchElementException
elif status in ErrorCode.NO_SUCH_FRAME:
exception_class = NoSuchFrameException
elif status in ErrorCode.NO_SUCH_WINDOW:
exception_class = NoSuchWindowException
elif status in ErrorCode.STALE_ELEMENT_REFERENCE:
exception_class = StaleElementReferenceException
elif status in ErrorCode.ELEMENT_NOT_VISIBLE:
exception_class = ElementNotVisibleException
elif status in ErrorCode.INVALID_ELEMENT_STATE:
exception_class = InvalidElementStateException
E selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
I am attaching screenshot of the webpage on which I am performing this action.
html_list = WebDriverWait(driver, 10).until(
EC.presence_of_all_elements_located((By.XPATH, "//*[#id = 'main-navigation']//li[.//a[not(#data-webtrekk-link-id='header.subnav')]]")))
for i in range(len(html_list)):
time.sleep(3)
html_list = WebDriverWait(driver, 10).until(
EC.presence_of_all_elements_located((By.XPATH, "//*[#id = 'main-navigation']//li[.//a[not(#data-webtrekk-link-id='header.subnav')]]")))
html_list[i].click()
there are many invisible elements , you should use the xpath locator as mentioned to find the element uniquely.
we are using time.sleep because presenceofallelement doesn't wait for all elements but for only first element
Difference between presenceOfElementLocated() and presenceOfAllElementsLocatedBy() is Selenium
When you click on element from items, it changes in DOM making it impossible to continue to iterate through items because of StaleElementReference. Inspect elements to see how the structure changes.
I would suggest iterating by index instead of elements
html_list = self.driver.find_element_by_xpath("//*[#id='main-navigation']/ul")
items = html_list.find_elements_by_tag_name("li")
for i, _j in enumerate(items, start=1):
self.driver.find_element_by_xpath("(//*[#id='main-navigation']/ul/li)[{}]".format(i))
time.sleep(20)

How to find out what WebDriverWait is successful?

I have a wait code for one of the items I need:
WebDriverWait(driver, 5).until(lambda driver: driver.find_elements(
By.XPATH, 'XPATH') or driver.find_elements(
By.XPATH, 'XPATH') or driver.find_elements(
By.XPATH, 'XPATH'))
Can I somehow find out which element I waited for?
I know that I can check for the existence of these elements like this:
if driver.find_elements(By.XPATH, 'XPATH') > 0:
True
elif driver.find_elements(By.XPATH, 'XPATH') > 0:
True
But it is not as beautiful as I want. I am interested in the possibility of identification inside the until function.
If you have any other ideas I would love to hear them out.
In Python, and operator works in special way.
Try this:
WebDriverWait(driver, 5).until(lambda driver: (driver.find_elements(
By.XPATH, 'XPATH1') and 1) or (driver.find_elements(
By.XPATH, 'XPATH2') and 2) or (driver.find_elements(
By.XPATH, 'XPATH3') and 3))
This will return 1, 2 or 3 based on which XPATH was found. You can replace these numbers with any String/expression as well. Only condition is it should be an expression which can pass if (expression) in python.

how to use "if" to test the presence of an element in the webpage with python?

I created a loop (while True) to automate a task on the site with python. This code clicks on two fields until an element appears on the page
(browser.find_element_by_id ('formComp: buttonBack').
When this element appears, I want the loop to stop and go to the next block of code.
I tested it that way, but it made a mistake. Python reported that the element "formComp: buttonback" was not found. But that's just it, if not found continue the loop:
while (browser.find_element_by_id('formComp:repeatCompromissoLista:0:tableRealizacao:0:subtableVinculacoes:0:vinculacao_input')):
vinc = wait.until(EC.presence_of_element_located((By.ID, 'formComp:repeatCompromissoLista:0:tableRealizacao:0:subtableVinculacoes:0:vinculacao_input')))
vinc = browser.find_element_by_id('formComp:repeatCompromissoLista:0:tableRealizacao:0:subtableVinculacoes:0:vinculacao_input')
vinc.send_keys('400')
enterElem5 = wait.until(EC.element_to_be_clickable((By.ID, 'formComp:buttonConfirmar')))
enterElem5 = browser.find_element_by_id('formComp:buttonConfirmar')
enterElem5.send_keys(Keys.ENTER)
time.sleep(int(segundosv))
if (browser.find_element_by_id('formComp:buttonRetornar')== True):
break
else:
continue
Try like this hope this helps.Check the length count of the button more than 0.
if (len(browser.find_elements_by_id('formComp:buttonRetornar'))>0):
break
else:
continue
find_element_by_id() does not return False when an element is not found. Instead, it raises selenium.common.exceptions.NoSuchElementException. You can handle the exception to get the flow control you are looking for:
try:
browser.find_element_by_id('formComp:buttonRetornar')
break
except NoSuchElementException:
continue

Resources