How to deal with nested iframes with Selenium and Python - python-3.x

I'm trying to automate a daily task, but I got stuck with some nested iframes. The script is working very well till the part of unselecting all document types, it fails.
I tried waiting until the element is clickable. Also, I tried accessing the element with id, css_selector and XPath but nothing helped.
I think the element is located within nested iframes, but can't get the correct sequence.
Here's my script:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
options = webdriver.ChromeOptions()
options.add_argument("disable-infobars")
#options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument('--disable-dev-shm-usage')
#options.add_argument('--disable-gpu')
options.add_argument("--start-maximized")
options.add_experimental_option("excludeSwitches",["ignore-certificate-errors"])
options.add_argument("--disable-blink-features=AutomationControlled")
options.add_argument('--ignore-certificate-errors')
options.add_argument('--ignore-ssl-errors')
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
link = r"https://countyfusion10.kofiletech.us/countyweb/loginDisplay.action?countyname=TylerWV"
options.add_experimental_option("prefs", {
"download.directory_upgrade":True,
"safebrowsing.enabled":True,
"download.prompt_for_download":False,
"plugins.always_open_pdf_externally": True,
})
driver = webdriver.Chrome(executable_path='./chromedriver', options=options)
driver.get(link)
#Login as Guest
driver.find_element_by_xpath(r'//*[#id="maindiv"]/table[2]/tbody/tr[1]/td[2]/table/tbody/tr/td/center/input').click()
time.sleep(5)
#Accept Disclaimer
driver.switch_to.frame("bodyframe")
driver.find_element_by_id(r'accept').click()
time.sleep(30)
#Deselect All types
driver.switch_to.default_content()
boxes = WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.XPATH,r'//*[#id="elemAllNamesJSP_20"]/table/tbody/tr/td[2]/span/input[1]')))
print(boxes)

I'm not sure you had to switch to the default content, anyway since you got out of the bodyframe iframe to access the "boxes" element you have switch to the first parent iframe first, then to the inner parent and then to the inner child. Like this:
driver = webdriver.Chrome(executable_path='./chromedriver', options=options)
driver.get(link)
#Login as Guest
driver.find_element_by_xpath(r'//*[#id="maindiv"]/table[2]/tbody/tr[1]/td[2]/table/tbody/tr/td/center/input').click()
#Accept Disclaimer
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it(By.NAME,"bodyframe"))
driver.find_element_by_id(r'accept').click()
#Deselect All types
driver.switch_to.default_content()
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it(By.NAME,"bodyframe"))
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it(By.NAME,"dynSearchFrame"))
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it(By.NAME,"criteriaframe"))
boxes = WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.XPATH,r'//*[#id="elemAllNamesJSP_20"]/table/tbody/tr/td[2]/span/input[1]')))
print(boxes)
I'm not sure what you trying to achieve by printing the web element "boxes" with print(boxes)...
Also I removed the hardcoded delays. Especially the 30 seconds sleep, you don't need it. Use webdriver explicit wait instead.
It will work much faster and still stable.

Related

Can't locate this input element selenium

I am trying to locate the input “to” field on this webpage: Link here
(sorry about having to login to see it)
My code:
name_field = driver.find_element_by_xpath("//form[#id='compose-message']/div[6]/div/div/div[2]/div/div/textarea")
But that gives the following error:
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//form[#id='compose-message']/div[6]/div/div/div[2]/div/div/textarea"}
(Session info: chrome=93.0.4577.63)
For some reason this page is the only page that has given me any issue.
Most of the textarea web elements are wrapped inside iframes.
Using explicit waits, does give your script a stability that it need.
Also prefer to use css selector over xpath.
In Selenium, to access elements which are inside of iframe, we need to switch the focus of webdriver to iframe. Like below
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "iframe[src^='/message/compose/']")))
and once you are inside this iframe you can interact with to input web element. Like below
WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.NAME, "to"))).send_keys('something here')
WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//span[text()='message']/following-sibling::div/descendant::textarea"))).send_keys('Some message')
Imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Also, remember to switch to default content when you are done with iframe.
driver.switch_to.default_content()
In order to work with elements which are outside of this iframe.
The Element you are trying to find is in an iframe. Was able to find and send text to the elements with below code.
from selenium import webdriver
from selenium.webdriver import ChromeOptions
import time
opt = ChromeOptions()
opt.add_argument("--user-data-dir=C:\\Users\\*****\\AppData\\Local\\Google\\Chrome\\User Data")
driver = webdriver.Chrome(executable_path="path to chromedriver.exe",options=opt)
driver.maximize_window()
driver.implicitly_wait(10)
driver.get("https://www.reddit.com/message/compose/")
driver.switch_to.frame(driver.find_element_by_xpath("//iframe[#class='saPujbGMyXRwqISHcmJH9']"))
driver.find_element_by_xpath("//input[#name='to']").send_keys("Sample Text")
driver.find_element_by_xpath("//div[#class='usertext']//textarea").send_keys("Sample Text")
time.sleep(5)
driver.quit()
First of all that element is inside an iframe. To access it you need to switch to that iframe. Also, your locator are not looking good.
Please try the following code:
wait = WebDriverWait(driver, 20)
wait.until(EC.frame_to_be_available_and_switch_to_it((By.XPATH,"//iframe[contains(#src,'message')]")))
name_field = wait.until(EC.visibility_of_element_located((By.XPATH, "//input[#name='to']")))
To use the above expected conditions you will need the following imports:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
use command CSS selector instead of XPath such as use below code.
yourwebsriver.find_elements_by_css_selector("textarea[name=text]").send_keys('something here')

Python Selenium - Select list item from unordered list

Website: https://vahan.parivahan.gov.in/vahan4dashboard/vahan/view/reportview.xhtml'
I'm trying to use selenium to download data from this website but it's setup in a confusing way. I need to figure out how to use the dropdown in the list called 'Y-Axis' and select 'Maker' from that list. Then I need to hit the 'refresh' button and the 'download excel' button. This is the html of the dropdown menu:
<select id="yaxisVar_input" name="yaxisVar_input" tabindex="-1" aria-hidden="true" onchange="PrimeFaces.ab({s:"yaxisVar",e:"change",f:"masterLayout_formlogin",p:"yaxisVar",u:"xaxisVar"});"><option value="Vehicle Category" data-escape="true">Vehicle Category</option><option value="Vehicle Class" selected="selected" data-escape="true">Vehicle Class</option><option value="Norms" data-escape="true">Norms</option><option value="Fuel" data-escape="true">Fuel</option><option value="Maker" data-escape="true">Maker</option></select>
This is the code I'm playing around with:
from selenium import webdriver
from selenium.webdriver.support.ui import Select
driver = webdriver.Chrome('C:/Users/abhay.singh/chromedriver')
driver.get('https://vahan.parivahan.gov.in/vahan4dashboard/vahan/view/reportview.xhtml')
# Get the y-axis selector
# select = Select(driver.find_element_by_id('yaxisVar_input'))
# select.select_by_visible_text('Maker').click()
# print(select.options)
# print([o.text for o in select.options])
driver.find_element_by_xpath("//select[#name='yaxisVar_input']/option[text()='Maker']").click()
I'd appreciate your help in figuring this out!
This is the best I could do. One thing to note is that selenium is not really meant for downloading, so I added a sleep at the end to ensure the download completes. It can also be done with scripts to monitor the download status, but I don't really know how to do it. I also had to add a sleep in the middle to make sure the "Maker" click was being captured correctly. I'm sure there is a better way to do that as well.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
with webdriver.Chrome() as driver:
driver.get("https://vahan.parivahan.gov.in/vahan4dashboard/vahan/view/reportview.xhtml")
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "label[id='yaxisVar_label']"))).click()
time.sleep(2)
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "li[data-label='Maker']"))).click()
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "j_idt61"))).click()
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "vchgroupTable:xls"))).click()
time.sleep(10)
You might be able to do this with a requests.post(). I didn't look into the headers or form data, but it is there.

Can not pick up the element using python selenium binding from a website for unknown reasons

This is the website "https://agta.org/directory/", no matter what i tried i was not able to click the search button, i gave a delay clicked the button manually hoping for the script to pick up elements at the next stage, but it did not worked as well, "find_element_by_xpath" is returning empty objects. I can't understand why is it happening that way when the x path is 100% accurate. I have used similar script for other site but never had any problem.
For clicking the search button i used java script, action chains as well, still failed.
#using selenium
#search=driver.find_element_by_xpath("//input[#class='DLGButton']").click()
#using js
#driver.execute_script("document.getElementsByClassName('DLGButton')[0].click()")
Here is my detailed code:
options = Options()
options.add_argument("--start-maximized")
driver = webdriver.Chrome(options=options, executable_path=r'D:\chromedriver.exe')
url='https://agta.org/directory/'
driver.get(url)
time.sleep(10)
search=driver.find_element_by_xpath("//input[#class='DLGButton']").click()
time.sleep(10)
listings=driver.find_elements_by_xpath("//div[#class='col-md-3']//a")
print(listings)
check=1
while(check==1):
index=1
for l in listings:
l=driver.find_element_by_xpath("(//div[#class='col-md-3']//a)[{}]".format(index)).get_attribute('href').strip()
print(l)
index=index+1
try:
clickNext=find_element_by_xpath("(//*[contains(text(), 'Next')])[1]").click()
time.sleep()
except Exception as e:
print(e)
check=0
Search button is in the frame. So you should switch the frame first.
I modify your code and click Search Button successfully.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome()
driver.maximize_window()
url='https://agta.org/directory/'
driver.get(url)
time.sleep(5)
# find the frame
frame = driver.find_element_by_xpath("//iframe[#class='directory_iframe']")
driver.switch_to.frame(frame)
# wait element to be clickable
search = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//input[#class='DLGButton']")))
# check button can be found
print(search.get_attribute("value"))
# Scroll the page to show this button (or it will fail to click)
driver.execute_script("arguments[0].scrollIntoView();", search)
search.click()
And you can use Explicit Waits instead of time.sleep()
Ex: To confirm the presence of the element within the DOM Tree.
frame = WebDriverWait(driver, 10).until(EC.presence_of_element_located
((By.XPATH, "//iframe[#class='directory_iframe']")))
Reference: https://selenium-python.readthedocs.io/waits.html

Dealing with reCAPTCHA in Python Selenium

I need to automate a web page using python selenium, but it encounters a reCaptcha, which is in another frame. I want to solve the captcha, and continue the script by clicking the login button, when the reCaptcha has been solved; However, this gets tricky, since a frame is involved, and the frame needs to switch back to the default content. Can anyone help me in this regard?
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
import time
browser = webdriver.Chrome()
browser.delete_all_cookies()
browser.maximize_window()
browser.get("https://developer.syntecx.org/ptcl_ebills/signin.php")
browser.switch_to.frame(browser.find_element_by_tag_name("iframe"))
browser.find_element_by_xpath("//*[#id='recaptcha-anchor']/div[1]").click()
time.sleep(20)
browser.switch_to_default_content()
browser.find_element_by_xpath("//*[#id='login']/button").click()
Once you fill in the Email and Password field to click on the recaptcha you can use the following Locator Strategies:
Code Block:
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get("https://developer.syntecx.org/ptcl_ebills/signin.php")
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input#email"))).send_keys("Asad_Ullah#stackoverflow.com")
driver.find_element_by_css_selector("input#password").send_keys("Asad_Ullah")
WebDriverWait(driver, 10).until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR,"iframe[src^='https://www.google.com/recaptcha/api2/anchor?']")))
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "span.recaptcha-checkbox.goog-inline-block.recaptcha-checkbox-unchecked.rc-anchor-checkbox"))).click()
driver.switch_to_default_content()
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button.btn.btn-primary.block.full-width.m-b"))).click()
Browser Snapshot:
Try do it:
Create a variable that get the main window:
mainWin = driver.current_window_handle
When u need to switch to main window:
driver.switch_to.window(mainWin)
Don't switch to the iframe.
Everything you need is in #g-recaptcha-response and [data-sitekey] which are both in main context.
You can simply wait for the checkbox to show the done icon and then the wait will be over
Like in the code below
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox()
driver.get("http://somedomain/url_that_delays_loading")
try:
element = WebDriverWait(driver, 100).until(
EC.presence_of_element_located((By.ID, "myDynamicElement"))
)
finally:
driver.quit() here
Here I found a solution when you encounter a reCaptcha in a login page.
Simple approach that works very good for me. Check out for login button in a login page. If it lost it's existance, than you can proceed to the next stage. So, my solution does not require any timeout. Happy automating your stuff :)
def check_exists_by_xpath(xpath):
try:
self.driver.find_element_by_xpath(xpath)
except NoSuchElementException:
return False
return True
login_btn_xpath = 'your login button xpath'
while True:
print('You can solve recaptcha during this period and hit the login button.')
if check_exists_by_xpath(login_btn_xpath) is False:
break
# TODO: you can go along from here!

Retrieving elements with custom HTML attributes

I have the following website: https://www.kvk.nl/handelsregister/publicaties/, where I would like to retrieve the login link with Selenium, Scrapy and Python. So for the relevant function, I have the following code:
def start_requests(self):
self.driver = webdriver.Chrome(executable_path=os.path.join(os.getcwd(), "Drivers", "chromedriver.exe"))
self.driver.get(self.initial_url)
test = access_page_wait.until(expected_conditions.visibility_of_element_located((By.CSS_SELECTOR, 'a[data-ui-test-class="linkCard_toegangscode"]')))
if test.is_displayed():
print("+1")
else:
print("-1")
However, this does not seem to work, since it just waits 15 seconds and then it stops. It will never reach +1 or -1.
Now my question is, how can we point selenium to the correct element. It also does not seem to work using XPATH find_elements_by_xpath("//a[#data-ui-test-class='linkCard_toegangscode']").
Should I use another selection approach and if yes, which one?
Because there is Frame which stopping you to access the element.Switch_To iframe and then access the element.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
import os
driver = webdriver.Chrome(executable_path=os.path.join(os.getcwd(), "Drivers", "chromedriver.exe"))
driver.get("https://www.kvk.nl/handelsregister/publicaties/")
driver.switch_to.frame(0)
test=WebDriverWait(driver,10).until(expected_conditions.visibility_of_element_located((By.CSS_SELECTOR, 'a[data-ui-test-class="linkCard_toegangscode"]')))
if test.is_displayed():
print("+1")
else:
print("-1")
Try the above code.It should print what you are looking after.

Resources