I'm trying to scrape a website with show more button; and I'm not able to click on it.
The website is: https://www.wtatennis.com/rankings/singles
And my code is:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver import ActionChains
from tqdm import tqdm
import time
options = Options()
options.add_argument("--headless")
browser = webdriver.Chrome(ChromeDriverManager().install(),options=options)
browser.get('https://www.wtatennis.com/rankings/singles')
action = ActionChains(browser)
showmore = browser.find_elements_by_xpath(".//button[contains(#class, 'btn widget-footer__more-button rankings__show-more js-show-more-button')]")
action.move_to_element(showmore).perform()
showmore.click()
time.sleep(5)
Has anyone any idea? Thanks!
Don't use './/' in your locator when you are starting the search from root, as there is no current element your locator won't find any element. Also you can use any attribute to find elements uniquely. see below code:
browser = webdriver.Chrome(options=options)
browser.get('https://www.wtatennis.com/rankings/singles')
WebDriverWait(browser,10).until(EC.element_to_be_clickable((By.XPATH,
'//*[#data-text="Accept Cookies"]'))).click()
WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH,
'//*[#data-text = "Show More"]'))).click()
use webdriver wait and data attributes
tu use wait import:
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
To wait till all elements are loaded you have to make sure last element is not changing , if its changing keep scrolling .
browser.get('https://www.wtatennis.com/rankings/singles')
WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH,
'//*[#data-text="Accept Cookies"]'))).click()
value = "start"
WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH,
'//*[#data-text = "Show More"]'))).click()
while(browser.find_element_by_xpath("(//tr[#class='rankings__row'])[last()]").text != value):
elem = browser.find_element_by_xpath(
'(//*[contains(text(),"Loading")])[2]')
value = browser.find_element_by_xpath(
"(//tr[#class='rankings__row'])[last()]").text
browser.execute_script("arguments[0].scrollIntoView()", elem)
WebDriverWait(browser, 10).until(EC.presence_of_all_elements_located((By.XPATH,
"//tr[#class='rankings__row']")))
try:
WebDriverWait(browser, 10).until_not(EC.text_to_be_present_in_element((By.XPATH,
"(//tr[#class='rankings__row'])[last()]"), value))
except:
None
Related
I am trying to collect data on a google map webpage, this is the link. link
This is the code I have tried. My idea is to scroll to the "website name" (you can find the website name once you scroll down) once is present in the browser. but it is not scrolling.
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.chrome.service import Service as ChromeService
from webdriver_manager.chrome import ChromeDriverManager
driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()))
driver.maximize_window()
driver.get("https://www.google.com/maps/place/Amsterdamsche+Athleten+Club+Hercules/#52.36937,4.8049968,16.25z/data=!4m5!3m4!1s0x47c5e3c9cbb3d913:0xef85f93ef996cc06!8m2!3d52.3692292!4d4.8056684")
img_result = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH,'//*[#id="QA0Szd"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[7]/div[5]/a/div[1]/div[2]/div[1]')))
driver.execute_script("arguments[0].scrollIntoView(true);",img_result)
print(img_result.text)
driver.close()
what is the solution for this?
EDIT:This is what I'm trying to get.
At least on my side I see no need to scroll.
The following code worked:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 20)
url = "https://www.google.com/maps/place/Amsterdamsche+Athleten+Club+Hercules/#52.36937,4.8049968,16z/data=!4m5!3m4!1s0x47c5e3c9cbb3d913:0xef85f93ef996cc06!8m2!3d52.3692292!4d4.8056684"
driver.get(url)
name = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "[data-item-id='authority']"))).text
print(name)
Output:
aachercules.nl
The same could be done with XPath instead of CSS Selectors.
This is the XPath I used:
name = wait.until(EC.visibility_of_element_located((By.XPATH, "//a[#data-item-id='authority']"))).text
click() method is not working in selenium python. I have used all of the methods which are available in selenium documentation. I want to apply automation to this URL.
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from time import sleep
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
ser = Service("D:\chromedriver")
op = webdriver.ChromeOptions()
driver = webdriver.Chrome(service=ser, options=op)
actionChains = ActionChains(driver)
driver.get('https://cultivatedculture.com/mailscoop/')
driver.maximize_window()
sleep(2)
# log = driver.find_element(By.LINK_TEXT, "LOG IN")
# sleep(2)
# log.click()
for i in range(3):
#-----------------------------------------------------
name = 'jay kakadiya'
domain = 'gmail.com'
inp = driver.find_element(By.ID ,'name')
inp.send_keys(name)
inp2 = driver.find_element(By.ID ,'domain')
inp2.send_keys(domain)
sleep(1)
btn = driver.find_element(By.XPATH, '//*[#id="find_btn"]')
sleep(1)
btn.click()
actionChains.move_to_element(btn).click().perform()
print("press click")
#-----------------------------------------------------------
# if i == 0:
#
# popup1 = driver.find_element(By.XPATH('//*[#id="jsSignupModalForm"]/div[2]/div/p[5]/span'))
# sleep(1)
# actionChains.move_to_element(popup1).click().perform()
# popup2 = driver.find_element(By.XPATH('//*[#id="jsLoginModalForm"]/div[2]/div/div[1]/div[3]'))
# sleep(1)
# actionChains.move_to_element(popup2).click().perform()
# driver.find_element_by_id('jsUserLoginModal').send_keys('pagiri2277#flowminer.com')
# driver.find_element_by_id('jsUserLoginModal').send_keys('jaykakadiya63522')
# sleep(1)
# popup3 = driver.find_element(By.XPATH, '//*[#id="jsLoginModalForm"]/div[2]/div/div[2]/div"]')
# actionChains.move_to_element(popup3).click().perform()
You can use the below css selector:
button#find_btn span
it is unique in HTMLDOM.
You can click it with using explicit wait like below:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button#find_btn span"))).click()
Imports:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Not sure why would you need a loop for this though.
Once you fillup the Full Name and Company Website fields next to click on the element Find It you need to induce WebDriverWait for the element_to_be_clickable() and you can use the following locator strategies:
driver.get('https://cultivatedculture.com/mailscoop/')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input#name[name='name']"))).send_keys("jay kakadiya")
driver.find_element(By.CSS_SELECTOR, "input#domain[name='domain']").send_keys("gmail.com")
driver.execute_script("arguments[0].click();", WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button#find_btn > span > span"))))
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Browser Snapshot:
I'm trying to get the links of the posts on this page, but they are apparently generated by clicking each of the post images. I'm using Selenium and beautifulsoup4 in Python 3.8.
Any idea how to get the links while selenium continues to the next pages?
url: https://www.goplaceit.com/cl/mapa?id_modalidad=1&tipo_pro//*[#id=%22gpi-property-list-container%22]/div[3]/div[1]/div[1]/imgpiedad=1%2C2&selectedTool=list#12/-33.45/-70.66667
after clicking on the image it opens a new tab with the following type of shortening url: https://www.goplaceit.com/propiedad/6198212
which sends me to a url type:
https://www.goplaceit.com/cl/propiedad/venta/departamento/santiago/6198212-depto-con-1d-1b-y-terraza-a-pasos-del-metro-toesca-bodega
My code:
from bs4 import BeautifulSoup
from selenium import webdriver
import time
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
import winsound
from timeit import default_timer as timer
from selenium.webdriver.common.keys import Keys
start = timer()
PROXY = "PROXY" # IP:PORT or HOST:PORT
path_to_extension = r"extension"
options = Options()
#options.add_argument("--incognito")
options.add_argument('load-extension=' + path_to_extension)
#options.add_argument('--disable-java')
options.headless = False
prefs = {"profile.default_content_setting_values.notifications" : 2}
prefs2 = {"profile.managed_default_content_settings.images": 2}
prefs.update(prefs2)
prefs3 = {"profile.default_content_settings.cookies": 2}
prefs.update(prefs3)
options.add_experimental_option("prefs",prefs)
options.add_argument("--start-maximized")
options.add_argument('--proxy-server=%s' % PROXY)
driver = webdriver.Chrome('chromedriver.exe', options=options)
driver.get('https://www.goplaceit.com/cl/')
WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="root"]/nav/div/div[2]/div[1]/button'))).click()
correo = driver.find_element(By.XPATH, '//*[#id="email"]')
correo.send_keys("Mail")
contraseña = driver.find_element(By.XPATH, '//*[#id="password"]')
contraseña.send_keys("password")
contraseña.send_keys(Keys.ENTER)
time.sleep(7)
elem.driver.find_element(By.XPATH, '//*[#id="gpi-main-landing-search-input"]/div/input')
elem.click()
elem.send_keys("keywords")
WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="gpi-main-landing-search-input"]/div/div[1]/ul/li[1]/a/div/div[1]'))).click()
buscador.send_keys(Keys.ENTER)
WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="root"]/div/div/div[1]/div[2]/div/div[1]/div/div[1]/button'))).click()
WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="custom-checkbox"]'))).click()
page_number = 0
max_page_number = 30
while page_number<=max_page_number:
WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.XPATH, '//button[contains(text(),"paginator-btn-right")]'))).click()
You can get easily the urls by clicking on an image, saving your url, coming back to the first page and repeating this for all the images:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
driver.get("https://www.goplaceit.com/cl/mapa?id_modalidad=1&tipo_propiedad=1%2C2&selectedTool=list#8/-33.958/-71.206")
images = WebDriverWait(driver, 30).until(EC.presence_of_all_elements_located((By.XPATH, "//div[#class='sc-iyvyFf ljSqTz']//img")))
urls = []
for i, image in enumerate(images):
window_before = driver.window_handles[0]
image.click()
driver.implicitly_wait(2)
window_after = driver.window_handles[i+1]
driver.switch_to.window(window_after)
urls.append(driver.current_url)
driver.switch_to.window(window_before)
Trying to create script which will subscribe to news automatically, but stuck with a problem, selenium not able to find email input and submit button. Everytime getting selenium.common.exceptions.NoSuchElementException:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
chrome_options = Options()
chrome_options.add_argument("--window-size=1920x1080")
path_to_chromedriver = 'chromedriver'
driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=path_to_chromedriver)
driver.get('https://dataengweekly.com/')
driver.find_element_by_tag_name("body").send_keys(Keys.PAGE_DOWN)
email_input = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CSS_SELECTOR, 'input[type="email"]'))
)
email_input.send_keys("email#test.com")
driver.find_element_by_css_selector('button.subscribe-btn').click()
time.sleep(10)
Note - Your subscription textbox is in a different iframe, To work with that iframe you need to first switch to that iframe.
Try below code and let me know if you need more clarification -
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver import ActionChains
import time
chrome_options = Options()
chrome_options.add_argument("--window-size=1920x1080")
driver = webdriver.Chrome(options=chrome_options)
wait = WebDriverWait(driver, 5)
action = ActionChains(driver)
driver.get('https://dataengweekly.com/')
iframe = driver.find_element_by_xpath('//iframe')
driver.switch_to.frame(iframe)
email_input = wait.until(EC.presence_of_element_located((By.XPATH, "//input[#type='email']")))
action.move_to_element(email_input).click().send_keys("email#test.com").perform()
driver.find_element_by_css_selector('button.subscribe-btn').click()
time.sleep(2)
I am trying to download history data and click on link to historical data. However even though the Xcode is correct I get this error:
NoSuchElementException: no such element: Unable to locate element.
Code trials:
from selenium.webdriver.support.ui import WebDriverWait
from selenium import webdriver
driver = webdriver.Chrome(executable_path='/Users/Documents/Coding/chromedriver')
url = "https://www.vanguardinvestor.co.uk/investments/vanguard-lifestrategy-100-equity-fund-accumulation-shares/price-performance?intcmpgn=blendedlifestrategy_lifestrategy100equityfund_fund_link"
driver.get(url)
wait = WebDriverWait(driver, 10)
elem = driver.find_element_by_xpath("//*[#id='prices-and-performance-tab']/div/div[4]/div[3]/div[1]/div[1]/div[3]/div/div/div[2]/div/table/tfoot/tr/td/a")
webdriver.ActionChains(driver).move_to_element(elem).click(elem).perform()
To click on the element wait for the page to load and element to be clickable and then click.Try entire snippet.
from selenium.webdriver.support.ui import WebDriverWait
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
driver = webdriver.Chrome(executable_path='/Users/Documents/Coding/chromedriver')
url = "https://www.vanguardinvestor.co.uk/investments/vanguard-lifestrategy-100-equity-fund-accumulation-shares/price-performance?intcmpgn=blendedlifestrategy_lifestrategy100equityfund_fund_link"
driver.get(url)
element = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//span[text()[contains(.,'Price & Performance')]]")))
element.click
To click on the element with text as Search for more historical prices you need to induce WebDriverWait for the element to be clickable and you can use the following solution:
Code Block:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument("--start-maximized")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(chrome_options=options, executable_path = r'C:\Utility\BrowserDrivers\chromedriver.exe' )
driver.get("https://www.vanguardinvestor.co.uk/investments/vanguard-lifestrategy-100-equity-fund-accumulation-shares/price-performance?intcmpgn=blendedlifestrategy_lifestrategy100equityfund_fund_link")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[#id='bannerButton']"))).click()
more_historical_prices = WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.LINK_TEXT, "Search for more historical prices")))
driver.execute_script("arguments[0].scrollIntoView(true);", more_historical_prices)
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.LINK_TEXT, "Search for more historical prices")))
Browser Snapshot: