When I run this, it only manages to scroll down once, and it throws a "Message: element not interactable" error. (it's supposed to scroll twice). When I tried to run it in a loop (made a try and except to ignore the error), and scrolled around with it manually, it would keep pushing me back up to a specific position. But that's strange, because I'm using arrow keys here, not a move to element:
ActionChains(driver).move_to_element(driver.sl.find_element_by_id('my-id')).perform()
I've tried: giving everything more time to load with sleep, hovering over the element and clicking it to make it interactable, using other methods to scroll such as this one and others like it: driver.execute_script("window.scrollTo(0, Y)")
I'm very lost at this point, don't know what to do
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from datetime import date
from datetime import datetime
from time import sleep
from random import *
import random, json, selenium, os.path, os
driver = webdriver.Chrome('/Users/apple/Downloads/chromedriver')
driver.maximize_window()
driver.get('https://instagram.com')
sleep(7)
username_form = driver.find_element_by_xpath('/html/body/div[1]/section/main/article/div[2]/div[1]/div/form/div/div[1]/div/label/input')
username_form.clear()
username_form.send_keys('ENTER INSTA USER HERE')
password_form = driver.find_element_by_xpath('/html/body/div[1]/section/main/article/div[2]/div[1]/div/form/div/div[2]/div/label/input')
password_form.clear()
password_form.send_keys('ENTER INSTA PASS HERE')
button_click = driver.find_element_by_xpath('/html/body/div[1]/section/main/article/div[2]/div[1]/div/form/div/div[3]/button')
try:
button_click.click()
except:
driver.execute_script("arguments[0].click();", button_click)
sleep(4)
driver.get('https://instagram.com/p/CQ_sfAeFl5s/')
sleep(4)
like_meter = driver.find_element_by_class_name('zV_Nj')
like_meter.click()
sleep(1)
try:
scroll_zone = driver.find_element_by_xpath('/html/body/div[5]/div/div/div[2]/div/div')
except:
scroll_zone = driver.find_element_by_xpath('/html/body/div[4]/div/div/div[2]/div/div')
scroll_zone.click()
sleep(0.5)
hover = ActionChains(driver).move_to_element(scroll_zone)
hover.perform()
sleep(0.5)
scroll_zone.send_keys(Keys.ARROW_DOWN)
scroll_zone.send_keys(Keys.ARROW_DOWN)
If you want to scroll that list of persons liked that page you can do this:
like_meter = driver.find_element_by_class_name('zV_Nj')
like_meter.click()
sleep(1)
elem = driver.find_element_by_css_selector("div[role='dialog'] div[style*='padding']")
for n in range(10):
driver.execute_script("arguments[0].scrollDown += 20", elem)
The range of 10 and 20 pixels scrolling can be changed according to your needs
Related
i'm new in python, so plz don't bite me =)
The problem is next, i can't understand how to scroll the modal window in instagram using python and selenium. My task is get all people who liked the post. Now i stuck after the modal is opening a
from lxml import html
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-setuid-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument('--headless')
driver = webdriver.Chrome('/usr/bin/chromedriver',
chrome_options=chrome_options)
driver.get('https://www.instagram.com/p/B2Y2nwpCiYa/?igshid=sv6ovbyg07hx')
driver.implicitly_wait(0.7)
for elem in driver.find_elements_by_xpath(
'.//span[#class = "glyphsSpriteGrey_Close u-__7"]'):
elem.click()
print('close login')
button = driver.find_elements_by_xpath('.//button[#class = "sqdOP
yWX7d _8A5w5 "]')
button[0].click()
driver.implicitly_wait(60)
#user_loc = By.CSS_SELECTOR,
'div.Igw0E.rBNOH.eGOV_.ybXk5._4EzTm.XfCBB.HVWg4'
user_loc = By.CSS_SELECTOR, 'button.sqdOP.L3NKy'
#user_loc = By.CSS_SELECTOR, 'div._7UhW9.xLCgt.MMzan.KV-D4.fDxYl'
# Wait for first XHR complete
wait(driver, 20).until(EC.visibility_of_element_located(user_loc))
# Get current length of user list
current_len = len(driver.find_elements(*user_loc))
print(current_len)
while True:
driver.find_element(*user_loc).send_keys(Keys.END)
try:
wait(driver, 10).until(lambda x:
len(driver.find_elements(*user_loc)) > current_len)
current_len = len(driver.find_elements(*user_loc))
# # Return full list of songs
except TimeoutException:
user_list = [user for user in driver.find_elements(*user_loc)]
break
driver.get_screenshot_as_file('test.png')
print(len(user_list))
I understand this question has been asked but I need some solution for this error:
Traceback (most recent call last):
File "goeventz_automation.py", line 405, in <module>
if login(driver) is not None:
File "goeventz_automation.py", line 149, in login
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/support/wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
This is the code where its getting error:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import TimeoutException
import urllib.request as request
import urllib.error as error
from PIL import Image
from selenium.webdriver.chrome.options import Options
import datetime as dt
import time
from common_file import *
from login_credentials import *
def login(driver):
global _email, _password
if waiter(driver, "//a[#track-element='header-login']") is not None:
#login = driver.find_element_by_xpath("//a[#track-element='header-login']")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
#login.click()
if waiter(driver,"//input[#id='user_email']") is not None:
email = driver.find_element_by_xpath("//input[#id='user_email']")
password = driver.find_element_by_xpath("//input[#id='password']")
email.send_keys(_email)
password.send_keys(_password)
driver.find_element_by_xpath("//button[#track-element='click-for-login']").click()
return driver
else:
print("There was an error in selecting the email input field. It may be the page has not loaded properly.")
return None
else:
print("There was an error in selecting the header-login attribute on the page.")
return None
if __name__ == '__main__':
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('/usr/bin/chromium/chromedriver',chrome_options=chrome_options)
#d.get('https://www.google.nl/')
#driver = webdriver.Chrome()
driver.maximize_window()
driver.get('https://www.goeventz.com/')
if login(driver) is not None:
print(create_event(driver))
I think there is some problem with Keys.ENTER, but I don't know how to solve this. I have tried every possible solution.............
This error message...
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
...implies that the desired element was not interactable when you tried to invoke click() on it.
A couple of facts:
When you initialize the Chrome browser always in maximized mode.
You can disable-extensions.
You need to disable-infobars as well.
I have used the same xpath which you have constructed and you can use the following solution:
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
options = webdriver.ChromeOptions()
options.add_argument("start-maximized");
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(chrome_options=options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get("https://www.goeventz.com/")
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
Browser Snapshot:
copy full xpath instead of copying only xpath. It will work
Instead of using login.send_keys(Keys.ENTER) you should use selenium click() method which would work fine for you.
You can check first if the element is clickable first and then you can click on it.
Like:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#track-element='header-login']"))).click()
Overview
It seems like you're having an XPATH problem finding the "Submit" button or your Submit button is not clickable, or your Submit button has some client side events attached to it (javascript/etc) that are required in order to effectively submit the page.
Calling the pw.submit() method in most cases should get rid of the need to wait for the submit button to become clickable and avoid any issues in locating the button in most cases. On many other websites, some of the necessary back-end processes are primed by client-side activities that are performed after the "submit" button is actually clicked (although on a side-note this is not considered best-practice because it makes the site less accessible, etc, I digress). Above all, it's important to watch your script execute and make sure that you're not getting any noticeable errors displayed on the webpage about the credentials that you're submitting.
Also, however, some websites require that you add a certain minimum amount of time between the entry of the username, password, and submitting the page in order for it to be considered a valid submitting process. I've even run in to websites that require you to use send_keys 1 at a time for usernames and passwords to avoid some anti-scraping technologies they employ. In these cases, I usually use the following between the calls:
from random import random, randint
def sleepyTime(first=5, second=10):
# returns the value of the time slept (as float)
# sleeps a random amount of time between the number variable in first
# and the number variable second (in seconds)
sleepy_time = round(random() * randint(first, second), 2)
sleepy_time = sleepy_time if sleepy_time > first else (first + random())
sleep(sleepy_time)
return sleepy_time
I don't see what use you have for making the _email and _password variables global, unless they are being changed somewhere in the login function and you want that change to be precipitated out to the other scopes.
How I would try to solve it
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException, TimeoutException
TIME_TIMEOUT = 20 # Twenty-second timeout default
def eprint(*args, **kwargs):
""" Prints an error message to the user in the console (prints to sys.stderr), passes
all provided args and kwargs along to the function as usual. Be aware that the 'file' argument
to print can be overridden if supplied again in kwargs.
"""
print(*args, file=sys.stderr, **kwargs)
def login(driver):
global _email, _password
try:
email = WebDriverWait(driver, TIME_TIMEOUT).until(EC.presence_of_element_located((By.XPATH, "//input[#id='user_email']")))
pw = WebDriverWait(driver, TIME_TIMEOUT).until(EC.presence_of_element_located((By.XPATH, "//input[#id='password']"))
pw.submit()
# if this doesn't work try the following:
# btn_submit = WebDriverWait(driver, TIME_TIMEOUT).until(EC.element_to_be_clickable((By.XPATH, "//button[#track-element='click-for-login']"))
# btn_submit.click()
# if that doesn't work, try to add some random wait times using the
# sleepyTime() example from above to add some artificial waiting to your email entry, your password entry, and the attempt to submit the form.
except NoSuchElementException as ex:
eprint(ex.msg())
except TimeoutException as toex:
eprint(toex.msg)
if __name__ == '__main__':
driver = webdriver.Chrome('/usr/bin/chromium/chromedriver',chrome_options=chrome_options)
#d.get('https://www.google.nl/')
#driver = webdriver.Chrome()
driver.maximize_window()
driver.get('https://www.goeventz.com/')
if login(driver) is not None:
print(create_event(driver))
For headless chrome browser you need to provide window size as well in chrome options.For headless browser selenium unable to know what your window size.Try that and let me know.
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('window-size=1920x1480')
I faced this error as well. Now check your browser if the element is inside the iframe. If so, use driver.find_element(By.CSS_SELECTOR, "#payment > div > div > iframe") and driver.switch_to.frame(iframe) Then you will be able to work out.
I'm using idle and selenium.
this is my code :
from selenium import webdriver
import time
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
options = webdriver.ChromeOptions()
options.add_argument('--ignore-certificate-errors')
options.binary_location = "/usr/bin/chromium"
driver = webdriver.Chrome()
driver.get('https://www.youtube.com/')
def check_exist_by_class():
try:
driver.find_element_by_class_name("ytp-ad-skip-button ytp-button")
#time.sleep(1)
return 0
except NoSuchElementException:
#time.sleep(1)
return 1
while True:
print(check_exist_by_class())
time.sleep(0.5)
I'm only getting 1 even if the skip add button is visible. tried using the x path but the x path for the button changes with the window size.
Update-
X path doesn't change with the window side. its kind of random. Any idea how to click the skip add button with selenium?
X path s of few ad buttons:
***//*[#id="skip-button:3f"]/span/button
//*[#id="skip-button:2v"]/span/button
//*[#id="skip-button:2v"]/span/button
//*[#id="skip-button:a"]/span/button
//*[#id="skip-button:a"]/span/button
//*[#id="skip-button:a"]/span/button
//*[#id="skip-button:2b"]/span/button***
I know this might not be what you asked for but alternatively you can just get an adblocker for Youtube and load that extension in Selenium:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
path_to_extension = r'C:\Users\YOUR_USER_NAME\Desktop\1.9.0_0'
chrome_options = Options()
chrome_options.add_argument('load-extension=' + path_to_extension)
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.create_options()
driver.get("http://www.google.com")
I am wanting to click all the Href tabs under the main headers and to navigate to those pages to scrape them. For speed of the job, I do am wanting to click the href without having to click the headers. My question is, is there a way to click these buttons even though it is not visible like the page on the right? It does not seem to be working for me. It seems to give me:
Traceback (most recent call last):
File "C:/Users/Bain3/PycharmProjects/untitled4/Centrebet2.py", line 58, in <module>
EC.element_to_be_clickable((By.XPATH, '(//*[#id="accordionMenu1_ulSports"]/li/ul/li/ul/li/a)[%s]' % str(index + 1)))).click()
File "C:\Users\Bain3\Anaconda3\lib\site-packages\selenium\webdriver\support\wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
I have replaced
EC.element_to_be_clickable((By.XPATH, '(//*[#id="accordionMenu1_ulSports"]/li/ul/li/ul/li/a)[%s]' % str(index + 1)))).click()
with
driver.find_element_by_xpath('(//*[#id="accordionMenu1_ulSports"]/li/ul/li/ul/li/a)[%s]' % str(index + 1)).click()
This however does not seem to remedy it as it only clicks visible elements.
My code below is:
from random import shuffle
from selenium.webdriver.support.ui import WebDriverWait as wait
from selenium import webdriver as web
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
from random import randint
from time import sleep
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import csv
import requests
import time
from selenium import webdriver
success = False
while not success:
try:
driver = webdriver.Chrome()
driver.set_window_size(1024, 600)
driver.maximize_window()
driver.get('http://centrebet.com/')
success = True
except:
driver.quit()
sleep(5)
sports = driver.find_element_by_id("accordionMenu1_ulSports")
if sports.get_attribute("style") == "display: none;":
driver.find_element_by_xpath('//ul[#id="menu_acc"]/li[3]/a').click()
driver.find_element_by_xpath(".//*[#data-type ='sports_l1'][contains(text(), 'Soccer')]").click()
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
options = driver.find_elements_by_xpath('//*[#id="accordionMenu1_ulSports"]/li/ul/li/ul/li/a')
# Get list of inetegers [1, 2, ... n]
indexes = [index for index in range(len(options))]
# Shuffle them
shuffle(indexes)
for index in indexes:
# Click on random option
wait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, '(//*[#id="accordionMenu1_ulSports"]/li/ul/li/ul/li/a)[%s]' % str(index + 1)))).click()
I have also tried:
driver.execute_script('document.getElementByxpath("//*[#id="accordionMenu1_ulSports"]/li/ul/li/ul/li/a").style.visibility = "visible";')
To remedy this. Though this simply gives an error. Any ideas on how to resolve this issue of invisible elements?
driver.execute_script('document.getElementByxpath("//*[#id="accordionMenu1_ulSports"]/li/ul/li/ul/li/a").style.visibility = "visible";')
gives you error because it's not correct way to use XPath in Javascript. Correct way you can find here
To scrape required data you can use below code:
import requests
import time
from selenium import webdriver
url = "http://centrebet.com/"
success = False
while not success:
try:
driver = webdriver.Chrome()
driver.set_window_size(1024, 600)
driver.maximize_window()
driver.get(url)
success = True
except:
driver.quit()
time.sleep(5)
sports = driver.find_element_by_id("accordionMenu1_ulSports")
links = [url + link.get_attribute("onclick").replace("menulink('", "").replace("')", "") for link in sports.find_elements_by_xpath('.//a[starts-with(#onclick, "menulink")]')]
for link in links:
print(requests.get(link).text)
Instead of clicking on each link, you can request content of each page with HTTP-GET
You can even try using JavascriptExecutor.
Use below code to make your style attribute = display:block;
driver.execute_script("arguments[0].style.display = 'none'", driver.find_element_by_xpath("//*[#id='accordionMenu1_ulSports']/li/ul/li/ul"))
Note : Make sure you are using correct xpath. your <ul> element is hidden not <a> so so take xpath of that <ul> tag only and try
this is the problem:
I am using selenium to download all the successful projects from this webpage ("https://www.rockethub.com/projects"). The url does not change if a click on any button.
I'm interested in successful project, thus I click on the button status and then I click on successful.
Once on this page I need to scroll down repedetly to make other urls appear.
Here is the problem. So far I have been not able to scroll down the page
This is my code:
from selenium.webdriver import Firefox
from selenium import webdriver
url="https://www.rockethub.com/projects"
link=[]
wd = webdriver.Firefox()
wd.get(url)
next_button = wd.find_element_by_link_text('Status')
next_button.click()
next_but = wd.find_element_by_link_text('Successful')
next_but.click()
wd.execute_script("window.scrollTo(0, document.body.scrollHeight);")
Any idea on how to solve this?
Thanks
Giangi
Since the content is updated dynamically, you need to wait for a change of the content before executing the next step:
class element_is_not(object):
""" An expectation for checking that the element returned by
the locator is not equal to a given element.
"""
def __init__(self, locator, element):
self.locator = locator
self.element = element
def __call__(self, driver):
new_element = driver.find_element(*self.locator)
return new_element if self.element != new_element else None
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
driver = webdriver.Firefox()
wait = WebDriverWait(driver, 10)
driver.get("https://www.rockethub.com/projects")
# get the last box
by_last_box = (By.CSS_SELECTOR, '.project-box:last-of-type')
last_box = wait.until(element_is_not(by_last_box, None))
# click on menu Status > Successful
driver.find_element_by_link_text('Status').click()
driver.find_element_by_link_text('Successful').click()
# wait for a new box to be added
last_box = wait.until(element_is_not(by_last_box, last_box))
# scroll down the page
driver.execute_script("window.scrollTo(0, document.documentElement.scrollHeight);")
# wait for a new box to be added
last_box = wait.until(element_is_not(by_last_box, last_box))
Run the wd.execute_script("window.scrollTo(0, document.body.scrollHeight);") in loop, since each time the script is executed only certain number of data is rerieved, so you have to execute it in a loop.
If you are just looking to retrieve all the successful projects at once and not interested in simulating the scrolling down to the page, then look at this answer, it may help.