Can't able to fetch span class text - python-3.x

Please find the attached image . I want to fetch the highlighted part in the image,
i want to fetch this attributes 1 · Trending ,#tuesdaymotivations,7,750 Tweets .
please advise.
URL to fetch=https://twitter.com/explore/tabs/trending
from selenium import webdriver
url = 'https://twitter.com/explore/tabs/trending'
# scrolling and scraping tweets
driver = webdriver.Chrome('/chromedriver')
driver.get(url)
trends = driver.find_element_by_xpath('//div[#data-testid="trends"]')
trend = trends[0]
trend.find_element_by_xpath('.//span').text
output:
Traceback (most recent call last):
File "/home/PycharmProject/Twitter_trending/ss.py", line 13, in <module>
trends = driver.find_element_by_xpath('//div[#data-testid="trends"]')
File "/usr/local/lib/python3.8/dist-packages/selenium/webdriver/remote/webdriver.py", line 394, in find_element_by_xpath
return self.find_element(by=By.XPATH, value=xpath)
File "/usr/local/lib/python3.8/dist-packages/selenium/webdriver/remote/webdriver.py", line 976, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "/usr/local/lib/python3.8/dist-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python3.8/dist-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//div[#data-testid="trends"]"}
(Session info: chrome=90.0.4430.72)
expected output:
1
·
Trending
#tuesdaymotivations
7,750 Tweets
2
·
Politics · Trending
#ModiResignOrRepeal
37.6K Tweets
3
·
Trending
#ThankfulTuesday
12.8K Tweets...etc

Try to fetch the text using JavaScript. Use this code.
span = trend.find_element_by_xpath('.//span')
text = driver.execute_script("return arguments[0].innerText", span)
This method can also provide text from nested elements. Still not working then try css selector instead of xpath.
trends = driver.find_elements_by_css_selector('div[data-testid="trends"]')
trend = trends[0]
span = trend.find_element_by_xpath('.//span')
text = driver.execute_script("return arguments[0].innerText", span)

Related

How can i grab a URL located in the network>headers tab without Selenium?

So i want to grab a URL located in the network tab > Headers but the specified URL is nowhere to be seen
here is my code so far
import requests
from bs4 import BeautifulSoup
link = 'https://www.tiktok.com/#docfrankhere/video/7128498731908893995?is_from_webapp=1&sender_device=pc&web_id=7111780332651578886'
r = requests.get(link)
response = r.status_code
head = r.headers['Request URL']
print(head)
Output
Traceback (most recent call last):
File "C:\Users\SEBAA\Desktop\copyu.py", line 10, in <module>
head = r.headers['Request URL']
File "C:\Users\SEBAA\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\structures.py", line 54, in __getitem__
return self._store[key.lower()][1]
KeyError: 'request url'
So even when i do
head = r.headers
print(head)
the output don't show me the URL i want to grab
i'm new to this any help would be appreciatedenter image description here

My selenium program isn't able to click on an input box

I have made a program to grab a youtube link and type it on a website but its not able to find the input box...plz tell me why
import time
from selenium import webdriver
import pyautogui as auto
url = 'https://www.youtube.com/results?search_query=closer+8d'
web = webdriver.Chrome('chromedriver.exe')
web.get(url)
time.sleep(3)
link = web.find_element_by_id('video-title')
link = link.get_attribute('href')
auto.hotkey('ctrl', 't')
auto.typewrite('https://www.youtubemp3.pro/en2')
auto.press('enter')
time.sleep(3)
search_bar = web.find_element_by_xpath('//*[#id="videoURL"]')
search_bar.click()
search_bar.send_keys(link)
The error I'm getting is:
Traceback (most recent call last):
File "e:\pranil\python\doing shit with python\songs downloader.py", line 17, in <module>
search_bar = web.find_element_by_xpath('//*[#id="videoURL"]')
File "C:\Users\Pranil.DESKTOP-TLQKP4G.000\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 394, in find_element_by_xpath
return self.find_element(by=By.XPATH, value=xpath)
File "C:\Users\Pranil.DESKTOP-TLQKP4G.000\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 978, in find_element
'value': value})['value']
File "C:\Users\Pranil.DESKTOP-TLQKP4G.000\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\Pranil.DESKTOP-TLQKP4G.000\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//*[#id="videoURL"]"}
(Session info: chrome=83.0.4103.116)
The xpath you provided to find an element doesn't seem to work ""//[#id="videoURL"]"" I also opened up a YouTube video to see what you were looking for but I couldn't find videoURL if you are trying to get the URL I recommend you have selenium click on the share button and grab the link from there.
import time
from selenium import webdriver
import pyautogui as auto
url = 'https://www.youtube.com/results?search_query=closer+8d'
web = webdriver.Chrome('./chromedriver')
web.get(url)
time.sleep(3)
link = web.find_element_by_id('video-title')
link = link.get_attribute('href')
web.execute_script("window.open('https://www.youtubemp3.pro/en2', 'new_window')")
web.switch_to_window(web.window_handles[-1])
ylink = web.find_element_by_id("videoURL")
ylink.click()
ylink.send_keys(link)
submit = web.find_element_by_name("submitForm")
submit.click()
you do not have to use pyautogui it will work better if you use seleniu
try to find the element with it's ID.
import time
from selenium import webdriver
url = 'https://www.youtube.com/results?search_query=closer+8d'
web = webdriver.Chrome('../assets/chromedriver')
web.get(url)
time.sleep(3)
link = web.find_element_by_id('video-title')
link = link.get_attribute('href')
web.get('https://www.youtubemp3.pro/en2/')
time.sleep(3)
search_bar = web.find_element_by_id('txt-url')
search_bar.click()
search_bar.send_keys(link)

how to click a "ng" object using selenium with python?

Im trying to automate clicking on some web elements in order to monitor the web page.
apparently this page is written using AngularJS. I came to this conclusion due to the ng-model, ng-repeat and so on used in this page.
using this code I try to click a specific element but I get an error that the item is not clickable
driver.get("https://sales.aig.co.il/travel")
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'chkDestination-title156')))
elm156 = driver.find_element_by_id("chkDestination-title156")
elm156.click()
no matter what I try to click, Im getting the same error: is not clickable...
this is the complete error:
DevTools listening on ws://127.0.0.1:55004/devtools/browser/a430d472-d53a-4ead-a199-a60d75a2fba6
Traceback (most recent call last):
File "AIGtravel.py", line 16, in <module>
elm156.click()
File "C:\Program Files\Python\Python37\lib\site-packages\selenium\webdriver\remote\webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "C:\Program Files\Python\Python37\lib\site-packages\selenium\webdriver\remote\webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "C:\Program Files\Python\Python37\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Program Files\Python\Python37\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementClickInterceptedException: Message: element click intercepted: Element <label class="ng-binding countries-item_name" for="chkDestination156" id="chkDestination-title156">...</label> is not clickable at point (923, 536). Other element would receive the click: <input type="checkbox" name="selectedDestination" id="chkDestination156" ng-model="step1VM.sharedPolicyService.destinationsSelectionObject.items[$index].selected" data-ng-change="step1VM.selectionChanged()" class="ng-pristine ng-untouched ng-valid ng-empty" aria-invalid="false">
(Session info: chrome=75.0.3770.142)
I've been digging into this a lot and finally came to a solution using WebDriverWait and this code:
driver.get("https://sales.aig.co.il/travel")
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'chkDestination-
title156')))
elm156 = driver.find_element_by_id("chkDestination156")
elm156.click()

selenium python, attempt to click button

I'm trying to click a "load more" button with the following code
browser.find_element_by_xpath('//*[#id="mainContent"]/div[1]/div/div[5]/div/div[1]').click()
browser.find_element_by_css_selector('#mainContent > div.left-panel > div > div.result-list > div > div.content').click()
browser.find_element_by_link_text('Load More').click()
I receive the following error:
Traceback (most recent call last):
File "C:\Users\David\eclipse-workspace\Web_Scrap\setup.py", line 38, in <module>
browser.find_element_by_css_selector('#mainContent > div.left-panel > div > div.result-list > div > div.content').click()
File "C:\Users\David\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "C:\Users\David\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webelement.py", line 628, in _execute
return self._parent.execute(command, params)
File "C:\Users\David\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 320, in execute
self.error_handler.check_response(response)
File "C:\Users\David\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: Element is not clickable at point (239, 698)
(Session info: chrome=68.0.3440.106)
(Driver info: chromedriver=2.41.578737 (49da6702b16031c40d63e5618de03a32ff6c197e),platform=Windows NT 10.0.17134 x86_64)
I've tried each of those 3 individually but can't seem to get selenium to click the button
When pressing inspect that is the following element I receive
The element code is as follows:
<div class="content" onclick="javascript:mtvn.btg.Controller.sendLinkEvent({ linkName:'PROFMIDPANE:LoadMore', linkType:'o' } );">Load More</div>
if anyone has any recommendations on how I can achieve this I would greatly appreciate it!
UPDATE:
I tried the two solutions recommended to me but unfortunately didn't work out, I will post it here if anyone is interested.
iamsankalp89 solution:
wait = WebDriverWait(driver, 10)
element = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#class='content' and text()='Load More']")))
element.click()
Error message:
Traceback (most recent call last):
File "C:\Users\David\eclipse-workspace\Web_Scrap\setup.py", line 39, in <module>
element.click()
File "C:\Users\David\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "C:\Users\David\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webelement.py", line 628, in _execute
return self._parent.execute(command, params)
File "C:\Users\David\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 320, in execute
self.error_handler.check_response(response)
File "C:\Users\David\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: Element is not clickable at point (239, 698)
(Session info: chrome=68.0.3440.106)
(Driver info: chromedriver=2.41.578737 (49da6702b16031c40d63e5618de03a32ff6c197e),platform=Windows NT 10.0.17134 x86_64)
Julian Moreno solution:
ActionChains(driver).move_to_element("//div[#class='content' and text()='Load More']").click("//div[#class='content' and text()='Load More']").perform()
Error Message:
Traceback (most recent call last):
File "C:\Users\David\eclipse-workspace\Web_Scrap\setup.py", line 42, in <module>
ActionChains(browser).move_to_element("//div[#class='content' and text()='Load More").click("//div[#class='content' and text()='Load More").perform()
File "C:\Users\David\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\common\action_chains.py", line 83, in perform
action()
File "C:\Users\David\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\common\action_chains.py", line 293, in <lambda>
Command.MOVE_TO, {'element': to_element.id}))
AttributeError: 'str' object has no attribute 'id'
Try this xpath
//div[#class='content' and text()='Load More']
The code is like this:
browser.find_element_by_xpath('//div[#class='content' and text()='Load More']').click()
Also use WebDriverWait
wait = WebDriverWait(driver, 10)
element = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#class='content' and text()='Load More']")))
element.click()
Try using ActionChains:
class selenium.webdriver.common.action_chains.ActionChains(driver)
ActionChains(driver).move_to_element(your_element).click(your_element).perform()
The move to element will move the mouse (cursor) to the middle of the element (your_element) then the perform function will perform the actions chained.
EDIT
Try this:
load_more = browser.find_element_by_css_selector("#mainContent > div.left-panel > div > div.result-list > div > div.content")
WebDriverWait(browser, timeout).until(EC.visibility_of(load_more))
browser.execute_script("return arguments[0].scrollIntoView(true);", load_more)
ActionChains(browser).move_to_element(load_more).click().perform()
The ActionChains(browser).move_to_element() accepts WebElement object or String which is the id of the element. Since the load more does not have an ID then the move_to_element() could not find the WebElement.
After analysing the web page, I noticed that move_to_element(load_more) moves to the load_more button (only until it is within the screen) but it also triggers scrolling down and that displays the footer of the web page. This footer covers the load_more button. Therefore, you need the browser.execute_script("return arguments[0].scrollIntoView(true);", load_more) which will basically keep scrolling until load_more is in view (regardless of the footer showing up).

Error while executing javascript code in python3 while using selenium

I was trying to remove the ad and other pop-up using selenium after opening the webpage. The pop-up is getting removed but the ad is not getting removed. There is some error in executing javascript code(using it remove ad). For which there is no reason given. Also, for the ad, when I open the webpage myself by typing the link then ad does not appear but when I run the program, in code generated browser ad appears(reason I don't know). I have attached both code and error. Here is the code
from selenium import webdriver
import time
from selenium.webdriver.common.by import By
driver = webdriver.Chrome()
driver.get("http://indianexpress.com/columnists/")
// To remove pop-up
later = driver.find_elements(By.CLASS_NAME, "iz_block_button")
later[0].click()
//to remove ad
all_iframes = driver.find_elements_by_tag_name("iframe")
if len(all_iframes) > 0:
print("Ad Found\n")
driver.execute_script("""
var elems = document.getElementsByTagName('iframe');
for(var i = 0, max = elems.length; i < max; i++)
{
elems[i].visibility=hidden;
}
""")
print('Total Ads: ' + str(len(all_iframes)))
else:
print('No frames found')
driver.close()
Error:-
Traceback (most recent call last):
File "/Users/arjungoyal/Desktop/untitled/a.py", line 23, in <module>
""")
File "/usr/local/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py", line 627, in execute_script
'args': converted_args})['value']
File "/usr/local/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py", line 312, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python3.6/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: hidden is not defined
(Session info: chrome=66.0.3359.181)
(Driver info: chromedriver=2.38.552518 (183d19265345f54ce39cbb94cf81ba5f15905011),platform=Mac OS X 10.13.4 x86_64)
Please someone tell what the error is and how to remove it.

Resources