I am tried to use a headless google chrome to scrapy some website content in macOS Big Sur, this is my python 3 code:
from webbrowser import Chrome
from dolphin.common.commonlogger import CommonLogger
from selenium.webdriver import chrome
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
class FetchMusic:
def __init__(self):
print("fetch...")
if __name__ == '__main__':
opts = Options()
opts.set_headless()
assert opts.headless # Operating in headless mode
browser = Chrome(executable_path=r"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
options=opts)
browser.implicitly_wait(3)
browser.get('https://ca.finance.yahoo.com/quote/AMZN/profile?p=AMZN')
results = browser.find_elements_by_xpath('//*[#id="quote-header-info"]/div[3]/div/div/span[1]')
for result in results:
print(result.text)
when I run this code, shows error:
Traceback (most recent call last):
File "/Users/dolphin/Library/Application Support/JetBrains/Toolbox/apps/PyCharm-P/ch-0/212.5457.59/PyCharm.app/Contents/plugins/python/helpers/pydev/pydevd.py", line 1483, in _exec
pydev_imports.execfile(file, globals, locals) # execute the script
File "/Users/dolphin/Library/Application Support/JetBrains/Toolbox/apps/PyCharm-P/ch-0/212.5457.59/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "/Users/dolphin/source/reddwarf/backend/pydolphin/dolphin/biz/music/fetch.py", line 18, in <module>
opts.set_headless()
AttributeError: 'Options' object has no attribute 'set_headless'
python-BaseException
Process finished with exit code 1
what should I do to make it work?
Instead of
opts = Options()
opts.set_headless()
please use
options = ChromeOptions()
options.add_argument("--headless")
or
options = Options()
options.add_argument("--headless")
Imports :
from selenium.webdriver.chrome.options import Options as ChromeOptions
Related
I am trying to monitor the network in the IE Webdriver, the following is my code:
import time
import psutil
import os
import json
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import trackin_config
caps = DesiredCapabilities.INTERNETEXPLORER
caps['loggingPrefs'] = {'performance': 'ALL'}
browser = webdriver.Ie(trackin_config.WEB_DRIVER, desired_capabilities=caps)
browser.get(trackin_config.MES_WEBSITE)
def process_browser_log_entry(entry):
response = json.loads(entry['message'])['message']
return response
browser_log = browser.get_log('performance')
events = [process_browser_log_entry(entry) for entry in browser_log]
events = [event for event in events if 'Network.response' in event['method']]
print(events)
However I am getting the following error:
File "c:/Adhil/Python/MES_BOT/trackin-bot.py", line 43, in <module>
browser_log = browser.get_log('performance')
File "C:\Users\Adh\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 1262, in get_log
return self.execute(Command.GET_LOG, {'type': log_type})['value']
File "C:\Users\Adh\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\Adh\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 208, in check_response
raise exception_class(value)
selenium.common.exceptions.WebDriverException: Message:
Please advise, is there any package i need to install or configure?
errors = {}
for entry in context.browser.get_log('browser'):
errors.update(entry)
I learning python recent, but I have some error.
environment python3 , chrome , webdriver(chrome)
from selenium import webdriver
import time
import random
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
driver = webdriver.Chrome("./chromedriver.exe")
mobile_emulation = { "deviceName": 'Nexus 5' }
chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
driver = webdriver.Remote(command_executor='https:xxx.com',desired_capabilities = chrome_options.to_capabilities())
driver.get("https:/xxx.com")
num = random.randint(11111111 , 99999999)
red = driver.find_element_by_class_name("***")
red.click()
numBox = driver.find_element_by_name("***")
numBox.send_keys(int(num))
reader = driver.find_element_by_id("***")
reader.send_keys("***")
comment = driver.find_element_by_css_selector(" ***")
comment.click()
and result error is here
Traceback (most recent call last):
File "C:\python\pad\pad.py", line 16, in <module>
driver = webdriver.Remote(command_executor='https:xxx.com',desired_capabilities = chrome_options.to_capabilities())
File "C:\Users\***\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 156, in __init__
self.start_session(capabilities, browser_profile)
File "C:\Users\***\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 254, in start_session
self.session_id = response['sessionId']
TypeError: string indices must be integers
I think the error because number of this code includes　Decimal. but I cant find such number .
please give me advise
This error message...
Traceback (most recent call last):
File "C:\python\pad\pad.py", line 16, in <module>
driver = webdriver.Remote(command_executor='https:xxx.com',desired_capabilities = chrome_options.to_capabilities())
.
TypeError: string indices must be integers
...implies that there was a TypeError while invoking webdriver.Remote() method.
As per your code trials as you are using webdriver.Remote() with the argument command_executor perhaps you were trying to execute your tests in Selenium Grid Configuration.
As per the documentation documentation:
command_executor : remote_connection.RemoteConnection object used to execute commands.
Example:
command_executor='http://127.0.0.1:4444/wd/hub'
The complete implementation:
driver = webdriver.Remote(command_executor='http://127.0.0.1:4444/wd/hub', desired_capabilities = chrome_options.to_capabilities())
Note: Here we have considered that the Selenium Grid Hub and Selenium Grid Node are configured, up and running successfully with default configuration on the localhost.
Solution (Python 3.6)
Your effective code block will be:
from selenium import webdriver
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("start-maximized")
chrome_options.add_argument('disable-infobars')
#driver = webdriver.Remote(command_executor='https:xxx.com', desired_capabilities = chrome_options.to_capabilities())
driver = webdriver.Remote(command_executor='http://127.0.0.1:4444/wd/hub', desired_capabilities = chrome_options.to_capabilities())
driver.get('https://www.google.co.in')
print("Page Title is : %s" %driver.title)
driver.quit()
E
======================================================================
ERROR: test_LoginCorrect (__main__.LoginCorrect)
----------------------------------------------------------------------
Traceback (most recent call last):
File "demo.py", line 12, in test_LoginCorrect
driver= webdriver.Remote(desired_capabilities=DesiredCapabilities().FIREFOX,command_executor='http://0.0.0.0:4444')
File "/usr/local/lib/python3.5/dist-packages/selenium/webdriver/remote/webdriver.py", line 156, in __init__
self.start_session(capabilities, browser_profile)
File "/usr/local/lib/python3.5/dist-packages/selenium/webdriver/remote/webdriver.py", line 248, in
start_session
self.session_id = response['sessionId']
TypeError: string indices must be integers
----------------------------------------------------------------------
Ran 1 test in 20.878s
//Formatting is not properly done
FAILED (errors=1)
My test case file :
import unittest
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
class LoginCorrect(unittest.TestCase):
def setUp(self):
self.driver = webdriver.Remote(command_executor='http://0.0.0.0:4444/wd/hub', desired_capabilities=DesiredCapabilities.FIREFOX)
def test_LoginCorrect(self):
user ="shubh"
pwd= "sha123#56su"
driver= webdriver.Remote(desired_capabilities=DesiredCapabilities().FIREFOX,command_executor='http://0.0.0.0:4444')
driver.get("http://0.0.0.0:8000/login")
elem = driver.find_element_by_id("id_username")
elem.send_keys(user)
elem = driver.find_element_by_id("id_password")
elem.send_keys(pwd)
driver.find_element_by_class_name('btn-block').click()
def tearDown(self):
self.driver.quit()
if __name__ == '__main__':
unittest.main()
My machine has Linux 16.04LTS and selenium 3.3.0
Can somebody figure out this problem.
I have been trying to set up chromes headless browser on my mac but I am getting errors.
I tried following these tutorials for reference:
https://intoli.com/blog/running-selenium-with-headless-chrome/
https://duo.com/decipher/driving-headless-chrome-with-python
and using these stackoflow pages
How to get Chromedriver in headless mode to run headless? and selenium.common.exceptions.WebDriverException: Message: 'chromedriver' executable needs to be in PATH error with chrome
The headless browser worked on phantomjs but I know selenium no longer wants us to use this. This is what I am running right now: (almost exactly one the responses on stack overflow)
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get("http://www.duo.com")
print("Chrome Browser Initialized in Headless Mode")
this is my response:
Traceback (most recent call last):
File "headless_browser.py", line 47, in <module>
driver = webdriver.Chrome(chrome_options=chrome_options)
File "/Users/BCohen/anaconda3/lib/python3.6/site-packages/selenium/webdriver/chrome/webdriver.py", line 75, in __init__
desired_capabilities=desired_capabilities)
File "/Users/BCohen/anaconda3/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py", line 154, in __init__
self.start_session(desired_capabilities, browser_profile)
File "/Users/BCohen/anaconda3/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py", line 243, in start_session
response = self.execute(Command.NEW_SESSION, parameters)
File "/Users/BCohen/anaconda3/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py", line 312, in execute
self.error_handler.check_response(response)
File "/Users/BCohen/anaconda3/lib/python3.6/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: cannot find Chrome binary
(Driver info: chromedriver=2.36.540469 (1881fd7f8641508feb5166b7cae561d87723cfa8),platform=Mac OS X 10.13.2 x86_64)
The error does gives us the hint as follows :
selenium.common.exceptions.WebDriverException: Message: unknown error: cannot find Chrome binary
(Driver info: chromedriver=2.36.540469 (1881fd7f8641508feb5166b7cae561d87723cfa8),platform=Mac OS X 10.13.2 x86_64)
The error says that the ChromeDriver didn't find the Chrome binary in the expected location.
There can be two solutions as follows :
As per the Requirements the Chrome installation needs to be in a definite location.
As an alternative you can pass the absolute path of the Chrome binary through an instance of Options class as follows :
from selenium.webdriver.chrome.options import Options
options = Options()
options.binary_location = "/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome"
Finally while initiating the WebDriver and WebClient you need to send the argument Key executable_path along with the Value chrome_path.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
options = Options()
options.binary_location = "/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome"
options = Options()
options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("start-maximized")
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
chrome_path = "/home/ec2-user/chrome/chromedriver"
driver = webdriver.Chrome(executable_path=chrome_path, chrome_options=options)
driver.get("http://www.duo.com")
print("Chrome Browser Initialized in Headless Mode")
Alternative
Invoking Google Chrome Browser in Headless Mode programatically have become much easier with the availability of the method set_headless(headless=True) as follows :
Documentation :
set_headless(headless=True)
Sets the headless argument
Args:
headless: boolean value indicating to set the headless option
Sample Code :
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
options = Options()
options.set_headless(headless=True)
driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=r'C:\path\to\chromedriver.exe')
driver.get("http://google.com/")
print ("Headless Chrome Initialized")
driver.quit()
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
option = Options()
option.headless = True
driver = webdriver.Chrome(executable_path=r'C:\path\to\chromedriver.exe', options = option)
My code is:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
baseurl = "https://www.google.ca/?gfe_rd=cr&ei=J5ooWerXOsf_8AebtKKICw&gws_rd=ssl"
search = "panda"
xpaths = { 'searchbox' : ".//*[#id='lst-ib']",
'submit' : ".//*[#id='tsf']/div[2]/div[3]/center/input[1]",
'img' : ".//*[#id='gbw']/div/div/div[1]/div[2]/a"
}
driver = webdriver.Firefox()
driver.get(baseurl)
driver.find_element_by_xpath(xpaths['searchbox']).clear()
driver.find_element_by_xpath(xpaths['searchbox']).send_keys(search)
driver.find_element_by_xpath(xpaths['submit']).click()
#driver.find_element_by_xpath(xpaths['img']).click()
Firefox opens, but nothing at all happens, and written in the terminal is the following:
Traceback (most recent call last):
File "sg1.py", line 21, in <module>
driver.find_element_by_xpath(xpaths['searchbox']).send_keys(search)
File "/usr/local/lib/python3.5/dist-packages/selenium/webdriver/remote/webelement.py", line 347, in send_keys
self._execute(Command.SEND_KEYS_TO_ELEMENT, {'value': keys_to_typing(value)})
File "/usr/local/lib/python3.5/dist-packages/selenium/webdriver/remote/webelement.py", line 494, in _execute
return self._parent.execute(command, params)
File "/usr/local/lib/python3.5/dist-packages/selenium/webdriver/remote/webdriver.py", line 236, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python3.5/dist-packages/selenium/webdriver/remote/errorhandler.py", line 192, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: Expected [object Undefined] undefined to be a string
Things to note:
Firefox is up to date.
When Firefox opens it opens a plain version and not the version that usually opens with add-ons such as adblocker, firebug, etc.
When i ran just the click on 'img' bit that is commented out it did what it was supposed too.
This issue is common with geckodriver v.015; In order to resolve this update your geckodriver version to 0.16 also selenium to 3.4.0.