clicking button on a webpage multiple times in excel vba - excel

I'm trying to click a button multiple times to get my page loaded fully. Webpage contains a button SHOW MORE instead of next page so in html coding behind the button remains same.
I am using the following excel-vba code to hit that button. It actually did click that button, but instead of showing the next results it shows the same result over and over again.
Could you please kindly show me how to make it right. Thanks in advance!
**' VARIABLE DECLARATION
Dim IE As Object
Dim county As String
Dim htmlDoc As Object
' CREATING OBJECT
Set IE = CreateObject("internetexplorer.application")
' WEBPAGE NAVIGATION
With IE
.navigate ("http://www.physiofirst.org.uk/find-physio/search-physio.html")
.Visible = True
End With
' WAITING FOR WEBPAGE TO LOAD
Do
DoEvents
Loop Until IE.readystate = 4
' SEARCHING ALL THE THE INDIVIDUAL STATE PHYSICIANS
Set htmlDoc = IE.document
Set searchbarvalue = htmlDoc.getelementsbyclassname("form-control mod-text-display")
i = 0
For Each classSearch In searchbarvalue
searchbarvalue(i).Value = "BRISTOL"
Next classSearch
While IE.busy
DoEvents
Wend
Set buttonclick = htmlDoc.getelementsbyclassname("btn btn-search")
i = 0
For Each buttonsearch In buttonclick
buttonclick(i).Click
Next buttonsearch
While IE.busy
DoEvents
Wend
Do
htmlDoc.getelementbyid("load-more-practice").Click
While IE.busy
DoEvents
Wend
Loop Until htmlDoc.getelementbyid("load-more-practice").Click = True
End Sub**

Try this code that Do Until data-page is = "1", because it starts on 2 and go to 1 when is completely loaded, with all Show More clicked.
Public Declare Sub Sleep Lib "kernel32" (ByVal dwMilliseconds As Long)
Sub test()
Dim IE As Object
Dim county As String
Dim htmlDoc As Object
Dim sURL As String
' CREATING OBJECT
Set IE = CreateObject("internetexplorer.application")
sURL = "http://www.physiofirst.org.uk/find-physio/search-physio.html"
' WEBPAGE NAVIGATION
With IE
.navigate (sURL)
.Visible = True
End With
WaitIE IE, 2000
' SEARCHING ALL THE THE INDIVIDUAL STATE PHYSICIANS
Set htmlDoc = IE.document
Set searchbarvalue = htmlDoc.getElementsByClassName("form-control mod-text-display")
i = 0
For Each classSearch In searchbarvalue
searchbarvalue(i).Value = "BRISTOL"
Next classSearch
WaitIE IE, 1000
Set buttonclick = htmlDoc.getElementsByClassName("btn btn-search")
buttonclick(0).Click
WaitIE IE, 1000
Set ShowMore = htmlDoc.getElementById("load-more-practice")
Do
ShowMore.Click
WaitIE IE, 2000
Loop Until ShowMore.getAttribute("data-page") = 1
'IE.Quit
'Set IE = Nothing
End Sub
Sub WaitIE(IE As Object, Optional time As Long = 250)
'Code from: https://stackoverflow.com/questions/33808000/run-time-error-91-object-variable-or-with-block-variable-not-set
Dim i As Long
Do
Sleep time
Debug.Print CStr(i) & vbTab & "Ready: " & CStr(IE.readyState = 4) & _
vbCrLf & vbTab & "Busy: " & CStr(IE.Busy)
i = i + 1
Loop Until IE.readyState = 4 Or Not IE.Busy
End Sub

Related

Error 91 when Internet Explorer Save prompt is shown

My goal is to open a site, input into the forms, save a report through a button, and then loop around and continue for the next entry. I am using the Microsoft WinHTTP, HTML object library, and Internet Controls.
This works so long as I have no "Open/Save As" prompt in Internet Explorer. It successfully loops through the results and clicks the report button, then restarts. However, once the prompt appears, it causes a break in line " svalue1.Value = ws.Cells(Lastrow, 1).Value " reporting Error 91, object not found, that I can't for the life of me understand. Why would the save prompt break a VBA script?
Commented out is some of the fixes I've tried, but nothing seems to resolve this issue. Am I missing something obvious?
Thank you for taking the time to help a newbie out with this! :)
Enum READYSTATE
READYSTATE_UNINITIALIZED = 0
READYSTATE_LOADING = 1
READYSTATE_LOADED = 2
READYSTATE_INTERACTIVE = 3
READYSTATE_COMPLETE = 4
End Enum
Sub Test()
Dim ws As Worksheet: Set ws = Sheets("Sheet1") 'set the Sheet you are using here
Dim ie As New InternetExplorerMedium
'Set ie = New InternetExplorerMedium
Dim HTML As HTMLDocument
Dim i As Long
Lastrow = ws.Range("A1").End(xlDown).Row
'get the last row with a value in column A
strURL = "www.coollinkExample.com" 'set your initial URL here
ie.Visible = True
ie.navigate strURL
Do While (ie.Busy Or ie.READYSTATE <> READYSTATE.READYSTATE_COMPLETE)
DoEvents
Loop
Set HTML = ie.document
'-----------------INIT-----------------
'For i = 1 To Lastrow Step 1 'loop from row 1 to Last
Do While i < Lastrow 'loop from row 1 to Last
'Do While (ie.Busy Or ie.READYSTATE <> READYSTATE.READYSTATE_COMPLETE)
'DoEvents
'Loop BUGS WHEN SAVE PROMPT IS UP?
'-----------------BODY-----------------
'Do your data scraping here, then below go back to your initial URL to repeat the process
delay 5
Set svalue1 = HTML.getElementById("userNameText")
ie.document.getElementById("userNameText").Focus
svalue1.Value = ws.Cells(Lastrow, 1).Value 'enter the value from current cell
delay 2
ie.document.getElementById("btnSearchUser").Focus
HTML.getElementsByName("btnSearchUser").Item.Click
delay 3
ie.document.getElementById("rptUsers_ctl00_ddlUserOptions").Focus
HTML.getElementsByName("rptUsers_ctl00_ddlUserOptions").Item.Click
delay 3
ie.document.getElementById("rptUsers_ctl00_ddlUserOptions_lnkTranscript").Focus
HTML.getElementsByName("rptUsers_ctl00_ddlUserOptions_lnkTranscript").Item.Click
delay 3
ie.document.getElementById("__ta").Focus
HTML.getElementsByName("__ta").Item.Click
delay 2
ie.document.getElementById("__tj").Focus
HTML.getElementsByName("__tj").Item.Click
delay 4
With ie.document.getElementById("ctl00_ContentPlaceHolder1_btnExport")
.Focus
.Click
End With
delay 5
Application.SendKeys "%{S}"
delay 3
'-----------------END-----------------
ie.navigate strURL
delay 5
'Exit For
'Next i
i = i + 1
Loop
End Sub
'// This function below works fine.
Private Sub delay(seconds As Long)
Dim endTime As Date
endTime = DateAdd("s", seconds, Now())
Do While Now() < endTime
DoEvents
Loop
End Sub
You have ie.navigate strURL at the end of the loop but are not waiting for the page to load fully.
Option Explicit
Sub Test()
Dim ws As Worksheet
Dim ie As InternetExplorerMedium
Dim doc As HTMLDocument, strURL As String
Dim i As Long, lastrow As Long
strURL = "www.coollinkExample.com" 'set your initial URL here
Set ie = New InternetExplorerMedium
ie.Visible = True
Set ws = Sheets("Sheet1")
With ws
lastrow = .Cells(.Rows.Count, "A").End(xlUp).Row
For i = 1 To lastrow
' navigate to site
ie.navigate strURL
Do While ie.Busy Or ie.READYSTATE <> 4 'READYSTATE.READYSTATE_COMPLETE)
DoEvents
Loop
Set doc = ie.document
delay 5
doc.getElementById("userNameText").innerText = Trim(.Cells(i, "A"))
delay 3
doc.getElementById("btnSearchUser").Click
delay 3
doc.getElementById("rptUsers_ctl00_ddlUserOptions").Click
delay 3
doc.getElementById("rptUsers_ctl00_ddlUserOptions_lnkTranscript").Click
delay 3
doc.getElementById("__ta").Click
delay 3
doc.getElementById("__tj").Click
delay 3
doc.getElementById("ctl00_ContentPlaceHolder1_btnExport").Click
delay 5
Application.SendKeys "%(S)"
delay 3
Next i
End With
ie.Quit
MsgBox "Done", vbInformation
End Sub

Retrieving a URL from Internet Explorer with VBA

I have written some VBA code in Excel to retrieve the latitude and longitude from a Google Maps URL and paste it into a cell in my worksheet. My problem is in retrieving the URL from internet explorer. Below I have two examples of my code, one macro returns an about:blank as though the object doesn't have the LocationURL property, and the other example seems like it is saving all of my previous searches, so it cycles through all of my previous searches and pastes the very first searches' URL. Example 2 uses a shell suggestion that I found online to reassign the properties to the oIE object. I can get both to slightly work, but neither will do exactly what I need from the macro.
Cell(8,8) is a hyperlink to google maps where I'm searching an address, and Cell(8,9) is where I want to paste the URL after google maps has redirected and has the latitude and longitude in the URL.
Example 1:
Sub CommandButton1_Click()
Dim ie As Object
Dim Doc As HTMLDocument
Set ie = CreateObject("InternetExplorer.Application")
ie.Visible = True
ie.Navigate "http://www.google.com/maps?q=" & Range("I7").Value
Do
DoEvents
Loop Until ie.ReadyState = 4
Set Doc = ie.Document
Cells(8, 9).Value = ie.LocationName
End Sub
Example 2:
Sub Macro()
Dim oIE, oShell, objShellWindows, strPath, X
strPath = Cells(8, 8)
Set oIE = CreateObject("InternetExplorer.Application")
'This is to resolve oIE.navigate "about:blank" issue
oIE.Top = 0
oIE.Left = 0
oIE.Width = 500
oIE.Height = 500
oIE.Navigate strPath
Do While oIE.Busy And oIE.ReadyState < 2
DoEvents
Loop
'Reassigning oIE.LocationName & vbCrLf & oIE.LocationURL values after redirect in IE
Set oShell = CreateObject("WScript.Shell")
Set objShellWindows = CreateObject("Shell.Application").Windows
For X = objShellWindows.Count - 1 To 0 Step -1
Set oIE = objShellWindows.Item(X)
If Not oIE Is Nothing Then
If StrComp(oIE.LocationURL, strPath, 1) = 0 Then
Do While oIE.Busy And oIE.ReadyState < 2
DoEvents
Loop
oIE.Visible = 2
Exit For
End If
End If
Cells(8, 9).Value = oIE.LocationURL
Set oIE = Nothing
Next
Set objShellWindows = Nothing
Set oIE = Nothing
End Sub
Thanks,
Andrew
Is this as simple as looping until the document.URL changes? In my timed loop I wait for the string safe=vss in the original page load to disappear.
Option Explicit
Public Sub GetNewURL()
Dim IE As New InternetExplorer, newURL As String, t As Date
Const MAX_WAIT_SEC As Long = 5
With IE
.Visible = True
.navigate2 "http://www.google.com/maps?q=" & "glasgow" '<==Range("I7").Value
While .Busy Or .readyState < 4: DoEvents: Wend
t = Timer
Do
DoEvents
newURL = .document.URL
If Timer - t > MAX_WAIT_SEC Then Exit Do
Loop While InStr(newURL, "safe=vss") > 0
Debug.Print newURL
End With
End Sub

VBA: copy data from website into excel

I have a VBA code that selects info from drop-down menus on a government website and then submits the query. The requested data then opens up in another IE page. I am trying to copy this data into excel; however, I am unable to do so.
My code currently copies the text on the first IE page that contains the drop-down menus. The government website is: http://www.osfi-bsif.gc.ca/Eng/wt-ow/Pages/FINDAT.aspx
I have look all over the internet for a solution but nothing seems to work...
Here is my code:
Sub GetOsfiFinancialData()
Dim UrlAddress As String
UrlAddress = "http://ws1.osfi-bsif.gc.ca/WebApps/FINDAT/DTIBanks.aspx?T=0&LANG=E"
Dim ie As Object
Set ie = CreateObject("internetexplorer.application")
With ie
.Silent = True
.Visible = False
.navigate UrlAddress
End With
Do Until Not ie.Busy And ie.readyState = 4
DoEvents
Loop
Application.Wait (Now() + TimeValue("00:00:05"))
'Select Bank
ie.document.getElementById("DTIWebPartManager_gwpDTIBankControl1_DTIBankControl1_institutionTypeCriteria_institutionsDropDownList").Value = Z005
'open window with financial data
Dim objButton
Set objButton = ie.document.getElementById("DTIWebPartManager_gwpDTIBankControl1_DTIBankControl1_submitButton")
objButton.Focus
objButton.Click
'select new pop-up window
marker = 0
Set objshell = CreateObject("Shell.Application")
IE_count = objshell.Windows.Count
For x = 0 To (IE_count - 1)
On Error Resume Next ' sometimes more web pages are counted than are open
my_title = objshell.Windows(x).document.Title
If my_title Like "Consolidated Monthly Balance Sheet" & "*" Then 'compare to find if the desired web page is already open
Set ie = objshell.Windows(x)
marker = 1
Exit For
Else
End If
Next
Do Until Not ie.Busy And ie.readyState = 4
DoEvents
Loop
Application.Wait (Now() + TimeValue("00:00:05"))
Dim doc As MSHTML.HTMLDocument
Dim tables As MSHTML.IHTMLElementCollection
Dim table As MSHTML.HTMLTable
Dim clipboard As MSForms.DataObject
Set doc = ie.document
Set tables = doc.getElementsByTagName("body")
Set table = tables(0)
Set clipboard = New MSForms.DataObject
'paste in sheets
Dim test
Set test = ActiveWorkbook.Sheets("Test")
clipboard.SetText table.outerHTML
clipboard.PutInClipboard
test.Range("A1").PasteSpecial xlPasteAll
clipboard.Clear
MsgBox ("Task Completed")
End Sub
Your help is greatly appreciated!
You were using the current test with document.Title. I found that For Each of all windows looking for the full title worked in combination with copy pasting the pop-up window outerHTML. No additional wait time was required.
Inside the For Each Loop, after you reset the IE instance to the new window, you can obtain the new URL with ie.document.url. As you already have the data loaded you might as well just copy paste it straight away in my opinion.
Code:
Option Explicit
Public Sub GetOsfiFinancialData()
Dim UrlAddress As String, objButton, ie As Object
UrlAddress = "http://ws1.osfi-bsif.gc.ca/WebApps/FINDAT/DTIBanks.aspx?T=0&LANG=E"
Set ie = CreateObject("internetexplorer.application")
With ie
.Silent = True
.Visible = False
.navigate UrlAddress
While .Busy Or .readyState < 4: DoEvents: Wend
.document.getElementById("DTIWebPartManager_gwpDTIBankControl1_DTIBankControl1_institutionTypeCriteria_institutionsDropDownList").Value = "Z005"
Set objButton = .document.getElementById("DTIWebPartManager_gwpDTIBankControl1_DTIBankControl1_submitButton")
objButton.Focus
objButton.Click
Dim objShellWindows As New SHDocVw.ShellWindows, currentWindow As IWebBrowser2
For Each currentWindow In objShellWindows
If currentWindow.document.Title = "Consolidated Monthly Balance Sheet - Banks, Trust and Loan" Then
Set ie = currentWindow
Exit For
End If
Next
Dim clipboard As Object
Set clipboard = GetObject("New:{1C3B4210-F441-11CE-B9EA-00AA006B1A69}")
clipboard.SetText ie.document.body.outerHTML
clipboard.PutInClipboard
ThisWorkbook.Worksheets("Sheet1").Cells(1, 1).PasteSpecial
.Quit
End With
End Sub
References (VBE > Tools > References):
Microsoft Internet Controls
I don't have time to get into all the stuff about controlling one browser from another, but I think you can figure that part out, especially since you made some great progress on this already. Get URL#2 from URL#1, like you are doing, but with some better data controls around it, and then do this...
Option Explicit
Sub Web_Table_Option_One()
Dim xml As Object
Dim html As Object
Dim objTable As Object
Dim result As String
Dim lRow As Long
Dim lngTable As Long
Dim lngRow As Long
Dim lngCol As Long
Dim ActRw As Long
Set xml = CreateObject("MSXML2.XMLHTTP.6.0")
With xml
.Open "GET", "http://ws1.osfi-bsif.gc.ca/WebApps/Temp/2f40b7ef-d024-4eca-a8a3-fb82153efafaFinancialData.aspx", False
.send
End With
result = xml.responseText
Set html = CreateObject("htmlfile")
html.body.innerHTML = result
Set objTable = html.getElementsByTagName("Table")
For lngTable = 0 To objTable.Length - 1
For lngRow = 0 To objTable(lngTable).Rows.Length - 1
For lngCol = 0 To objTable(lngTable).Rows(lngRow).Cells.Length - 1
ThisWorkbook.Sheets("Sheet1").Cells(ActRw + lngRow + 1, lngCol + 1) = objTable(lngTable).Rows(lngRow).Cells(lngCol).innerText
Next lngCol
Next lngRow
ActRw = ActRw + objTable(lngTable).Rows.Length + 1
Next lngTable
End Sub

scrape with excel vba changing input data before scraping

In am trying to change the data on the website for an input field and have that information refreshed on the page. I have updated the input field, but I am not sure how to refresh the page so that the inner table uses the new data from the input field
Below is my code:
Dim IE As InternetExplorer
Dim htmldoc As HTMLDocument
Dim ieURL As String
Dim sPicker As String
ieURL = "https://www.investing.com/commodities/crude-oil-historical-data"
sPicker = "10/01/2017 - 12/31/2017"
'Open InternetExplorer
Set IE = New InternetExplorer
IE.Visible = True
IE.Navigate ieURL
Set htmldoc = IE.document 'Document webpage
' wait until the page loads before doing anything
Do Until (IE.readyState = 4 And Not IE.Busy)
DoEvents ' DoEvents releases the macro and lets excel do other thing while it waits
Loop
Dim drp As HTMLFormElement
Set drp = htmldoc.getElementById("widgetFieldDateRange")
drp.innerText = sPicker 'Set the new timeframe for scraping
Dim inpt As HTMLInputElement
Set inpt = htmldoc.getElementById("picker")
inpt.Value = sPicker 'Set the new timeframe for scraping
' wait until the page loads before doing anything
Do Until (IE.readyState = 4 And Not IE.Busy)
DoEvents ' DoEvents releases the macro and lets excel do other thing while it waits
Loop
Thanks for your help
Try the below script. It should solve the issue.
Sub Web_Data()
Dim IE As New InternetExplorer, html As New HTMLDocument
Dim post As Object, elem As Object, t_data As Object
Dim trow As Object, tcel As Object
With IE
.Visible = True
.navigate "https://www.investing.com/commodities/crude-oil-historical-data"
While .readyState < 4: DoEvents: Wend
Set html = .document
End With
Application.Wait Now + TimeValue("00:00:05")
html.getElementById("widgetFieldDateRange").Click
Application.Wait Now + TimeValue("00:00:03")
Set post = html.getElementById("startDate")
post.innerText = ""
post.Focus
Application.SendKeys "10/01/2017"
Application.Wait Now + TimeValue("00:00:03")
Set elem = html.getElementById("endDate")
elem.innerText = ""
elem.Focus
Application.SendKeys "12/31/2017"
Application.Wait Now + TimeValue("00:00:03")
html.getElementById("applyBtn").Click
Application.Wait Now + TimeValue("00:00:03")
Set t_data = html.getElementById("curr_table")
For Each trow In t_data.Rows
For Each tcel In trow.Cells
y = y + 1: Cells(x + 1, y) = tcel.innerText
Next tcel
y = 0
x = x + 1
Next trow
End Sub
Reference to add to the library:
1. Microsoft Internet Controls
2. Microsoft HTML Object Library

Navigating through websites

My code is able to interact with the first webpage it brings up, but then as soon as i navigate away from it, my code gives me a '424' object required error. Here is an example.
EX: go to yahoo.com - search for 'Earth' - click on the Yahoo logo to return to the yahoo homepage
Sub InternetPractice()
Dim ie As InternetExplorer
Set ie = New InternetExplorer
ie.Visible = True
ie.navigate "https://www.yahoo.com"
Do While ie.Busy = True Or ie.readyState <> 4: DoEvents: Loop
ie.document.getElementById("uh-search-box").Value = "Earth"
ie.document.getElementById("uh-search-button").Click
Do While ie.Busy = True Or ie.readyState <> 4: DoEvents: Loop
ie.document.getElementById("logo").Click
End Sub
Using VBA with the internet is new to me and any help is appreciated!
EDIT:
I think you are not giving enough time for ie to load... It is working for me.
So you can try this code that gives more time and try to load 3 times before aborting.
Option Explicit
Public Declare Sub Sleep Lib "kernel32" (ByVal dwMilliseconds As Long)
Sub InternetPractice2()
Dim IE As InternetExplorer
Dim HTMLDoc As HTMLDocument
Dim logo As Object
Dim n As Long
Set IE = New InternetExplorer 'Set IE = CreateObject("InternetExplorer.Application")
With IE
.Silent = True
.Visible = True
.Navigate "https://www.yahoo.com"
End With
WaitIE IE, 5000
Set HTMLDoc = IE.Document
HTMLDoc.getElementById("uh-search-box").Value = "Earth"
HTMLDoc.getElementById("uh-search-button").Click
load:
WaitIE IE, 10000
Set logo = HTMLDoc.getElementById("logo")
If Not logo Is Nothing Then
logo.Click
ElseIf logo Is Nothing And n < 4 Then
n = n + 1
GoTo load
End If
WaitIE IE, 5000
'Set ie to Nothing
IE.Quit
Set IE = Nothing
End Sub
Sub WaitIE(IE As Object, Optional time As Long = 250)
'Code from: https://stackoverflow.com/questions/33808000/run-time-error-91-object-variable-or-with-block-variable-not-set
Dim i As Long
Do
Sleep time
Debug.Print CStr(i) & vbTab & "Ready: " & CStr(IE.READYSTATE = 4) & _
vbCrLf & vbTab & "Busy: " & CStr(IE.Busy)
i = i + 1
Loop Until IE.READYSTATE = 4 Or Not IE.Busy
End Sub
Original Answer
Try closing ie in the end of the code and add some delay, because if you step the code with F8, you will see that it works:
Sub InternetPractice2()
Dim ie As InternetExplorer
Set ie = New InternetExplorer
ie.Visible = True
ie.navigate "https://www.yahoo.com"
Do While ie.Busy = True Or ie.READYSTATE <> 4: DoEvents: Loop
ie.Document.getElementById("uh-search-box").Value = "Earth"
ie.Document.getElementById("uh-search-button").Click
'Add delay After .Click
Do While ie.Busy = True Or ie.READYSTATE <> 4: DoEvents: Loop
Application.wait Now+Timevalue("00:00:05")
ie.Document.getElementById("logo").Click
'Set ie to Nothing
Set ie = Nothing
End Sub
Also close all iexplorer.exe on Windows Task Manager.

Resources