Cycling Through List of URLs - excel

This code works great when cycling through URLs such as google, yahoo, ect
But I am really trying to cycle through web pages as shown here.
\\FMC9050101\Proj\6513_OAK3\Jobads\slide1.htm
\\FMC9050101\Proj\6513_OAK3\Jobads\slide2.htm
\\FMC9050101\Proj\6513_OAK3\Jobads\slide3.htm
The 1st webpage opens up perfect, but I get and Automation Error, "The object invoked has disconnected from its clients" on this line, as the next page is cycled in... the idea is to replace the existing page without opening a new tab.
While .Busy Or .ReadyState <> 4: DoEvents: Wend
**** Code ***
Dim wsSheet As Worksheet, Rows As Long, links As Variant, IE As Object, link As Variant
Set wb = ThisWorkbook
Set wsSheet = wb.Sheets("Sheet1")
Set IE = New InternetExplorer
Rows = wsSheet.Cells(wsSheet.Rows.Count, "A").End(xlUp).Row
links = wsSheet.Range("A1:A" & Rows)
With IE
.Visible = True
For Each link In links
.navigate (link)
While .Busy Or .ReadyState <> 4: DoEvents: Wend
MsgBox .Document.body.innerText
Next link
End With

Okay, changed the strategy where I am reading the list of URLs from a server verses an excel sheet which is where I was going with this after the other issues were sorted out. Using an Admin account, this version works perfectly.
Sub Run_SlideShow()
'
Dim x As Integer
Dim wsSheet As Worksheet, Rows As Long, links As Variant, IE As Object, link
As Variant
Dim FilePath As String, Filter As String, F As Variant, I As Integer
'
Application.Calculation = xlCalculationManual
Application.ScreenUpdating = False
Application.EnableEvents = True
'
Set wb = ThisWorkbook
Set wsSheet = wb.Sheets("Sheet2")
Filter = "*.htm"
Set IE = CreateObject("Internetexplorer.Application")
IE.Visible = False
FilePath = "\\FMC9050101\PROJ\6513_OAK3\Jobads"
For x = 1 To 9999 ' run for 30 hours, use scheduled task to kill excel and
restart every 24 hours
'
ArrFile = GetFileList(FilePath + "\" + Filter)
Select Case IsArray(ArrFile)
Case True
For I = LBound(ArrFile) To UBound(ArrFile)
F = ArrFile(I)
link = (FilePath & "\" & F)
IE.Navigate link
IE.Visible = True
'Application.StatusBar = "Loading " & link
Do While IE.Busy
Application.Wait DateAdd("s", 2, Now) ' set slide time here
Loop
Next
Case False 'no files found
MsgBox "No matching files"
End Select
Next x
'
Application.Calculation = xlCalculationAutomatic
Application.ScreenUpdating = True
Application.EnableEvents = True
'
Set IE = Nothing
Application.StatusBar = ""
'
End Sub

Related

Scrape CDC vaccination data using VBA

I am trying to scrape the vaccination data from the below CDC website:
https://covid.cdc.gov/covid-data-tracker/#vaccinations
I have tried querySelectorAll but no luck. Can anyone help take a look? Much appreciated!
Sub useClassnames()
Dim appIE As Object
Set appIE = CreateObject("internetexplorer.application")
With appIE
.navigate "https://covid.cdc.gov/covid-data-tracker/#vaccinations"
.Visible = False
End With
Do While appIE.Busy
DoEvents
Loop
Set allRowOfData = appIE.document.getElementById("maincontent")
Debug.Print allRowOfData.innerHTML
'Set element = appIE.document.querySelectorAll(".container mt-5")
'For Each elements In element
' Debug.Print elements
'Next elements
'For Each element In allRowOfData
'Debug.Print element
'Next element
End Sub
Here you have, just change your worksheet name or number :)
Option Explicit
Const updatedCol = 1
Const dosesDistributedColVal = 2
Const peopleInicVaccColVal = 3
Sub useClassnames()
'declare worksheet variable and set headers
Dim targetWsh As Worksheet: Set targetWsh = ThisWorkbook.Sheets(1)
targetWsh.Cells(1, 1).Value = "Last Update"
targetWsh.Cells(1, 2).Value = "Doses Distributed"
targetWsh.Cells(1, 3).Value = "People Initiating Vaccination"
Dim lstRegisterRow As Long: lstRegisterRow = targetWsh.Range("A" & targetWsh.Rows.Count).End(xlUp).Row + 1
'open IE and navigate to site
Dim appIE As InternetExplorer: Set appIE = New InternetExplorer
appIE.navigate "https://covid.cdc.gov/covid-data-tracker/#vaccinations"
appIE.Visible = False
While appIE.Busy = True Or appIE.readyState < 4: DoEvents: Wend
Dim oHtmlDoc As HTMLDocument: Set oHtmlDoc = appIE.document
Dim oHtmlElementColl As IHTMLElementCollection
'Get and write last update date
Application.Wait (Now + TimeValue("0:00:02")) 'wait 2 secs to avoid error, if recieve error, add seconds as needed
Set oHtmlElementColl = oHtmlDoc.getElementsByTagName("small")
targetWsh.Cells(lstRegisterRow, updatedCol) = oHtmlElementColl(0).innerHTML
'Get and write Doses Distributed and People Initiating Vaccination
Set oHtmlElementColl = oHtmlDoc.GetElementsByClassName("card-number")
targetWsh.Cells(lstRegisterRow, dosesDistributedColVal) = oHtmlElementColl(0).innerText
targetWsh.Cells(lstRegisterRow, peopleInicVaccColVal) = oHtmlElementColl(1).innerText
appIE.Quit
End Sub

Retrieve the URL from the Internet Explorer

i am struggling on the following: I try to open an URL(Link), become redirected to a new URL and retrieve this new URL into a Cell in my excel worksheet.
I have written the following code but it is not retrieving the new URL and is not quitting the Internet Explorer at the end:
Sub Get_URL()
Dim ISIN As String
Dim Link As String
Dim IE As Object
ISIN = Range("A1").Value
Link = "https://www.finanzen.net/suchergebnis.asp?_search=" & ISIN
Set IE = CreateObject("InternetExplorer.Application")
With IE
.Visible = True
.Navigate Link
End With
Range("A2") = IE.LocationURL
Set IE = Nothing
IE.Quit
End Sub
The ISIN in A1 is KYG875721634.
I would be very glad if some of you guys could find the problem. Thank you very much!
Greetings, Robin
This works. Read the comments please:
Sub Get_URL()
Dim ISIN As String
Dim Link As String
Dim IE As Object
'ISIN = Range("A1").Value
ISIN = "KYG875721634"
Link = "https://www.finanzen.net/suchergebnis.asp?_search=" & ISIN
Set IE = CreateObject("InternetExplorer.Application")
With IE
.Visible = True
.Navigate Link
End With
'You must wait to load the page
Do While IE.readystate <> 4: DoEvents: Loop
'Range("A2") = IE.LocationURL
MsgBox IE.LocationURL
'First quit the IE
'because this step needs the reference ;-)
IE.Quit
Set IE = Nothing
End Sub
There're two issues in your code:
You need to add a wait to load page.
Do Until IE.readyState = 4
DoEvents
Loop
First quit IE. If you first set IE to Nothing then there will be no reference to IE when quit.
IE.Quit
Set IE = Nothing
The final code is like this which can work well:
Sub Get_URL()
Dim ISIN As String
Dim Link As String
Dim IE As Object
ISIN = Range("A1").Value
Link = "https://www.finanzen.net/suchergebnis.asp?_search=" & ISIN
Set IE = CreateObject("InternetExplorer.Application")
IE.Visible = True
IE.Navigate Link
Do Until IE.readyState = 4
DoEvents
Loop
Range("A2") = IE.LocationURL
IE.Quit
Set IE = Nothing
End Sub
Result:

Retrieving a URL from Internet Explorer with VBA

I have written some VBA code in Excel to retrieve the latitude and longitude from a Google Maps URL and paste it into a cell in my worksheet. My problem is in retrieving the URL from internet explorer. Below I have two examples of my code, one macro returns an about:blank as though the object doesn't have the LocationURL property, and the other example seems like it is saving all of my previous searches, so it cycles through all of my previous searches and pastes the very first searches' URL. Example 2 uses a shell suggestion that I found online to reassign the properties to the oIE object. I can get both to slightly work, but neither will do exactly what I need from the macro.
Cell(8,8) is a hyperlink to google maps where I'm searching an address, and Cell(8,9) is where I want to paste the URL after google maps has redirected and has the latitude and longitude in the URL.
Example 1:
Sub CommandButton1_Click()
Dim ie As Object
Dim Doc As HTMLDocument
Set ie = CreateObject("InternetExplorer.Application")
ie.Visible = True
ie.Navigate "http://www.google.com/maps?q=" & Range("I7").Value
Do
DoEvents
Loop Until ie.ReadyState = 4
Set Doc = ie.Document
Cells(8, 9).Value = ie.LocationName
End Sub
Example 2:
Sub Macro()
Dim oIE, oShell, objShellWindows, strPath, X
strPath = Cells(8, 8)
Set oIE = CreateObject("InternetExplorer.Application")
'This is to resolve oIE.navigate "about:blank" issue
oIE.Top = 0
oIE.Left = 0
oIE.Width = 500
oIE.Height = 500
oIE.Navigate strPath
Do While oIE.Busy And oIE.ReadyState < 2
DoEvents
Loop
'Reassigning oIE.LocationName & vbCrLf & oIE.LocationURL values after redirect in IE
Set oShell = CreateObject("WScript.Shell")
Set objShellWindows = CreateObject("Shell.Application").Windows
For X = objShellWindows.Count - 1 To 0 Step -1
Set oIE = objShellWindows.Item(X)
If Not oIE Is Nothing Then
If StrComp(oIE.LocationURL, strPath, 1) = 0 Then
Do While oIE.Busy And oIE.ReadyState < 2
DoEvents
Loop
oIE.Visible = 2
Exit For
End If
End If
Cells(8, 9).Value = oIE.LocationURL
Set oIE = Nothing
Next
Set objShellWindows = Nothing
Set oIE = Nothing
End Sub
Thanks,
Andrew
Is this as simple as looping until the document.URL changes? In my timed loop I wait for the string safe=vss in the original page load to disappear.
Option Explicit
Public Sub GetNewURL()
Dim IE As New InternetExplorer, newURL As String, t As Date
Const MAX_WAIT_SEC As Long = 5
With IE
.Visible = True
.navigate2 "http://www.google.com/maps?q=" & "glasgow" '<==Range("I7").Value
While .Busy Or .readyState < 4: DoEvents: Wend
t = Timer
Do
DoEvents
newURL = .document.URL
If Timer - t > MAX_WAIT_SEC Then Exit Do
Loop While InStr(newURL, "safe=vss") > 0
Debug.Print newURL
End With
End Sub

how to select a specific dropdown element in webpage by excel vba

Currently, I am working on a webpage to extract some data related to my work. Here I am facing a problem while selecting the drop-down menu in the production list. I want to select the "All" option in the drop-down list. By default, it takes only 25 values in the production table.
Here is my code
Sub Production_table()
Dim objIE As InternetExplorer
Dim ele As Object
Dim ele2 As Object
Dim ele3 As Object
Dim s As String
Dim i As Integer
i = 2
Application.ScreenUpdating = False
Application.DisplayAlerts = False
Do While Range("A" & i).Value <> ""
Set objIE = New InternetExplorer
objIE.Visible = True
source = "https://secure.conservation.ca.gov/WellSearch/Details?api=01120716&District=&County=&Field=&Operator=&Lease=&APINum=&address=&ActiveWell=false&ActiveOp=false&Location=&sec=&twn=&rge=&bm=&PgStart=0&PgLength=10&SortCol=6&SortDir=asc&Command=Search
objIE.navigate source"
Do While objIE.Busy = True Or objIE.readyState <> 4: DoEvents: Loop
Sheets.Add
ActiveSheet.Name = "Data"
On Error Resume Next
Set ele2 = objIE.document.getElementById("productionTable_length"). _
getElementsByTagName("label")
For Each ele3 In ele2
If ele3.Value = -1 Then
ele3.Focus
ele3.Selected = True
ele3.Click
Exit For
End If
Next
webpage link-- https://secure.conservation.ca.gov/WellSearch/Details?api=01120716&District=&County=&Field=&Operator=&Lease=&APINum=&address=&ActiveWell=false&ActiveOp=false&Location=&sec=&twn=&rge=&bm=&PgStart=0&PgLength=10&SortCol=6&SortDir=asc&Command=Search
Please help me on the same.
I always recommend using Chrome Browser for such cases as it let's you inspect the element in a userfriendly way.
Try this
Sub Sample()
Dim objIE As Object, ele As Object, opt As Object
Set objIE = CreateObject("InternetExplorer.Application")
objIE.Visible = True
Source = "https://secure.conservation.ca.gov/WellSearch/Details?api=01120716&District=&County=&Field=&Operator=&Lease=&APINum=&address=&ActiveWell=false&ActiveOp=false&Location=&sec=&twn=&rge=&bm=&PgStart=0&PgLength=10&SortCol=6&SortDir=asc&Command=Search"
objIE.navigate Source
Do While objIE.Busy = True Or objIE.readyState <> 4: DoEvents: Loop
Set ele = objIE.document.getElementsByTagName("select")
For Each opt In ele
If opt.getAttribute("name") = "tblWellRecords_length" Then
opt.Focus
opt.Value = -1
Exit For
End If
Next opt
End Sub
I was looking for the "All" element to click on the production data. it's changing the value of the drop-down if I fix the attribute name as "productionTable_length" but it's not clicking the element. If it is clicked then the total number of shows will be changed from 1-25 to 1-166. Please help me on that.... – Partha 2 hours ago
You did not say about that :) Your question was how to select a specific dropdown element in webpage by excel vba For which I gave you the answer. For what you want, there is a different approach.
Try this
Sub Sample()
Dim objIE As Object, source As String, script As String
Set objIE = CreateObject("InternetExplorer.Application")
objIE.Visible = True
source = "https://secure.conservation.ca.gov/WellSearch/Details?api=01120716&District=&County=&Field=&Operator=&Lease=&APINum=&address=&ActiveWell=false&ActiveOp=false&Location=&sec=&twn=&rge=&bm=&PgStart=0&PgLength=10&SortCol=6&SortDir=asc&Command=Search"
objIE.navigate source
Do While objIE.Busy = True Or objIE.readyState <> 4: DoEvents: Loop
script = "$('select[name=productionTable_length]').val('-1'); $('select[name=productionTable_length]').trigger('change');"
objIE.document.parentWindow.execScript script, "jscript"
End Sub

VBA: copy data from website into excel

I have a VBA code that selects info from drop-down menus on a government website and then submits the query. The requested data then opens up in another IE page. I am trying to copy this data into excel; however, I am unable to do so.
My code currently copies the text on the first IE page that contains the drop-down menus. The government website is: http://www.osfi-bsif.gc.ca/Eng/wt-ow/Pages/FINDAT.aspx
I have look all over the internet for a solution but nothing seems to work...
Here is my code:
Sub GetOsfiFinancialData()
Dim UrlAddress As String
UrlAddress = "http://ws1.osfi-bsif.gc.ca/WebApps/FINDAT/DTIBanks.aspx?T=0&LANG=E"
Dim ie As Object
Set ie = CreateObject("internetexplorer.application")
With ie
.Silent = True
.Visible = False
.navigate UrlAddress
End With
Do Until Not ie.Busy And ie.readyState = 4
DoEvents
Loop
Application.Wait (Now() + TimeValue("00:00:05"))
'Select Bank
ie.document.getElementById("DTIWebPartManager_gwpDTIBankControl1_DTIBankControl1_institutionTypeCriteria_institutionsDropDownList").Value = Z005
'open window with financial data
Dim objButton
Set objButton = ie.document.getElementById("DTIWebPartManager_gwpDTIBankControl1_DTIBankControl1_submitButton")
objButton.Focus
objButton.Click
'select new pop-up window
marker = 0
Set objshell = CreateObject("Shell.Application")
IE_count = objshell.Windows.Count
For x = 0 To (IE_count - 1)
On Error Resume Next ' sometimes more web pages are counted than are open
my_title = objshell.Windows(x).document.Title
If my_title Like "Consolidated Monthly Balance Sheet" & "*" Then 'compare to find if the desired web page is already open
Set ie = objshell.Windows(x)
marker = 1
Exit For
Else
End If
Next
Do Until Not ie.Busy And ie.readyState = 4
DoEvents
Loop
Application.Wait (Now() + TimeValue("00:00:05"))
Dim doc As MSHTML.HTMLDocument
Dim tables As MSHTML.IHTMLElementCollection
Dim table As MSHTML.HTMLTable
Dim clipboard As MSForms.DataObject
Set doc = ie.document
Set tables = doc.getElementsByTagName("body")
Set table = tables(0)
Set clipboard = New MSForms.DataObject
'paste in sheets
Dim test
Set test = ActiveWorkbook.Sheets("Test")
clipboard.SetText table.outerHTML
clipboard.PutInClipboard
test.Range("A1").PasteSpecial xlPasteAll
clipboard.Clear
MsgBox ("Task Completed")
End Sub
Your help is greatly appreciated!
You were using the current test with document.Title. I found that For Each of all windows looking for the full title worked in combination with copy pasting the pop-up window outerHTML. No additional wait time was required.
Inside the For Each Loop, after you reset the IE instance to the new window, you can obtain the new URL with ie.document.url. As you already have the data loaded you might as well just copy paste it straight away in my opinion.
Code:
Option Explicit
Public Sub GetOsfiFinancialData()
Dim UrlAddress As String, objButton, ie As Object
UrlAddress = "http://ws1.osfi-bsif.gc.ca/WebApps/FINDAT/DTIBanks.aspx?T=0&LANG=E"
Set ie = CreateObject("internetexplorer.application")
With ie
.Silent = True
.Visible = False
.navigate UrlAddress
While .Busy Or .readyState < 4: DoEvents: Wend
.document.getElementById("DTIWebPartManager_gwpDTIBankControl1_DTIBankControl1_institutionTypeCriteria_institutionsDropDownList").Value = "Z005"
Set objButton = .document.getElementById("DTIWebPartManager_gwpDTIBankControl1_DTIBankControl1_submitButton")
objButton.Focus
objButton.Click
Dim objShellWindows As New SHDocVw.ShellWindows, currentWindow As IWebBrowser2
For Each currentWindow In objShellWindows
If currentWindow.document.Title = "Consolidated Monthly Balance Sheet - Banks, Trust and Loan" Then
Set ie = currentWindow
Exit For
End If
Next
Dim clipboard As Object
Set clipboard = GetObject("New:{1C3B4210-F441-11CE-B9EA-00AA006B1A69}")
clipboard.SetText ie.document.body.outerHTML
clipboard.PutInClipboard
ThisWorkbook.Worksheets("Sheet1").Cells(1, 1).PasteSpecial
.Quit
End With
End Sub
References (VBE > Tools > References):
Microsoft Internet Controls
I don't have time to get into all the stuff about controlling one browser from another, but I think you can figure that part out, especially since you made some great progress on this already. Get URL#2 from URL#1, like you are doing, but with some better data controls around it, and then do this...
Option Explicit
Sub Web_Table_Option_One()
Dim xml As Object
Dim html As Object
Dim objTable As Object
Dim result As String
Dim lRow As Long
Dim lngTable As Long
Dim lngRow As Long
Dim lngCol As Long
Dim ActRw As Long
Set xml = CreateObject("MSXML2.XMLHTTP.6.0")
With xml
.Open "GET", "http://ws1.osfi-bsif.gc.ca/WebApps/Temp/2f40b7ef-d024-4eca-a8a3-fb82153efafaFinancialData.aspx", False
.send
End With
result = xml.responseText
Set html = CreateObject("htmlfile")
html.body.innerHTML = result
Set objTable = html.getElementsByTagName("Table")
For lngTable = 0 To objTable.Length - 1
For lngRow = 0 To objTable(lngTable).Rows.Length - 1
For lngCol = 0 To objTable(lngTable).Rows(lngRow).Cells.Length - 1
ThisWorkbook.Sheets("Sheet1").Cells(ActRw + lngRow + 1, lngCol + 1) = objTable(lngTable).Rows(lngRow).Cells(lngCol).innerText
Next lngCol
Next lngRow
ActRw = ActRw + objTable(lngTable).Rows.Length + 1
Next lngTable
End Sub

Resources