Trying to copy the text of a web page into Excel - excel

Question

I am using this code to open Edge so that I can grab the text of the web page. I can get Edge to open and display the page, and I can copy the text of the page manually and then finish executing the code and it pastes it as needed, but I'm not at the skill level to figure out how to make Excel copy the text. It's the last step that I need.
Any help would be greatly appreciated.
I'm looking to copy the entire page, and not just one section of it.
Sub LOADEdge()
Set obj = CreateObject("Shell.Application")
obj.ShellExecute "microsoft-edge:https://maps.googleapis.com/maps/api/directions/json?origin=Disneyland&destination=Universal+Studios+HOllywood&key=your_API_Key"
'This is where I manually grab the text from the page.
MsgBox "wiating"
Worksheets("sheet1").Range("A1").PasteSpecial Page = xlpagevalues
End Sub
I've tried to research how to copy web pages but most require Selenium to be installed and I can't get my IT department to do that for me. I've found work arounds on how to open the browser but I can't find anything that shows me how to copy the text of the page other than:
IE.ExecWB 17, 0 '// SelectAll
IE.ExecWB 12, 2 '// Copy selection
but that doesn't work.

Answer 1

While it is possible, I am not sure that VBA is a good tool for web scraping. I tried it a few times, and it never gave me the results I was looking for. Here are some alternatives to look at:
Using Google Sheets
Using Python
There is also an add-in for Microsoft One-Note that lets you save the entire content of a webpage to OneNote, and it works quite well.

Answer 2

It is node.js with exceljs and axios library.
This code will write excel file.
It needs to paste your 'API_KEY' string at line 4.
Save as get-map.js
const axios = require('axios')
const Excel = require('exceljs');
const API_KEY ='your_API_Key'
const columnName = (index) => {
var cname = String.fromCharCode(65 + ((index - 1) % 26))
if (index > 26)
cname = String.fromCharCode(64 + (index - 1) / 26) + cname
return cname;
}
const getMapInformation = async (origin, destination) => {
try {
origin = origin.replace(/ /g,"+") // replace from space to '+'
destination = destination.replace(/ /g,"+") // replace from space to '+'
const resp = await axios.get(
`https://maps.googleapis.com/maps/api/directions/json?origin=${origin}&destination=${destination}&key=${API_KEY}`
);
return Promise.resolve(resp.data);
} catch (err) {
return Promise.reject(error);
}
}
const writeMapInformation = (fileName, data) => {
try {
const workbook = new Excel.Workbook();
// make workbook with 'map' name
const ws = workbook.addWorksheet("map")
// Start Cell A1 for title column
let headerColumn = 1
let row_number = 1
// #1 geocoded_waypoints section
for (let waypoint in data.geocoded_waypoints) {
for (let key in data.geocoded_waypoints[waypoint]) {
ws.getCell(columnName(headerColumn) + String(row_number)).value = key
ws.getCell(columnName(headerColumn + 1) + String(row_number)).value = data.geocoded_waypoints[waypoint][key]
row_number++;
}
row_number++; // skip one row
}
// #2 routes section
for (let index in data.routes) {
for (let key in data.routes[index]) {
if(key == 'bounds') {
ws.getCell(columnName(headerColumn) + String(row_number)).value = key
for (let sub_key in data.routes[index][key]) {
ws.getCell(columnName(headerColumn + 1) + String(row_number)).value = sub_key
ws.getCell(columnName(headerColumn + 2) + String(row_number)).value = data.routes[index][key][sub_key]
row_number++;
}
}
if(key == 'summary') {
ws.getCell(columnName(headerColumn) + String(row_number)).value = key
ws.getCell(columnName(headerColumn + 1) + String(row_number)).value = data.routes[index][key]
}
if(key == 'legs') {
ws.getCell(columnName(headerColumn) + String(row_number)).value = key
row_number++;
for (let sub_key in data.routes[index][key][0]) {
if(sub_key != 'steps') {
ws.getCell(columnName(headerColumn + 1) + String(row_number)).value = sub_key
ws.getCell(columnName(headerColumn + 2) + String(row_number)).value = data.routes[index][key][0][sub_key]
row_number++;
}
if(sub_key == 'steps') {
for (let sub_child_key in data.routes[index][key][0][sub_key]) {
for (let grand_key in data.routes[index][key][0][sub_key][sub_child_key]) {
// skip two fileds (html_instructions & polyline)
if(grand_key != 'html_instructions' && grand_key != 'polyline') {
ws.getCell(columnName(headerColumn + 1) + String(row_number)).value = grand_key
ws.getCell(columnName(headerColumn + 2) + String(row_number)).value = data.routes[index][key][0][sub_key][sub_child_key][grand_key]
row_number++;
}
}
}
}
}
row_number++;
}
}
}
workbook.xlsx
.writeFile(fileName)
.then(() => {
console.log('file created');
})
.catch(err => {
console.log(err.message);
});
} catch (err) {
console.log(err.message);
}
}
// get map direction (from, to)
getMapInformation('Disneyland', 'Universal Studios Hollywood')
.then((result) => {
writeMapInformation('map.xlsx', result)
})
Install libraries
npm install axios exceljs
Run it
node get-map.js
Results