NodeJS program to scrape table columns from Wikipedia - node.js

Question

I am in the process of learning Node JS and how to web scrape. I thought it would be a good approach to extract columns from a wikipedia page https://en.wikipedia.org/wiki/List_of_S%26P_500_companies
I've been learning about web scraping with Cheerio but am unsure how to code this in NodeJS. I've become familiar with html selectors to identify elements on a page but am not sure how to extract into a program. I plan on extracting this information into a list. I am hoping to extract the Symbol and Security columns on the table on the wiki page.
Below is the code I have compiled and the result I am getting. I have created a const based off a selector on the webpage. I believe it is supposed to return all the values in the column based off the selector.
var AWS = require("aws-sdk");
var AWS = require("aws-sdk/global");
AWS.config.apiVersions = {
dynamodb: '2012-08-10'
};
var dynamodb = new AWS.DynamoDB();
const cheerio = require('cheerio');
const axios = require('axios');
const express = require('express');
async function getStocks() {
try {
const url = ' https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'
const { data } = await axios({
method: "GET",
url: url,
})
const $ = cheerio.load(data)
const elemSelector = '#constituents > tbody > tr:nth-child(1) > td'
$(elemSelector).each((parentIdx, parentElem) => {
console.log(parentIdx)
})
console.log($)
} catch (err) {
console.error(err)
}
}
getStocks()
Result
[Function: initialize] {
html: [Function: html],
xml: [Function: xml],
text: [Function: text],
parseHTML: [Function: parseHTML],
root: [Function: root],
contains: [Function: contains],
merge: [Function: merge],
load: [Function: load],
_root: Node {
type: 'root',
name: 'root',
parent: null,
prev: null,
next: null,
children: [ [Node], [Node] ],
'x-mode': 'no-quirks'
},
_options: { xml: false, decodeEntities: true },
fn: Cheerio { constructor: [Function: LoadedCheerio] }
}
[Finished in 2.384s]

Answer 1

This should help. Follow the comments to understand what is happening. You didn't specify how you wanted the data output, so I have them as arrays but you can add code to map the table data with the table headers now that you have the raw data.
// parses HTML
var $ = cheerio.load(data);
// the number of columns you want to target on the table, starting from the left column
var number_of_columns = 2;
// targets the specific table with a selector
var html_table = $('table#constituents');
// gets table header titles; loops through all th data, and pulls an array of the values
var table_header = html_table.find('th').map(function() {return $(this).text().trim();}).toArray();
// removes columns after the number of columns specified
table_header = table_header.slice(0, number_of_columns);
// gets table cell values; loops through all tr rows
var table_data = html_table.find('tbody tr').map(function(tr_index) {
// gets the cells value for the row; loops through each cell and returns an array of values
var cells = $(this).find('td').map(function(td_index) {return $(this).text().trim();}).toArray();
// removes columns after the number of columns specified
cells = cells.slice(0, number_of_columns);
// returns an array of the cell data generated
return [cells];
// the filter removes empty array items
}).toArray().filter(function(item) {return item.length;});
// output the table headers
console.log('table_header', table_header);
// output the table data
console.log('table_data', table_data);

Answer 2

In nodejs you can user puppeteer for your web scraping. it's a very good library with a lot of browser settings and query selectors options!
You can find more about the library reading the documentation at https://pptr.dev/
If you want to integrate it with an API I can help you too.