How to scrape multi-level links using puppeteer js? - node.js

Question

I am scraping table rows of site page using Puppeteer. I have the code to scrape content and assign them to an object for each in the table. In each table row there is a link that I need to open in a new page (puppeteer) and then scrape for a particular element then assign it to the same object and return the whole object with the new keys to puppeteer. How is that possible with Puppeteer?
async function run() {
const browser = await puppeteer.launch({
headless: false
})
const page = await browser.newPage()
await page.goto('https://tokenmarket.net/blockchain/', {waitUntil: 'networkidle0'})
await page.waitFor(5000)
var onlink = ''
var result = await page.$$eval('table > tbody tr .col-actions a:first-child', (els) => Array.from(els).map(function(el) {
//running ajax requests to load the inner page links.
$.get(el.children[0].href, function(response) {
onlink = $(response).find('#page-wrapper > main > div.container > div > table > tbody > tr > td:nth-child(2)').text()
})
return {
icoImgUrl: el.children[0].children[0].children[0].currentSrc,
icoDate: el.children[2].innerText.split('\n').shift() === 'To be announced' ? null : new Date( el.children[2].innerText.split('\n').shift() ).toISOString(),
icoName:el.children[1].children[0].innerText,
link:el.children[1].children[0].children[0].href,
description:el.children[3].innerText,
assets :onlink
}
}))
console.log(result)
UpcomingIco.insertMany(result, function(error, docs) {})
browser.close()
}
run()

Answer 1

If you try opening a new tab for each ICO page in parallel you might end up with 100+ pages loading at the same time.
So the best thing you could do is to first collect the URLs and then visit them one by one in a loop.
This also allows keeping the code simple and readable.
For example (please, see my comments):
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://tokenmarket.net/blockchain/');
// Gather assets page urls for all the blockchains
const assetUrls = await page.$$eval(
'.table-assets > tbody > tr .col-actions a:first-child',
assetLinks => assetLinks.map(link => link.href)
);
const results = [];
// Visit each assets page one by one
for (let assetsUrl of assetUrls) {
await page.goto(assetsUrl);
// Now collect all the ICO urls.
const icoUrls = await page.$$eval(
'#page-wrapper > main > div.container > div > table > tbody > tr > td:nth-child(2) a',
links => links.map(link => link.href)
);
// Visit each ICO one by one and collect the data.
for (let icoUrl of icoUrls) {
await page.goto(icoUrl);
const icoImgUrl = await page.$eval('#asset-logo-wrapper img', img => img.src);
const icoName = await page.$eval('h1', h1 => h1.innerText.trim());
// TODO: Gather all the needed info like description etc here.
results.push([{
icoName,
icoUrl,
icoImgUrl
}]);
}
}
// Results are ready
console.log(results);
browser.close();