Puppeteer crashes 'Error: net::ERR_ABORTED at' - node.js

Question

I have a NodeJS script using puppeteer to download few hundred thousand documents from a website. And Each file takes about 1.5s to get the download link. Due to high download count, and because of that massive amount of time it takes, I thought it would be a good idea to download multiple at a time.
So my idea was to open the list of pages the files contains using main.js, and send batches of x files to x child processes then open each in a new browser and then to download. (Due to the nature of this website, they don't have a direct download path, it generates the path on the process. So plugging download url directly is not an option.)
main
const fs = require('fs');
const puppeteer = require('puppeteer');
const path = require('path');
const fork = require('child_process').fork;
var zip = require ('./zip_files');
var fld = require('./create_folders');
var log = require('./login');
var ls = [];
const docType = ['pdf', 'word', 'spreadsheet'];
const docExt = ['.pdf', '.docx', '.xlsx'];
const login_url = 'https://www.scribd.com/login';
const login_email = '';
const login_password = '';
const t_out = 10000;
/*
Link Format
https://www.scribd.com/search?query=Exporters+in+Andra+Pradesh&content_type=documents&page=1&filetype=pdf
*/
var pages=[];
const datasetF1 = 'wordset_11.csv';
const datasetF2 = 'wordset_22.csv';
var data_1 = [];
var data_2 = [];
var fileCount = 1000000;
data_1 = fs.readFileSync(datasetF1).toString().split(',');
data_2 = fs.readFileSync(datasetF2).toString().split(',');
(async() => {
const browser = await puppeteer.launch(({
headless: false,
userDataDir: path.join(__dirname,'user_data/main'),
}));
console.log(browser)
const page = (await browser.pages())[0];
//await log.login(page, login_url, login_email, login_password, t_out);
child();
//await page.goto('https://www.scribd.com/', { waitUntil: 'networkidle2' });
for(let i = 0; i < data_1.length; i++){
for(let j = 0; j < data_2.length; j++){
let folder = data_1[i].replace(/\n/gm, '').replace(/\r/gm, '') + ' ' + data_2[j].replace(/\n/gm, '').replace(/\r/gm, '');
let searchTerm = data_1[i].replace(/\n/gm, '').replace(/\r/gm, '') + ' ' + data_2[j].replace(/\n/gm, '').replace(/\r/gm, '');
for(let pageNum = 1; pageNum < (Math.ceil(fileCount/42) +1) && pageNum < 236; pageNum++){
//maxPageNum = 235
let docType = 'pdf';
let query = 'https://www.scribd.com/search?query='+searchTerm+'&content_type=documents&page='+pageNum+'&filetype='+docType;
console.log(pageNum);
await page.goto(query, { waitUntil: 'networkidle2' });
fs.appendFileSync('progress/query.txt', query + '\n');
if (pageNum == 1){
await page.waitForXPath('//div[#class="_7a1igU"]', { timeout: t_out }).then(async() => {
let fileCountR = await page.$x('//div[#class="_7a1igU"]');
let fileCountS = await (await fileCountR[0].getProperty('textContent')).jsonValue();
var fileCount = parseInt(fileCountS.split('of ').pop().split(' results').shift().replace(/,/g, ''));
console.log(fileCount);
}).catch( e => {
console.log('ERROR1');
console.log(e);
//console.log(e);
});
}
await page.waitForXPath('//section[#data-testid="search-results"]', { timeout: t_out }).then(async() => {
let searchResults = await page.$x('//section[#data-testid="search-results"]');
await searchResults[0].waitForXPath('//div/ul/li');
let docPages = await searchResults[0].$x('//div/ul/li');
await download(browser, /*page,*/ docPages, folder);
}).catch( e => {
console.log('ERROR2');
console.log(e);
});
}
//await zip.zipFolder(down_path + folder, zip_path + folder + '.zip');
}
}
//browser.close();
})();
async function child(){
for(let a = 0; a < 3; a++){
ls[a] = fork('download.js');
}
}
async function download(browser, docPages, folder){
let child_count = 3;
const paged = await new Promise(async (resolve, reject) => {
for(let b = 0; b < docPages.length; b+=child_count){
await send_data(browser, docPages, folder, child_count, b);
console.log("b : b");
}
});
}
async function send_data(browser, docPages, folder, child_count, b){
let c = 0;
await new Promise(async (resolve, reject) => {
for(let a = 0; a < child_count; a++){
let urlR = await docPages[a+b].$x('//article/a');
let url = await (await urlR[0].getProperty('href')).jsonValue();
fs.appendFileSync('progress/page.txt', url + '\n');
ls[a].on('message', (msg) => {
if(msg == 'Executed'){
}if(msg == 'done'){
c++
if(c == (child_count-1)){
resolve({});
}
}
});
ls[a].send([url, folder, a]);
}
});
}
child (download.js)
const puppeteer = require('puppeteer');
//const page = (await browser.pages())[0];
const down_path = 'downloads/';
const error_path = 'errors/';
const fs = require('fs');
const path = require('path');
var log = require('./login');
const login_url = 'https://www.scribd.com/login';
const login_email = '';
const login_password = '';
const t_out = 1000;
const dowload_errorFile = 'errors.txt';
var browser;
var flag = 0;
const t_outforL = 100000;
process.on('message', async (obj)=> {
console.log("CHILD: data received from parent process", obj);
console.log("Process Starting : ", process.pid);
url = obj[0];
loc = obj[1];
ins = obj[2];
if(!flag){
browser = await puppeteer.launch(({
headless: false,
userDataDir: path.join(__dirname,'user_data/'+ins),
}));
const downPage = (await browser.pages())[0];
await downPage.goto('https://www.scribd.com', { waitUntil: 'networkidle2' });
flag = 1;
}
await download(url, loc);
});
process.send('Executed');
async function download(url, folder){
const downPage = (await browser.pages())[0];
await downPage.goto(`${url}`, { waitUntil: 'networkidle2' });
await downPage.waitForXPath('//button[#data-e2e="download-button"]', { timeout: 1000 }).then(async ()=>{
let dwnld = await downPage.$x('//button[#data-e2e="download-button"]');
const client = await downPage.target().createCDPSession()
await client.send('Page.setDownloadBehavior', {
behavior: 'allow',
downloadPath: path.join(__dirname, 'downloads/' + folder),
})
await dwnld[0].click();
await downPage.waitForXPath('//a[#class="wrapper__filled-button download_selection_btn"]');
let clicks = await downPage.$x('//a[#class="wrapper__filled-button download_selection_btn"]');
await clicks[0].click();
await downPage.waitForTimeout(500);
process.send('done');
}).catch( e => {
console.log('No download exists');
console.log(e);
});
await downPage.waitForTimeout(2500);
//process.exit();
}
It runs perfectly fine for a round or 2, but then puppeteer crashes with
Error: net::ERR_ABORTED at https://www.scribd.com/document/456861933
at navigate (D:\Projects\Upwork\Project 6 (Rustom Vazifdar)\code\node_modules\puppeteer-core\lib\cjs\puppeteer\common\Frame.js:238:23)
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async Frame.goto (D:\Projects\Upwork\Project 6 (Rustom Vazifdar)\code\node_modules\puppeteer-core\lib\cjs\puppeteer\common\Frame.js:207:21)
at async CDPPage.goto (D:\Projects\Upwork\Project 6 (Rustom Vazifdar)\code\node_modules\puppeteer-core\lib\cjs\puppeteer\common\Page.js:439:16)
at async download (D:\Projects\Upwork\Project 6 (Rustom Vazifdar)\code\download.js:59:3)
at async process.<anonymous> (D:\Projects\Upwork\Project 6 (Rustom Vazifdar)\code\download.js:53:2)