Queueing Asynchronous tasks - node.js

Question

I'm trying to create a script that takes a list of URL's, goes to the site and takes a screenshot.
I have managed to get this to work with puppeteer. However the problem I've had is when I have say 50 URLs in the list, it will attempt to launch puppet sessions for all of them at once, which means that most time out before the site loads and it can take a screenshot.
I've found I can successfully run 10 at once, so I'm wanting to set up a queueing system to do this.
parser.on('readable', function(){
while(record = parser.read()){
counter +=1;
console.log(record.URL);
(async (url = record.URL, name = record.shortURL, counter1 = counter) => {
const browser = await puppeteer.launch( {defaultViewport: {width: 1024, height:768} } );
const page = await browser.newPage();
await page.goto(url);
title = await page.title();
domainRegex = /^(?:https?:\/\/)?(?:[^#\n]+#)?(?:www\.)?([^:\/\n?]+)/img;
match = domainRegex.exec(url);
width = 1024;//await page.viewport().width;
height = 1000;//await page.viewport.height();
await page.screenshot({path: "Screenshots/"+counter1+". "+match[1] + "- " +title.replace(/[\W_]+/g,"")+".jpg", clip : {x:0, y:0, width: width, height: height}});
await browser.close();
})();
}
});

Answer 1

The code below will initially launch 10 sessions. Once each session finishes, it will dequeue the next record and launch another one, until there are no more records remaining. This will ensure that a max of 10 will be running at the same time.
parser.on('readable', async () => {
const maxNumberOfSessions = 10;
let counter = 0;
await Promise.all(Array.from({length: maxNumberOfSessions}, dequeueRecord));
console.log("All records have been processed.");
function dequeueRecord() {
const nextRecord = parser.read();
if(nextRecord) return processRecord(nextRecord).then(dequeueRecord);
}
async function processRecord(record) {
const number = ++counter;
console.log("Processing record #" + number + ": " + record.URL);
const browser = await puppeteer.launch({defaultViewport: {width: 1024, height: 768}});
const page = await browser.newPage();
await page.goto(record.URL);
const title = await page.title();
const domainRegex = /^(?:https?:\/\/)?(?:[^#\n]+#)?(?:www\.)?([^:\/\n?]+)/img;
const match = domainRegex.exec(record.URL);
const width = 1024; // await page.viewport().width;
const height = 1000; // await page.viewport().height;
await page.screenshot({path: "Screenshots/" + number + ". " + match[1] + "- " + title.replace(/[\W_]+/g, "") + ".jpg", clip: {x: 0, y: 0, width, height}});
await browser.close();
}
});

Answer 2

If you want to run all of them serially, you can turn this into a async function and await it. This way, it will run one by one.
// let's separate it for readability
async function getRecord(record, counter) {
const url = record.URL,
name = record.shortURL,
counter1 = counter;
const browser = await puppeteer.launch({
defaultViewport: {
width: 1024,
height: 768
}
});
const page = await browser.newPage();
await page.goto(url);
title = await page.title();
domainRegex = /^(?:https?:\/\/)?(?:[^#\n]+#)?(?:www\.)?([^:\/\n?]+)/img;
match = domainRegex.exec(url);
width = 1024; //await page.viewport().width;
height = 1000; //await page.viewport.height();
await page.screenshot({
path: "Screenshots/" + counter1 + ". " + match[1] + "- " + title.replace(/[\W_]+/g, "") + ".jpg",
clip: {
x: 0,
y: 0,
width: width,
height: height
}
});
await browser.close();
}
parser.on('readable', async function() { // <-- here we make it async
while (record = parser.read()) {
counter += 1;
console.log(record.URL);
await getRecord(record, counter) // <-- and we await each call
}
});
There are other ways like Promise.map and for..of, but let's keep this simpler for now.

Answer 3

You might want to take a look at puppeteer-cluster (disclaimer: I'm the author).
You can do it like this:
(async () => {
// create a cluster that handles 10 parallel browsers
const cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_BROWSER,
maxConcurrency: 10,
});
// define the task
await cluster.task(async ({ page, data: { counter, record} }) => {
const url = record.URL;
await page.goto(url);
title = await page.title();
domainRegex = /^(?:https?:\/\/)?(?:[^#\n]+#)?(?:www\.)?([^:\/\n?]+)/img;
match = domainRegex.exec(url);
width = 1024;//await page.viewport().width;
height = 1000;//await page.viewport.height();
await page.screenshot({path: "Screenshots/"+counter+". "+match[1] + "- " +title.replace(/[\W_]+/g,"")+".jpg", clip : {x:0, y:0, width: width, height: height}});
});
// queue your jobs
parser.on('readable', function () {
while (record = parser.read()) {
counter += 1;
cluster.queue({ counter, record });
}
});
})();
This will handle 10 parallel browser instances and will also take care of browser crashes and error handling.

Answer 4

If you want to run a set of promises in sequence you could make use of Promise.mapSeries from Bluebird package. I understand that this would mean adding an additional package, but it's simple and does not need you to build a queuing system.
http://bluebirdjs.com/docs/api/promise.mapseries.html