Puppeteer: goto() and waitForSelector() always fail on Cloud Functions - node.js

Question

I am using Puppeteer to do some webscraping which is executed on a scheduled pubsub Cloud Function. The issue that I have is that the page.goto() and page.waitForSelector() never ever completes when I deploy my function onto Firebase Cloud Function. The script works fine locally on my machine.
Here is my implementation so far:
//Scheduled pubsub function at ./functions/index.js
exports.scraper = functions.pubsub
.onRun((context) => {
var scraper = new ScraperManager();
return scraper.start();
})
//Entry function
ScraperManager.prototype.start = async function() {
var webpagePromises = []
for (const agency of agencies) {
for (page_num = 0; page_num < num_of_pages; page_num++) {
const url = setupUrl(agency, page_num); //Returns a url
const webpagePromise = getWebpage(agency, page_num, url)
webpagePromises.push(webpagePromise)
}
}
return Promise.all(webpagePromises)
}
async function getWebpage(agency, page_num, url) {
var data = {}
const browser = await puppeteer.launch(constants.puppeteerOptions);
try {
const page = await browser.newPage();
await page.setUserAgent(constants.userAgent);
await page.goto(url, {timeout: 0});
console.log("goto completes")
await page.waitForSelector('div.main_container', {timeout: 0});
console.log("waitFor completes")
const body = await page.$eval('body', el => el.innerHTML);
data['html'] = body;
return data;
} catch (err) {
console.log("Puppeteer error", err);
return;
} finally {
await browser.close();
}
}
//PuppeteerOptions in constants file
puppeteerOptions: {
headerless: true,
args: [
'--disable-gpu',
'--no-sandbox',
]
}
Note that the {timeout: 0} is necessary as the page.goto and page.waitForSelector() takes more time than the default timeout value of 30000ms. Even with the timeout disabled, both goto and waitForSelector() never completes, ie both the console.log() statements do not get logged. The above script works fine when running the script locally, ie console.log() does print out correctly, but never works when deployed on Cloud Functions. The entire cloud function always get timedout (presently set at 300s) without any logs printed.
Would anybody be able to advice?

Answer 1

I had the same kind of issue: a stupid page.$eval() to get a simple node would never return (actually it was taking more than 3 minutes ...) or crash the page after more than 5 minutes on a virtual private server, while the same script was doing fine on my local computer.
Looking at the virtual server's RAM usage (around 99%), I've come to the conclusion that this kind of script cannot run on a server with only 2 GB or RAM.
:-(