Firebase function will not deploy when requiring outside packages - node.js

Question

I am having trouble deploying my web scraping function and do not know how to fix the issue.
Index.js
const functions = require("firebase-functions");
const pup = require("puppeteer");
const WebsiteData = require('./schema');
exports.scrape = functions
.runWith({ memory: '1GB' })
.pubsub.schedule('0 0 * * *')
.onRun(async (context) => {
const browser = await pup.launch({
headless: true,
args: ['--no-sandbox', '--disable-setuid-sandbox']
});
const page = await browser.newPage();
await page.setUserAgent(
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36'
)
var pages = [
{
name: 'aave-protocol',
fee:'0.09',
url: 'https://aave.com',
},
{
name: 'uniswap-v2',
fee:'0.30',
url: 'https://uniswap.org',
},
{
name: 'dydx',
fee:'0.00',
url: 'https://dydx.exchange',
},
{
name: 'dodo-bsc',
fee:'0.00',
url: 'https://dodoex.io',
},
{
name: 'cream-finance',
fee:'0.03',
url: 'https://cream.finance',
},
{
name: 'multiplier-finance',
fee:'0.06',
url: 'https://multiplier.finance',
}
]
var result = [];
for (var each in pages) {
await page.goto(`https://www.dapp.com/app/${pages[each].name}`, { waitUntil: 'networkidle0' })
var users = await page.evaluate(() => {
return document.querySelector('#app > div.app-container > div.root.dapp-detail > section.detail-page-outer > div > div.left-sec > div.stats-card > div.chart-section > div:nth-child(2) > div:nth-child(4) > span.value > span').innerText
})
var volume = await page.evaluate(() => {
return document.querySelector('#app > div.app-container > div.root.dapp-detail > section.detail-page-outer > div > div.left-sec > div.stats-card > div.chart-section > div:nth-child(3) > div:nth-child(4) > span.value > span:nth-child(2)').innerText
})
var obj = { "name": `${pages[each].name}`, "nDUniqueUsers": `${users}`, "nDTransactions": `${volume}`, "fee": `${pages[each].fee}`, "url": `${pages[each].url}`};
result.push(obj);
}
await browser.close();
const websiteMongo = new WebsiteData({ sites: result });
await websiteMongo.save(function(err,data){
if (err) {
console.log(err);
return null;
}
});
console.log("Done.")
return null;
});
The function is meant to use puppeteer to open up around 5 pages collect the info and upload the data to a MongoDB database. The code works perfectly in local host, but when I run firebase deploy, I get an error. Here is the error message: "Function failed on loading user code. This is likely due to a bug in the user code. Error message: Error: please examine your function logs to see the error cause: https://cloud.google.com/functions/docs/monitoring/logging#viewing_logs. Additional troubleshooting documentation can be found at https://cloud.google.com/functions/docs/troubleshooting#logging. Please visit https://cloud.google.com/functions/docs/troubleshooting for in-depth troubleshooting documentation."}"
I know that the problem consists of these two lines:
const pup = require("puppeteer");
const WebsiteData = require('./schema');
When I comment out those two lines I can deploy the function. Here is the code for schema.js:
var mongoose = require('mongoose')
mongoose.connect("URI");
var Schema = mongoose.Schema
var websiteData = new Schema({
sites: [{
name: {
type: String,
required : true,
},
nDUniqueUsers: {
type: String,
required : true,
},
nDTransactions: {
type: String,
required : true,
}, fee: {
type: String,
required : true,
}, url: {
type: String,
required : true,
}
}],
})
var WebsiteData = mongoose.model("SiteData", websiteData);
module.exports = WebsiteData
I do not know how to fix this. Any help would be greatly appreciated.

Answer 1

I suspect you haven't included "puppeteer" in the dependencies section of the package.json deployed alongside your function. Or possibly you've inadvertently included it in the devDependencies section instead of dependencies.
// package.json
{
// ...
"dependencies": {
"puppeteer": "^13.5.1" // be sure you have this
}
}