Firebase + Node: Updating two refs results in 304 errors - node.js

What I have is a function that resets a ref when called. It works when updating only one ref.
This is the code for that part of the function:
if (streakVal !== 0) {
//reset
const uid = item.child('uid').val();
ref.child(uid).update({ streak: 0 }).catch(err => {
res.status(500).send(err);
});
}
I also want to update another ref in a different part of the database. I've included it in the same function, as the only difference is the ref location. The part then looks like this:
if (streakVal !== 0) {
//reset
const uid = item.child('uid').val();
ref.child(uid).update({ streak: 0 }).then(() => {
boardRef.child(uid).update({ score: 0 }).catch(err => {
res.status(500).send(err);
});
}).catch(err => {
res.status(500).send(err);
});
}
The first snippet works. However, the second results in the error Function execution took 800 ms, finished with status code: 304 I'm wondering why this is and how to fix it. Maybe I'm not structuring it correctly as I'm new to Node. I'm sure that's the correct path to both refs. Here is the full function:
export const resetStreak = functions.https.onRequest((req, res) => {
const ref = db.ref('users');
const boardRef = db.ref('streakLeaderboard');
ref.once('value').then(snap => {
snap.forEach(item => {
const streakVal = item.child('streak').val();
const lastQuestTimestamp = item.child('lastQuest').val();
const today = new Date();
const d = new Date(lastQuestTimestamp);
if (sameDay(today, d) === false) {
if (streakVal !== 0) {
//reset
const uid = item.child('uid').val();
ref.child(uid).update({ streak: 0 }).then(() => {
boardRef.child(uid).update({ score: 0 }).catch(err => {
res.status(500).send(err);
});
}).catch(err => {
res.status(500).send(err);
});
}
}
})
}).catch(err => {
res.status(500).send(err);
});
Thank you!

You should return your Promise in your HTTPS functions (docs). Your code should become:
export const resetStreak = functions.https.onRequest((req, res) => {
const ref = db.ref('users');
const boardRef = db.ref('streakLeaderboard');
return ref.once('value').then(snap => {
const promises = []
snap.forEach(item => {
const streakVal = item.child('streak').val();
const lastQuestTimestamp = item.child('lastQuest').val();
const today = new Date();
const d = new Date(lastQuestTimestamp);
if (sameDay(today, d) === false) {
if (streakVal !== 0) {
//reset
const uid = item.child('uid').val();
promises.push(ref.child(uid).update({ streak: 0 }).then(() => {
return boardRef.child(uid).update({ score: 0 });
}));
}
}
})
return Promise.all(promises);
}).then(() => {
res.sendStatus(200);
}).catch(err => {
res.status(500).send(err);
});

Related

how to work with response object in nodejs stream, exceljs and worker thread

I am using worker thread and stream at same time in node JS project. At initial I was not able to pass res object through main process to worker thread. I saw many stackoverflow question and solution and wrote a solution which works great. I created a Readable stream in main thread and writable stream in worker thread. while doing this, I have done a huge calculation from more than 10 table and export data which takes nearly 1 minutes for processing.
code:
router.get("/downloadAll", (req, res) => {
new Promise((resolve, reject) => {
const promise = [];
promise.push(Dashboard.DUser());
promise.push(Dashboard.DDUser());
promise.push(Dashboard.DDLUser());
promise.push(Dashboard.Din());
promise.push(Dashboard.Str());
promise.push(Dashboard.R());
promise.push(Dashboard.Q());
Promise.all(promise).catch(err => err)
.then(results => {
const worker = new Worker(`${process.cwd()}/src/route/modules/dashboard/worker.js`, {
workerData: { results }
});
const fileHeaders = [
{
name: "Content-Type",
value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
}
];
res.setHeader("Content-Disposition", `attachment; filename="Stream.xlsx`);
fileHeaders.forEach(header => res.setHeader(header.name, header.value));
const readStream = new Readable({
read() {}
});
readStream.pipe(res);
worker.on("message", message => {
readStream.push(message);
});
worker.on("exit", code => {
console.log("exit", code);
resolve(true);
//if (code !== 0) reject(new Error(`stopped with ${code} exit code`));
});
});
})
.then(() => res.end())
.catch(err => console.log(err));
});
WORKER THREAD:
const { workerData, parentPort } = require("worker_threads");
const { Writable } = require("stream");
const Excel = require("exceljs");
const writableStream = new Writable();
// writableStream.on("message", () => {});
writableStream._write = (chunk, encoding, next) => {
parentPort.postMessage(chunk);
next();
};
const createWorkbook = () => {
const workbook = new Excel.stream.xlsx.WorkbookWriter({
stream: writableStream, // stream to server response
useStyles: true // not sure about this one, check with it turned off.
});
workbook.title = "Serious";
workbook.creator = "SS";
workbook.created = new Date();
return workbook;
};
const createSheet = workbook => {
workerData.results.forEach((result, index) => {
const worksheet = workbook.addWorksheet(result.title, {
properties: { outlineLevelCol: 1 }
});
worksheet.columns = Object.keys(result.data[0]).map(item => {
return { header: item, key: item };
});
result.data.forEach(row => worksheet.addRow(row).commit);
});
};
const workbook = createWorkbook();
createSheet(workbook);
workbook.commit();
The above code works fine and is fast for small calculation. when I have huge computation it is showing processing for 1 minutes and finish processing and download the xls file. so i updated the code to:
router.get("/downloadAll", (req, res) => {
const worker = new Worker(`${process.cwd()}/src/worker/worker.js`);
const fileHeaders = [
{
name: "Content-Type",
value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
}
];
const today = new Date();
res.setHeader(
"Content-Disposition",
`attachment; filename=Q-${today.getFullYear()}${String(today.getMonth() + 1).padStart(2, "0")}${String(
today.getDate()
).padStart(2, "0")}.xlsx`
);
fileHeaders.forEach(header => res.setHeader(header.name, header.value));
const readStream = new Readable({
read() {}
});
readStream.pipe(res);
worker.on("message", message => {
readStream.push(message);
});
worker.on("exit", code => {
console.log("exit", code);
res.end();
//if (code !== 0) reject(new Error(`stopped with ${code} exit code`));
});
});
and worker thread code:
const { workerData, parentPort } = require("worker_threads");
const { Writable } = require("stream");
const Excel = require("exceljs");
const { resolve } = require("path");
const db = require(`${process.cwd()}/src/modules/db.module`);
const Dashboard = require(`${process.cwd()}/src/route/modules/dashboard.model`);
const promise = [];
promise.push(Dashboard.DUser());
promise.push(Dashboard.DDUser());
promise.push(Dashboard.DDLUser());
promise.push(Dashboard.Din());
promise.push(Dashboard.Str());
promise.push(Dashboard.R());
promise.push(Dashboard.Q());
Promise.all(promise).catch(err => err)
.then(results => { const writableStream = new Writable();
// writableStream.on("message", () => {});
writableStream._write = (chunk, encoding, next) => {
console.log(chunk.toString());
parentPort.postMessage(chunk);
next();
};
const createWorkbook = () => {
const workbook = new Excel.stream.xlsx.WorkbookWriter({
stream: writableStream, // stream to server response
useStyles: true // not sure about this one, check with it turned off.
});
workbook.creator = "ss";
workbook.created = new Date();
return workbook;
};
const createSheet = workbook => {
results.forEach((result, index) => {
// console.log(result);
const worksheet = workbook.addWorksheet(result.title, {
properties: { outlineLevelCol: 1 }
});
worksheet.columns = Object.keys(result.data[0]).map(item => {
return { header: item, key: item };
});
result.data.forEach(row => worksheet.addRow(row).commit);
});
};
The above code doesnot work correctly. I can get the data from callback from promise but when its downloading its shows 300kb , 200b,1byte and ends to 0 but it does not download.
if I try to insert the promise inside createsheet then i am getting error:
Error [ERR_UNHANDLED_ERROR]: Unhandled error. ({ message: 'queue closed', code: 'QUEUECLOSED', data: undefined })
code:
const createSheet = workbook => {
let promise = [];
/**
* get count of all the user list
*/
promise.push(Dashboard.DDPro());
Promise.all(promise)
.then(results => {
results.forEach((result, index) => {
console.log(result);
const worksheet = workbook.addWorksheet(result.title, {
properties: { outlineLevelCol: 1 }
});
worksheet.columns = Object.keys(result.data[0]).map(item => {
return { header: item, key: item };
});
result.data.forEach(row => worksheet.addRow(row).commit);
});
})
.catch(err => console.log(err));
};
can any body helps me solve the problem.

Problem to use a Map in Firebase Functions

I am trying to get the length of a Map and I keep getting "undefined". Could please someone tell me what am I doing wrong?
This is the part of the code that gives me problems.
const GYMdetail: { [key: string]: number} = {};
GYMdetail[`${doc.data().name} (${doc.data().personalID})`] = 650;
const subtotal = 650 * GYMdetail.size;
This is the complete function code
export const addGymMonthlyExpense =
functions.https.onRequest((request, response) => {
const query1 = admin.firestore().collection("users");
const query = query1.where("subscriptions.gym.active", "==", true);
query.get()
.then(async (allUsers) => {
allUsers.docs.forEach(async (doc) => {
if (doc != undefined) {
const houseForGym = doc.data().subscriptions.gym.house;
await admin.firestore()
.doc(`houses/${houseForGym}/expenses/2022-04`)
.get().then((snapshot) => {
if (snapshot.data() == undefined) {
console.log(`${houseForGym}-${doc.data().name}: CREAR!!`);
} else if (snapshot.data()!.issued == false) {
let detail: { [key: string]: any} = {};
const GYMdetail: { [key: string]: number} = {};
detail = snapshot.data()!.detail;
GYMdetail[
`${doc.data().name} (${doc.data().personalID})`
] = 650;
const subtotal = 650 * GYMdetail.size;
detail["GYM"] = {"total": subtotal, "detail": GYMdetail};
snapshot.ref.set({"detail": detail}, {merge: true});
}
return null;
})
.catch((error) => {
console.log(
`${houseForGym} - ${doc.data().name}: ${error}`);
response.status(500).send(error);
return null;
});
}
});
response.send("i");
})
.catch((error) => {
console.log(error);
response.status(500).send(error);
});
});
Since you are executing an asynchronous call to the database in your code, you need to return a promise from the top-level code; otherwise Cloud Functions may kill the container when the final } executes and by that time the database load won't be done yet.
So:
export const addGymMonthlyExpense =
functions.https.onRequest((request, response) => {
const query1 = admin.firestore().collection("users");
const query = query1.where("subscriptions.gym.active", "==", true);
return query.get()
...
Next you'll need to ensure that all the nested get() calls also get a chance to finish before the Functions container gets terminated. For that I recommend not using await for each nested get call, but a single Promise.all for all of them:
query.get()
.then(async (allUsers) => {
const promises = [];
allUsers.docs.forEach((doc) => {
const houseForGym = doc.data().subscriptions.gym.house;
promises.push(admin.firestore()
.doc(`houses/${houseForGym}/expenses/2022-04`)
.get().then((snapshot) => {
...
});
});
response.send("i");
return Promise.all(promises);
})
.catch((error) => {
console.log(error);
response.status(500).send(error);
});

one of my friend is trying to automate a process in which the bot will post instagram stories as a video from a specific folder

below is the working code in which it can post images but is there any way i can also share videos as instagram story?
the error i get when i try to post video instead of image are:**
error image
PS D:\Softwares\programming\Insta Bot\story> node index.js
18:45:11 - info: Dry Run Activated
18:45:11 - info: Post() called! ======================
18:45:11 - debug: 1 files found in ./images/
18:45:11 - warn: Record file not found, saying yes to D:\Softwares\programming\Insta Bot\story\images\meme.mp4
18:45:11 - debug: Read File Success
18:45:11 - error: undefined
(MAIN CODE)
index.js
const logger = require("./logger.js")
const { random, sleep } = require('./utils')
require('dotenv').config();
const { IgApiClient, IgLoginTwoFactorRequiredError } = require("instagram-private-api");
const ig = new IgApiClient();
const Bluebird = require('bluebird');
const inquirer = require('inquirer');
const { CronJob } = require('cron');
const path = require("path");
const fs = require("fs");
const fsp = fs.promises;
const sharp = require("sharp");
//==================================================================================
const statePath = "./etc/state.conf";
const recordPath = "./etc/usedfiles.jsonl";
const imgFolderPath = "./images/";
const dryrun = true;
const runOnStart = true;
//==================================================================================
(async () => { // FOR AWAIT
// LOGIN TO INSTAGRAM
if (!dryrun) {
await login();
logger.info("Log In Successful");
} else {
logger.info("Dry Run Activated");
}
// SCHEDULER
// logger.silly("I'm a schedule, and I'm running!! :)");
const job = new CronJob('38 43 * * * *', post, null, true); //https://crontab.guru/
if (!runOnStart) logger.info(`Next few posts scheduled for: \n${job.nextDates(3).join("\n")}\n`);
else post();
// MAIN POST COMMAND
async function post() {
logger.info("Post() called! ======================");
let postPromise = fsp.readdir(imgFolderPath)
.then(filenames => {
if (filenames.length < 1) throw new Error(`Folder ${imgFolderPath} is empty...`)
logger.debug(`${filenames.length} files found in ${imgFolderPath}`);
return filenames;
})
.then(filenames => filenames.map(file => path.resolve(imgFolderPath + file)))
.then(filenames => pickUnusedFileFrom(filenames, filenames.length))
.then(filename => {
if (!dryrun) registerFileUsed(filename)
return filename
})
.then(fsp.readFile)
.then(async buffer => {
logger.debug("Read File Success "); //TODO move this to previous then?
return sharp(buffer).jpeg().toBuffer()
.then(file => {
logger.debug("Sharp JPEG Success");
return file
})
})
.then(async file => {
if (!dryrun) {
// await sleep(random(1000, 60000)) //TODO is this necessary?
return ig.publish.story({ file })
.then(fb => logger.info("Posting successful!?"))
}
else return logger.info("Data not sent, dryrun = true")
})
.then(() => logger.info(`Next post scheduled for ${job.nextDates()}\n`))
.catch(logger.error)
}
})();
//=================================================================================
async function login() {
ig.state.generateDevice(process.env.IG_USERNAME);
// ig.state.proxyUrl = process.env.IG_PROXY;
//register callback?
ig.request.end$.subscribe(async () => {
const serialized = await ig.state.serialize();
delete serialized.constants; // this deletes the version info, so you'll always use the version provided by the library
await stateSave(serialized);
});
if (await stateExists()) {
// import state accepts both a string as well as an object
// the string should be a JSON object
const stateObj = await stateLoad();
await ig.state.deserialize(stateObj)
.catch(err => logger.debug("deserialize: " + err));
} else {
let standardLogin = async function() {
// login like normal
await ig.simulate.preLoginFlow();
logger.debug("preLoginFlow finished");
await ig.account.login(process.env.IG_USERNAME, process.env.IG_PASSWORD);
logger.info("Logged in as " + process.env.IG_USERNAME);
process.nextTick(async () => await ig.simulate.postLoginFlow());
logger.debug("postLoginFlow finished");
}
// Perform usual login
// If 2FA is enabled, IgLoginTwoFactorRequiredError will be thrown
return Bluebird.try(standardLogin)
.catch(
IgLoginTwoFactorRequiredError,
async err => {
logger.info("Two Factor Auth Required");
const {username, totp_two_factor_on, two_factor_identifier} = err.response.body.two_factor_info;
// decide which method to use
const verificationMethod = totp_two_factor_on ? '0' : '1'; // default to 1 for SMS
// At this point a code should have been sent
// Get the code
const { code } = await inquirer.prompt([
{
type: 'input',
name: 'code',
message: `Enter code received via ${verificationMethod === '1' ? 'SMS' : 'TOTP'}`,
},
]);
// Use the code to finish the login process
return ig.account.twoFactorLogin({
username,
verificationCode: code,
twoFactorIdentifier: two_factor_identifier,
verificationMethod, // '1' = SMS (default), '0' = TOTP (google auth for example)
trustThisDevice: '1', // Can be omitted as '1' is used by default
});
},
)
.catch(e => logger.error('An error occurred while processing two factor auth', e, e.stack));
}
return
//================================================================================
async function stateSave(data) {
// here you would save it to a file/database etc.
await fsp.mkdir(path.dirname(statePath), { recursive: true }).catch(logger.error);
return fsp.writeFile(statePath, JSON.stringify(data))
// .then(() => logger.info('state saved, daddy-o'))
.catch(err => logger.error("Write error" + err));
}
async function stateExists() {
return fsp.access(statePath, fs.constants.F_OK)
.then(() => {
logger.debug('Can access state info')
return true
})
.catch(() => {
logger.warn('Cannot access state info')
return false
});
}
async function stateLoad() {
// here you would load the data
return fsp.readFile(statePath, 'utf-8')
.then(data => JSON.parse(data))
.then(data => {
logger.info("State load successful");
return data
})
.catch(logger.error)
}
}
async function registerFileUsed( filepath ) {
let data = JSON.stringify({
path: filepath,
time: new Date().toISOString()
}) + '\n';
return fsp.appendFile(recordPath, data, { encoding: 'utf8', flag: 'a+' } )
.then(() => {
logger.debug("Writing filename to record file");
return filepath
})
}
function pickUnusedFileFrom( filenames, iMax = 1000) {
return new Promise((resolve, reject) => {
let checkFileUsed = async function ( filepath ) {
return fsp.readFile(recordPath, 'utf8')
.then(data => data.split('\n'))
.then(arr => arr.filter(Boolean))
.then(arr => arr.map(JSON.parse))
.then(arr => arr.some(entry => entry.path === filepath))
}
let trythis = function( iMax, i = 1) {
let file = random(filenames);
checkFileUsed(file)
.then(async used => {
if (!used) {
logger.info(`Unused file found! ${file}`);
resolve(file);
} else if (i < iMax) {
logger.debug(`Try #${i}: File ${file} used already`);
await sleep(50);
trythis(iMax, ++i)
} else {
reject(`I tried ${iMax} times and all the files I tried were previously used`)
}
})
.catch(err => {
logger.warn("Record file not found, saying yes to " + file);
resolve(file);
})
}( iMax );
})
}

Trouble with promise in firebase functions

This code either runs once or a max of 100 times. I have a dummy data file with 6000 records as this is the average that it will have to handle.
Currently using the Blaze plan.
The code was working somewhat, I set up a new project and now I get this issue.
export const uploadPatrons = functions.storage
.object()
.onFinalize((object, context) => {
let patronPromise: any[];
patronPromise = [];
if (object.name === 'patrons/upload.csv') {
admin
.storage()
.bucket()
.file('/patrons/upload.csv')
.download({})
.then(data => {
Papa.parse(data.toString(), {
header: true,
skipEmptyLines: true,
complete: result => {
result.data.forEach(x => {
x.inside = false;
x.arrived = false;
x.img = false;
x.arrivedTime = null;
const newPromise = admin
.firestore()
.collection('patrons')
.add({ ...x })
.then(doc => {
console.log(doc);
})
.catch(err => {
console.log(err);
});
patronPromise.push(newPromise);
});
}
});
})
.catch(err => {
console.log(err);
});
}
return Promise.all(patronPromise)
.catch(err => {
console.log(err);
});
});
All it has to do is read the file from the storage, parse it and add each record to the firebase collection
Function returned undefined, expected Promise or value
This is the error I get in the logs
Because of your first promise may be shutdown even it not finish. So try to follow the rule promise/always-return
export const uploadPatrons = functions.storage
.object()
.onFinalize((object, context) => {
if (object.name === 'patrons/upload.csv') {
return admin.storage().bucket()
.file('/patrons/upload.csv')
.download({})
.then(data => {
let patronPromise: any[];
patronPromise = [];
Papa.parse(data.toString(), {
header: true,
skipEmptyLines: true,
complete: result => {
result.data.forEach(x => {
x.inside = false;
x.arrived = false;
x.img = false;
x.arrivedTime = null;
const newPromise = admin.firestore()
.collection('patrons')
.add({
...x
})
patronPromise.push(newPromise);
});
}
});
return Promise.all(patronPromise)
})
.then(result=>{
//return Promise.resolve or something
})
.catch(err=>{
console.log(err)
})
}
else{
//also return if it's nothing
}
});
You're ignoring the promise that admin.storage().bucket().file('/patrons/upload.csv').download({}) returns, which means that the function may get aborted.
I think it should be closer to this:
export const uploadPatrons = functions.storage
.object()
.onFinalize((object, context) => {
let patronPromise: any[];
patronPromise = [];
if (object.name === 'patrons/upload.csv') {
return admin.storage().bucket()
.file('/patrons/upload.csv')
.download({})
.then(data => {
Papa.parse(data.toString(), {
header: true,
skipEmptyLines: true,
complete: result => {
result.data.forEach(x => {
x.inside = false;
x.arrived = false;
x.img = false;
x.arrivedTime = null;
const newPromise = admin.firestore()
.collection('patrons')
.add({
...x
})
patronPromise.push(newPromise);
});
// TODO: return the Promise.all(patronPromise) here
}
});
})
}
});

Getting infinite loop in firebase cloud function

I am using firestore to store the data in firebase. To get the count i am using cloud function. When i try to add / update / delete an entry in one collection it starts the infinite loop with another collection.
Example:
I am having a user table and agent table when i add/update/delete a user it should get updated in the agent table.
Though i have used separate functions for users and agent still i am getting an infinite loop.can anyone tell me how to resolve it
Query to update the user in user and agent table:
export const addUser = (values) =>
db
.collection('users')
.add(values)
.then((docRef) => {
let customer = { customer: {} };
customer.customer[docRef.id] = {
id: docRef.id,
name: values.name,
commission: values.agent.commission
};
let agentId = values.agent.id;
db.collection('agents')
.doc(agentId)
.set(customer, { merge: true });
});
Cloud function for user:
const functions = require("firebase-functions");
const admin = require("firebase-admin");
exports = module.exports = functions.firestore
.document("users/{userUid}")
.onWrite(
(change, context) =>
new Promise((resolve, reject) => {
let dashboardId;
getDashboardId();
})
);
getDashboardId = () => {
admin.firestore().collection('dashboard').get().then((snapshot) => {
if (snapshot.size < 1) {
dashboardId = admin.firestore().collection('dashboard').doc().id;
} else {
snapshot.docs.forEach((doc) => {
dashboardId = doc.id;
});
}
return updateUser(dashboardId);
}).catch((error) => {
console.log('error is', error);
});
}
updateUser = (id) => {
admin.firestore().collection('users').where('isDeleted', '==', false).get().then((snap) => {
let usersData = {users: snap.size};
return admin.firestore().collection('dashboard').doc(id).set(usersData, {merge: true});
}).catch((error) => {
console.log('error is', error);
});
}
Cloud function for agent:
const functions = require("firebase-functions");
const admin = require("firebase-admin");
exports = module.exports = functions.firestore
.document("agents/{agentUid}")
.onWrite(
(change, context) =>
new Promise((resolve, reject) => {
let dashboardId;
getDashboardId();
})
);
getDashboardId = () => {
admin.firestore().collection('dashboard').get().then((snapshot) => {
if (snapshot.size < 1) {
dashboardId = admin.firestore().collection('dashboard').doc().id;
} else {
snapshot.docs.forEach((doc) => {
dashboardId = doc.id;
});
}
return updateAgent(dashboardId);
}).catch((error) => {
console.log('error is', error);
});
}
updateAgent = (id) => {
admin.firestore().collection('agents').where('isDeleted', '==', false).get().then((snap) => {
let agentsData = {agents: snap.size};
return admin.firestore().collection('dashboard').doc(id).set(agentsData, {merge: true});
}).catch((error) => {
console.log('error is', error);
});
}

Resources