Update large number of documents - node.js

Question

I am trying to update the content of large number of documents in firebase, I have tried the following:
First, reading all documents on client side and looping over the documents and updating them by refence.
the problem here is that I am doing intensive operation on the client side and that would be unpredictable, therefore I switched to Firebase's Functions.
Second, reading all documents in firebase functions and then updating them using bulkwriter
here's the code:
exports.testingFunction1 = functions.runWith({
timeoutSeconds: 540,
memory: "8GB",
}).https.onCall(async (data, context) => {
const storeId = data.text;
if (!(typeof storeId === 'string') || storeId.length === 0) {
throw new functions.https.HttpsError('invalid-argument', 'The function must be called with ' +
'one arguments containing the storeId.');
}
if (!context.auth) {
throw new functions.https.HttpsError('failed-precondition', 'The function must be called ' +
'while authenticated.');
}
const uid = context.auth.uid;
const name = context.auth.token.name || null;
const picture = context.auth.token.picture || null;
const email = context.auth.token.email || null;
let bulk1 = new admin.firestore().bulkWriter();
let products = await admin.firestore().collection("Products").get(); //here's the problems's source
products.forEach((document) => {
bulk1.update(document.ref, { "IsStorePublished": true });
});
await bulk1.flush().then(() => {
return { "result": "Success!" };
})
.catch((error) => {
throw new functions.https.HttpsError('unknown', error.message, error);
});
return { "Result": "Success" }
});
the problem here appears when I try to read more that about 8000 documents at a single time, I get the following error although I have changed the memory limitations for the function to the max possible:
FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap
out of memory
Is there a good way to achieve this task ?

Answer 1

For anyone interested, I have solved the issue as following:
the problem with reading a large amount of data to update it in firebase functions is that the memory is filling up, so I have made a recursive function enables reading 500 items at a time and apply the "BulkWrite" to only 500 documents at the time, and to achieve this I used the "startAfter" reading method.
This is the main firebase v1 function:
where it reads the first 500 items and apply the function "operationToDo" to them, and then calls the recursive function to continue the process.
exports.testingFunction1 = functions.runWith({
timeoutSeconds: 540,
memory: "8GB",
}).https.onCall(async (data, context) => {
const storeId = data.text;
if (!(typeof storeId === 'string') || storeId.length === 0) {
throw new functions.https.HttpsError('invalid-argument', 'The function must be called with ' +
'one arguments "storeId" containing the storeId.');
}
if (!context.auth) {
throw new functions.https.HttpsError('failed-precondition', 'The function must be called ' +
'while authenticated.');
}
const uid = context.auth.uid;
const name = context.auth.token.name || null;
const picture = context.auth.token.picture || null;
const email = context.auth.token.email || null;
let first = admin.firestore()
.collection("Products")
.orderBy("Name").where("Store", '==', storeId)
.limit(500);
await first.get().then(
async (documentSnapshots) => {
if (documentSnapshots.docs.length == 0) {
return;
} else {
await operationToDo(documentSnapshots, "update", { "IsStorePublished": false })
}
let lastVisible =
documentSnapshots.docs[documentSnapshots.size - 1];
await recursivePublishingTheStore(lastVisible, storeId);
},
);
return { "Result": "Success" }
});
The recursive function:
async function recursivePublishingTheStore(lastVisible, storeId) {
let next = admin.firestore()
.collection("Products")
.orderBy("Name").where("Store", '==', storeId)
.startAfter(lastVisible)
.limit(500);
await next.get().then(
async (documentSnapshots) => {
if (documentSnapshots.docs.length == 0) {
return;
} else {
await operationToDo(documentSnapshots, "update", { "IsStorePublished": false })
let lastVisible =
documentSnapshots.docs[documentSnapshots.size - 1];
await recursivePublishingTheStore(lastVisible, storeId);
}
}
);
}
The operation can be anything but in my case it would be "update":
async function operationToDo(documents, operation, value) {
let bulk1 = new admin.firestore().bulkWriter();
documents.forEach((document) => {
if (operation == 'update')
bulk1.update(document.ref, value);
});
await bulk1.flush().then(() => {
})
.catch((error) => {
throw new functions.https.HttpsError('unknown', error.message, error);
});
}
The performance of the code above is pretty good, for updating about 15k documents it would take about 2 minutes.
Note: I have chosen the number 500 randomly, different number might work and might perform better, and I will be experimenting with it this week.