exports.resetDailyFinalKills = functions.pubsub
.schedule("58 16 * * *")
.onRun(async (context) => {
const players = firestore.collection("players");
const goodtimes = await players.where("final_kills", ">", 0);
goodtimes.forEach((snapshot) => {
snapshot.ref.update({final_kills: 0});
});
return null;
});
So I have this cloud function, and when I force run it nothing happens at all like it just says the function was successful but the final_kills field never gets updated. Can anyone help?
Like I obviously have a player here which has a final_kills value that is greater than 0. So why doesn't this reset that back down to zero?
Note sure if I am missing something here but:
You actually try to iterate over the Query object firebase creates when using the where() function on your collections. You actually never fetch the data from the database.
const players = firestore.collection("players");
// fetch the objects from firestore
const goodtimes = await players.where("final_kills", ">", 0).get();
// iterate over the docs you receive
goodtimes.docs.forEach((snapshot) => {
snapshot.ref.update({ final_kills: 0 });
});
Edit (regarding your comment):
Make sure you set your timezone properly after your .schedule() function:
// timezone in my case
functions.pubsub.schedule('5 11 * * *').timeZone('Europe/Berlin')
Check this list as a reference for your correct timezone.
I have an application that makes API calls to another system, and it queues these API calls in a queue using Bull and Redis.
However, occasionally it gets bogged down with lots of API calls, or something stops working properly, and I want an easy way for users to check if the system is just "busy". (Otherwise, if they perform some action, and 10 minutes later it hasn't completed, they'll keep trying it again, and then we get a backlog of more entries (and in some cases data issues where they've issued duplicate parts, etc.)
Here's what a single "key" looks like for a successful API call in the queue:
HSET "bull:webApi:4822" "timestamp" "1639085540683"
HSET "bull:webApi:4822" "returnvalue" "{"id":"e1df8bb4-fb6c-41ad-ba62-774fe64b7882","workOrderNumber":"WO309967","status":"success"}"
HSET "bull:webApi:4822" "processedOn" "1639085623027"
HSET "bull:webApi:4822" "data" "{"id":"e1df8bb4-fb6c-41ad-ba62-774fe64b7882","token":"eyJ0eXAiOiJKV1QiL....dQVyEpXt64Fznudfg","workOrder":{"members":{"lShopFloorLoad":true,"origStartDate":"2021-12-09T00:00:00","origRequiredQty":2,"requiredQty":2,"requiredDate":"2021-12-09T00:00:00","origRequiredDate":"2021-12-09T00:00:00","statusCode":"Released","imaItemName":"Solid Pin - Black","startDate":"2021-12-09T00:00:00","reference":"HS790022053","itemId":"13840402"}},"socketId":"3b9gejTZjAXsnEITAAvB","type":"Create WO"}"
HSET "bull:webApi:4822" "delay" "0"
HSET "bull:webApi:4822" "priority" "0"
HSET "bull:webApi:4822" "name" "__default__"
HSET "bull:webApi:4822" "opts" "{"lifo":true,"attempts":1,"delay":0,"timestamp":1639085540683}"
HSET "bull:webApi:4822" "finishedOn" "1639085623934"
You can see in this case it took 83 seconds to process. (1639085540 - 1639085623)
I'd like to be able to provide summary metrics like:
Most recent API call was added to queue X seconds ago
Most recent successful API call completed X seconds ago and took XX seconds to
complete.
I'd also like to be able to provide a list of the 50 most recent API calls, formatted in a nice way and tagged with "success", "pending", or "failed".
I'm fairly new to Redis and Bull, and I'm trying to figure out how to query this data (using Redis in Node.js) and return this data as JSON to the application.
I can pull a list of keys like this:
// #route GET /status
async function status(req, res) {
const client = createClient({
url: `redis://${REDIS_SERVER}:6379`
});
try {
client.on('error', (err) => console.log('Redis Client Error', err));
await client.connect();
const value = await client.keys('*');
res.json(value)
} catch (error) {
console.log('ERROR getting status: ', error.message, new Date())
res.status(500).json({ message: error.message })
} finally {
client.quit()
}
}
Which will return ["bull:webApi:3","bull:webApi:1","bull:webApi:2"...]
But how can I pull the values associated to the respective keys?
And how can I find the key with the highest number, and then pull the details for the "last 50". In SQL, it would be like doing a ORDER BY key_number DESC LIMIT 50 - but I'm not sure how to do it in Redis.
I'm a bit late here, but if you're not set on manually digging around in Redis, I think Bull's API, in particular Queue#getJobs(), has everything you need here, and should be much easier to work with. Generally, you shouldn't have to reach into Redis to do any common tasks like this, that's what Bull is for!
If I understand you goal correctly, you should be able to do something like:
const Queue = require('bull')
async function status (req, res) {
const { listNum = 10 } = req.params
const api_queue = new Queue('webApi', `redis://${REDIS_SERVER}:6379`)
const current_timestamp_sec = new Date().getTime() / 1000 // convert to seconds
const recent_jobs = await api_queue.getJobs(null, 0, listNum)
const results = recent_jobs.map(job => {
const processed_on_sec = job.processedOn / 1000
const finished_on_sec = job.finishedOn / 1000
return {
request_data: job.data,
return_data: job.returnvalue,
processedOn: processed_on_sec,
finishedOn: finished_on_sec,
duration: finished_on_sec - processed_on_sec,
elapsedSinceStart: current_timestamp_sec - processed_on_sec,
elapsedSinceFinished: current_timestamp_sec - finished_on_sec
}
})
res.json(results)
}
That will get you the most recent numList* jobs in your queue. I haven't tested this full code, and I'll leave the error handling and adding of your custom fields to the job data up to you, but the core of it is solid and I think that should meet your needs without ever having to think about how Bull stores things in Redis.
I also included a suggestion on how to deal with timestamps a bit more nicely, you don't need to do string processing to convert milliseconds to seconds. If you need them to be integers you can wrap them in Math.floor().
* at least that many, anyway - see the second note below
A couple notes:
The first argument of getJobs() is a list of statuses, so if you want to look at just completed jobs, you can pass ['completed'], or completed and active, do ['completed', 'active'], etc. If no list is provided (null) it defaults to all statuses.
As mentioned in the reference I linked, the limit here is per state - so you'll likely get more than listNum jobs back. It doesn't seem like that should be a problem for your use case, but if it is, you can sort the list returned (probably by job id) and just return the first listNum - you're guaranteed to get at least that many (assuming there are that many jobs in your queue), and won't get more than 6*listNum (since there are 6 states).
Folks new to Bull can get nervous about instantiating a Queue object to do stuff like this - but don't be! By itself a Queue instance doesn't do anything, it's just an interface to the given queue. It won't start processing jobs until you call process() to add a processor. This is, incidentally, also how you'd add jobs from a separate process than you run your queues in, but of course nothing will be added unless you call add().
So I've figured out how to pull the data I need. I'm not saying it's a good method, and I'm open to suggestions; but it seems to work to provide a filtered JSON return with the needed data, without changing how the queue functions.
Here's what it looks like:
// #route GET /status/:listNum
async function status(req, res) {
const { listNum = 10} = req.params
const client = createClient({
url: `redis://${REDIS_SERVER}:6379`
});
try {
client.on('error', (err) => console.log('Redis Client Error', err));
await client.connect();
// Find size of queue database
const total_keys = await client.sendCommand(['DBSIZE']);
const upper_value = total_keys;
const lower_value = total_keys - listNum;
// Generate array
const range = (start, stop) => Array.from({ length: (start - stop) + 1}, (_, i) => start - (i));
var queue_ids = range(upper_value, lower_value)
queue_ids = queue_ids.filter(function(x){ return x > 0 }); // Filer out anything less than zero
// Current timestamp in seconds
const current_timestamp = parseInt(String(new Date().getTime()).slice(0, -3)); // remove microseconds ("now")
var response = []; // Initialize array
for(id of queue_ids){ // Loop through queries
// Query value
var value = await client.HGETALL('bull:webApi:'+id);
if(Object.keys(value).length !== 0){ // if returned a value
// Grab most of the request (exclude the token & socketId to save space, not used)
var request_data = JSON.parse(value.data)
request_data.token = '';
request_data.socketId = '';
// Grab & calculate desired times
const processedOn = value.processedOn.slice(0, -3); // remove microseconds ("start")
const finishedOn = value.finishedOn.slice(0, -3); // remove microseconds ("done")
const duration = finishedOn - processedOn; // (seconds)
const elapsedSinceStart = current_timestamp - processedOn;
const elapsedSinceFinished = current_timestamp - finishedOn;
// Grab the returnValue
const return_data = value.returnValue;
// ignoring queue keys of: opts, priority, delay, name, timestamp
const object_data = {request_data: request_data, processedOn: processedOn, finishedOn: finishedOn, return_data: return_data, duration: duration, elapsedSinceStart: elapsedSinceStart, elapsedSinceFinished: elapsedSinceFinished }
response.push(object_data);
}
}
res.json(response);
} catch (error) {
console.log('ERROR getting status: ', error.message, new Date());
res.status(500).json({ message: error.message });
} finally {
client.quit();
}
}
It's looping the Redis query, so I wouldn't want to use this for hundreds of keys, but for 10 or even 50 I'm thinking it should work.
For now I've resorted to getting the total number of keys and working backwards:
await client.sendCommand(['DBSIZE']);
In my case it will return a total number slightly higher than the highest key id (~ a handful of status keys), but at least gets close, and then I just filter out any non-responses.
I've looked at ZRANGE a bit, but I can't figure out how to get it to give me the last id. When I have a Redis database (Bull Queue) like this:
If there's a simple Redis command I can run that will return "3", I'd probably use that instead. (since bull:webApi:3 has the highest number)
(In actual use case, this might be 9555 or some high number; I just want to get the highest numbered key that exists.)
For now I'll try using the method I've come up with above.
Per Firebase cloud functions docs, "Events are delivered at least once, but a single event may result in multiple function invocations. Avoid depending on exactly-once mechanics, and write idempotent functions."
When looking at a firestore cloud function doc example below of a restaurant rating, they are using an increment counter to calculate the total number of ratings. What are some of the best ways to maintain the accuracy of this counter by using an idempotent function?
Is it reasonable to store the context.eventId in a subcollection document field and only execute the function if the new context.eventId is different?
function addRating(restaurantRef, rating) {
// Create a reference for a new rating, for use inside the transaction
var ratingRef = restaurantRef.collection('ratings').doc();
// In a transaction, add the new rating and update the aggregate totals
return db.runTransaction((transaction) => {
return transaction.get(restaurantRef).then((res) => {
if (!res.exists) {
throw "Document does not exist!";
}
// Compute new number of ratings
var newNumRatings = res.data().numRatings + 1;
// Compute new average rating
var oldRatingTotal = res.data().avgRating * res.data().numRatings;
var newAvgRating = (oldRatingTotal + rating) / newNumRatings;
// Commit to Firestore
transaction.update(restaurantRef, {
numRatings: newNumRatings,
avgRating: newAvgRating
});
transaction.set(ratingRef, { rating: rating });
});
});
}
Is it reasonable to store the context.eventId in a subcollection
document field and only execute the function if the new
context.eventId is different?
Yes, for your use case, using the Cloud Function eventId is the best solution to make you Cloud Function idempotent. I guess you have already watched this Firebase video.
In the Firebase doc from which you have taken the code in your question, you will find at the bottom, the similar code for a Cloud Function. I've adapted this code as follows, in order to check if a doc with ID = eventId exists in a dedicated ratingUpdateIds subcollection:
exports.aggregateRatings = functions.firestore
.document('restaurants/{restId}/ratings/{ratingId}')
.onWrite(async (change, context) => {
try {
// Get value of the newly added rating
const ratingVal = change.after.data().rating;
const ratingUpdateId = context.eventId;
// Get a reference to the restaurant
const restRef = db.collection('restaurants').doc(context.params.restId);
// Get a reference to the ratingUpdateId doc
const ratingUpdateIdRef = restRef.collection("ratingUpdateIds").doc(ratingUpdateId);
// Update aggregations in a transaction
await db.runTransaction(async (transaction) => {
const ratingUpdateIdDoc = await transaction.get(ratingUpdateIdRef);
if (ratingUpdateIdDoc.exists) {
// The CF is retried
throw "The CF is being retried";
}
const restDoc = await transaction.get(restRef);
// Compute new number of ratings
const newNumRatings = restDoc.data().numRatings + 1;
// Compute new average rating
const oldRatingTotal = restDoc.data().avgRating * restDoc.data().numRatings;
const newAvgRating = (oldRatingTotal + ratingVal) / newNumRatings;
// Update restaurant info and set ratingUpdateIdDoc
transaction
.update(restRef, {
avgRating: newAvgRating,
numRatings: newNumRatings
})
.set(ratingUpdateIdRef, { ratingUpdateId })
});
return null;
} catch (error) {
console.log(error);
return null;
}
});
PS: I made the assumption that the Cloud Function eventId can be used as a Firestore document ID. I didn't find any doc or info stating the opposite. In case using the eventId as an ID would be a problem, since you execute the transaction in a Cloud Function (and therefore use the Admin SDK), you could query the document based on a field value (where you store the eventId) instead of getting it through a Reference based on its ID.
I am trying to update a field in my document in Firestore. The general location of the document would be "/games/{userId}/userGames/{gameId}. And in this game, there is a property called "status" which changes accordingly to the games start and end time.
As you can guess, the if the start time is bigger than the "now" timestamp and the status is "TO_BE_PLAYED", the game will begin and the status will be 1, "BEING_PLAYED". Also, if the end time is bigger than the "now" timestamp and the status is "BEING_PLAYED", the game will end, therefore the status will be 2, "PLAYED". I want to create a cloud function that is capable to do so.
However, even if the function logs output 'ok', the values are never updated. Unfortunately, I do not have that much experience in Javascript too.
THE CODE
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
const STATUS_PLAYED = 2;
const STATUS_BEING_PLAYED = 1;
const STATUS_TO_BE_PLAYED = 0;
exports.handleBeingPlayedGames = functions.runWith({memory: "2GB"}).pubsub.schedule('* * * * *')
.timeZone('Europe/Istanbul') // Users can choose timezone - default is America/Los_Angeles
.onRun(async () => {
// current time & stable
// was Timestamp.now();
const now = admin.firestore.Timestamp.fromDate( new Date());
const querySnapshot = await db.collection("games").get();
const promises = [];
querySnapshot.forEach( doc => {
const docRef = doc.ref;
console.log(docRef);
promises.push(docRef.collection("userGames").where("status", "==", STATUS_BEING_PLAYED).where("endtime", "<", now).get());
});
const snapshotArrays = await Promise.all(promises);
const promises1 = [];
snapshotArrays.forEach( snapArray => {
snapArray.forEach(snap => {
promises1.push(snap.ref.update({
"status": STATUS_PLAYED,
}));
});
});
return Promise.all(promises1);
});
exports.handleToBePlayedGames = functions.runWith({memory: "2GB"}).pubsub.schedule('* * * * *')
.onRun(async () => {
// current time & stable
// was Timestamp.now();
const now = admin.firestore.Timestamp.fromDate(new Date());
const querySnapshot = await db.collection("games").get();
const promises = [];
querySnapshot.forEach( async doc => {
const docData = await doc.ref.collection("userGames").where("status", "==", STATUS_TO_BE_PLAYED).where("startTime", ">", now).get();
promises.push(docData);
});
const snapshotArrays = await Promise.all(promises);
const promises1 = [];
snapshotArrays.forEach( snapArray => {
snapArray.forEach(snap => {
promises1.push(snap.ref.update({
"status": STATUS_BEING_PLAYED,
}));
});
});
return Promise.all(promises1);
});
Okay, so this answer goes to lurkers trying to solve this problem.
First I tried to solve this problem by brute force and not including much thinking and tried to acquire the value in subcollection. However, as I searched, I've found that denormalizing (flattening) data actually solves the problem a bit.
I created a new directory under /status/{gameId} with the properties
endTime, startTime, and status field and I actually did it on a single level by using promises. Sometimes denormalizing data can be your savior.
How can startTime be greater than now? Is it set by default to a date in the future?
My current assumption is that a game cannot set it's status to STATUS_BEING_PLAYED because of the inconsistency with startTime. Moreover, a game cannot have the status STATUS_PLAYED because it depends on having STATUS_BEING_PLAYED, which cannot have.
My recommendation would be to set the field startTime and endTime to null by default. If you do so you can check if a game has to be set to STATUS_BEING_PLAYED with this:
doc.ref.collection("userGames")
.where("status", "==", STATUS_TO_BE_PLAYED)
.where("startTime", "<", now)
.where("endTime", "==", null)
.get();
You could check if a game has to be on STATUS_PLAYED with this (exactly as you did):
docRef.collection("userGames")
.where("status", "==", STATUS_BEING_PLAYED)
.where("endtime", "<", now)
.get();
Now there's something that you should wonder, is this the best approach to change a game's status? You are querying the whole game library of a user every single minute as you know read operations are charged so this approach would imply meaningful charges. Maybe you should simply use update the game's status when the game is started and closed.
Also notice that the equals operation is ==, not =.
My goal is to delete all the message nodes 24 hours after they were sent using Firebase Cloud Functions and the Realtime Database. I tried copy and pasting the answer from this post however for some reason the messages delete directly after they were created rather than the 24 hours later. If someone could help me solve this problem I would really appreciate it. I have tried multiple different answers based on the same issue and they haven't worked for me.
Here is my index.js file:
'use strict';
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
// Cut off time. Child nodes older than this will be deleted.
const CUT_OFF_TIME = 24 * 60 * 60 * 1000; // 2 Hours in milliseconds.
exports.deleteOldMessages = functions.database.ref('/Message/{chatRoomId}').onWrite(async (change) => {
const ref = change.after.ref.parent; // reference to the parent
const now = Date.now();
const cutoff = now - CUT_OFF_TIME;
const oldItemsQuery = ref.orderByChild('seconds').endAt(cutoff);
const snapshot = await oldItemsQuery.once('value');
// create a map with all children that need to be removed
const updates = {};
snapshot.forEach(child => {
updates[child.key] = null;
});
// execute all updates in one go and return the result to end the function
return ref.update(updates);
});
And my database structure is:
In the comments you indicated that you're using Swift. From that and the screenshot it turns out that you're storing the timestamp in seconds since 1970, while the code in your Cloud Functions assumes it is in milliseconds.
The simplest fix is:
// Cut off time. Child nodes older than this will be deleted.
const CUT_OFF_TIME = 24 * 60 * 60 * 1000; // 2 Hours in milliseconds.
exports.deleteOldMessages = functions.database.ref('/Message/{chatRoomId}').onWrite(async (change) => {
const ref = change.after.ref.parent; // reference to the parent
const now = Date.now();
const cutoff = (now - CUT_OFF_TIME) / 1000; // convert to seconds
const oldItemsQuery = ref.orderByChild('seconds').endAt(cutoff);
const snapshot = await oldItemsQuery.once('value');
// create a map with all children that need to be removed
const updates = {};
snapshot.forEach(child => {
updates[child.key] = null;
});
// execute all updates in one go and return the result to end the function
return ref.update(updates);
});
Also see my answer here: How to remove a child node after a certain date is passed in Firebase cloud functions?