ESOCKETTIMEDOUT Cloud Functions for Firebase - node.js

I am using Cloud Functions for Firebase together with my Firebase Realtime Database in order to do some data management for my app.
One of my functions though seems to get terminated since it takes about 100-150 seconds to complete. This happens with error : ESOCKETTIMEDOUT.
Is there a way to prevent this?
Here is my function:
function getTopCarsForUserWithPreferences(userId, genres) {
const pathToCars = admin.database().ref('cars');
pathTocars.orderByChild("IsTop").equalTo(true).once("value").then(function(snapshot) {
return writeSuggestedCars(userId, genres, snapshot);
}).catch(reason => {
console.log(reason)
})
}
function writeSuggestedCars(userId, genres, snapshot) {
const carsToWrite = {};
var snapCount = 0
snapshot.forEach(function(carSnapshot) {
snapCount += 1
const carDict = carSnapshot.val();
const carGenres = carDict.taCarGenre;
const genre_one = genres[0];
const genre_two = genres[1];
if (carGenres[genre_one] === true ||carGenres[genre_two] == true) {
carsToWrite[carSnapshot.key] = carDict
}
if (snapshot.numChildren() - 1 == snapCount) {
const pathToSuggest = admin.database().ref('carsSuggested').child(userId);
pathToSuggest.set(carsToWrite).then(snap => {
}).catch(reason => {
console.log(reason)
});
}
});
}
The getTopCarsForUserWithPreferences gets called when a user adds preferences. Also the cars table has about 50k entries.

Well you need to return everytime you use a async task.
Edit: you return 'writeSuggestedCars' but I think it never returns a value. I do not have a compiler, but I thought it was return Promise.resolved(). Can you insert it where I putted 'HERE'?
Maybe this will work:
function getTopCarsForUserWithPreferences(userId, genres) {
const pathToCars = admin.database().ref('cars');
return pathTocars.orderByChild("IsTop").equalTo(true).once("value").then(function(snapshot) {
return writeSuggestedCars(userId, genres, snapshot);
}).catch(reason => {
console.log(reason)
})
}
function writeSuggestedCars(userId, genres, snapshot) {
const carsToWrite = {};
var snapCount = 0
snapshot.forEach(function(carSnapshot) {
snapCount += 1
const carDict = carSnapshot.val();
const carGenres = carDict.taCarGenre;
const genre_one = genres[0];
const genre_two = genres[1];
if (carGenres[genre_one] === true ||carGenres[genre_two] == true) {
carsToWrite[carSnapshot.key] = carDict
}
if (snapshot.numChildren() - 1 == snapCount) {
const pathToSuggest = admin.database().ref('carsSuggested').child(userId);
return pathToSuggest.set(carsToWrite).then(snap => {
// 'HERE' I think return promise/Promise.resolve() will work
}).catch(reason => {
console.log(reason)
});
}
});
}

Related

How can I efficiently filter multiple values in mongodb

I'm trying to filter some data by themes and am puzzled as to how I can go about doing it. Say I have two items with the themes ['People', 'Technology', 'Culture'] and ['Economy', 'Technology', 'Culture'], if the query is Technology, I am able to see both of these items appearing. But if the query is Technology and Culture, I'm not able to see either of them because ["Technology", "Culture"] =/= ['People', 'Technology', 'Culture'] and vice versa. My code searches for the exact list, not it's components, if the query was Technology and Culture then I want both of those items to show up since it is inside that list.
I'm not sure how to do this, I'm using MERN stack and here is my backend code:
const Project = require('../models/projectModel')
const mongoose = require('mongoose')
// get all projects
const getProjects = async (req, res) => {
const projects = await Project.find().sort({ createdAt: -1 })
// const test = await Project.find({assignment_type: 1 || 2}).sort({ createdAt: -1 })
// console.log(test)
res.status(200).json(projects)
}
// get filtered project
const getFilteredProjects = async (req, res) => {
var request = {}
console.log(req.query.sdg)
console.log('t' + req.query.theme)
console.log('a' + req.query.assignment_type)
// var t = ["Economical", "Technological"]
// const test = await Project.find({theme: ("Technological" && "Economical")}).sort({ createdAt: -1 })
// const test = await Project.find({
// $and:
// }).sort({ createdAt: -1 })
// console.log(test)
// Function to separate commas from string
function separateCommas(str) {
let t = []
for (let i = 0; i < str.length; i++) {
if (str[i] === ',') {
console.log(i)
t.push(i)
}
}
let themeArray = []
if (t.length === 1) {
let theme1 = str.slice(0, t[0])
let theme2 = str.slice(t[0]+1)
themeArray.push(theme1)
themeArray.push(theme2)
}
if (t.length === 2) {
let theme1 = str.slice(0, t[0])
let theme2 = str.slice(t[0]+1, t[1])
let theme3 = str.slice(t[1]+1)
themeArray.push(theme1)
themeArray.push(theme2)
themeArray.push(theme3)
}
request["theme"] = themeArray.sort()
}
// See if sdg selected
if (req.query.sdg !== '') {
request["sdg"] = req.query.sdg
}
// See if assignment type selected
if (req.query.assignment_type !== '') {
request["assignment_type"] = req.query.assignment_type
}
// See if theme selected
if (req.query.theme !== '') {
if (req.query.theme.length > 14) {
separateCommas(req.query.theme)
}
else {
request["theme"] = req.query.theme
}
}
console.log(request)
const projects = await Project.find(request).sort({ createdAt: -1 })
res.status(200).json(projects)
}
module.exports = {
getProjects,
getProject,
createProject,
deleteProject,
updateProject,
getFilteredProjects
}
This is how my backend code receives the data from the database, it sends it in this format where there can be multiple theme's:
{
sdg: 'SDG 2: Zero Hunger',
assignment_type: 'Discussion Topics',
theme: 'Economy'
}

async function doesn't wait of inside await in nodejs

I am implementing function monthlyRevenue.
Simply, it will return total monthly revenue,and it takes arguments of station array which will make revenues, month and year.
Problem
Inside of this function I have getStationPortion which will fetch the revenue portion of user's.
So I would like to make it return object like this.
stationsPortion = {station1 : 30, station2 : 20}
In the monthlyRevenue
const stationPortions = await getStationPortions(stations)
console.log("portion map", stationPortions //it will be shown very beginning with empty
getStationPortions
const getStationPortions = async (stations) => {
let stationPortions = {}
stations.map(async (value) => {
const doc = await fdb.collection('Stations').doc(value).get()
if (!doc.exists) {
console.log("NO DOC")
} else {
stationPortions[value] = doc.data().salesPortion
console.log(stationPortions) //it will be shown at the last.
}
})
return stationPortions
}
I thought that async function should wait for the result, but it does not.
I am kind of confusing if my understanding is wrong.
Thank you
(by the way, fdb is firebase admin(firestore)
Working code
const getStationPortions = async (stations) => {
let stationPortions = {}
await Promise.all(stations.map(async (value) => {
const doc = await fdb.collection('Stations').doc(value).get()
if (!doc.exists) {
console.log("NO DOC")
} else {
stationPortions[value] = doc.data().salesPortion
console.log(stationPortions)
}
}))
return stationPortions
}
module.exports = router;

How to make Mongoose update work with await?

I'm creating a NodeJS backend where a process reads in data from a source, checks for changes compared to the current data, makes those updates to MongoDB and reports the changes made. Everything works, except I can't get the changes reported, because I can't get the Mongoose update action to await.
The returned array from this function is then displayed by a Koa server. It shows an empty array, and in the server logs, the correct values appear after the server has returned the empty response.
I've digged through Mongoose docs and Stack Overflow questions – quite a few questions about the topic – but with no success. None of the solutions provided seem to help. I've isolated the issue to this part: if I remove the Mongoose part, everything works as expected.
const parseJSON = async xmlData => {
const changes = []
const games = await Game.find({})
const gameObjects = games.map(game => {
return new GameObject(game.name, game.id, game)
})
let jsonObj = require("../sample.json")
Object.keys(jsonObj.items.item).forEach(async item => {
const game = jsonObj.items.item[item]
const gameID = game["#_objectid"]
const rating = game.stats.rating["#_value"]
if (rating === "N/A") return
const gameObject = await gameObjects.find(
game => game.bgg === parseInt(gameID)
)
if (gameObject && gameObject.rating !== parseInt(rating)) {
try {
const updated = await Game.findOneAndUpdate(
{ _id: gameObject.id },
{ rating: rating },
{ new: true }
).exec()
changes.push(
`${updated.name}: ${gameObject.rating} -> ${updated.rating}`
)
} catch (error) {
console.log(error)
}
}
})
return changes
}
Everything works – the changes are found and the database is updated, but the reported changes are returned too late, because the execution doesn't wait for Mongoose.
I've also tried this instead of findOneAndUpdate():
const updated = await Game.findOne()
.where("_id")
.in([gameObject.id])
.exec()
updated.rating = rating
await updated.save()
The same results here: everything else works, but the async doesn't.
As #Puneet Sharma mentioned, you'll have to map instead of forEach to get an array of promises, then await on the promises (using Promise.all for convenience) before returning changes that will then have been populated:
const parseJSON = async xmlData => {
const changes = []
const games = await Game.find({})
const gameObjects = games.map(game => {
return new GameObject(game.name, game.id, game)
})
const jsonObj = require("../sample.json")
const promises = Object.keys(jsonObj.items.item).map(async item => {
const game = jsonObj.items.item[item]
const gameID = game["#_objectid"]
const rating = game.stats.rating["#_value"]
if (rating === "N/A") return
const gameObject = await gameObjects.find(
game => game.bgg === parseInt(gameID)
)
if (gameObject && gameObject.rating !== parseInt(rating)) {
try {
const updated = await Game.findOneAndUpdate(
{ _id: gameObject.id },
{ rating: rating },
{ new: true }
).exec()
changes.push(
`${updated.name}: ${gameObject.rating} -> ${updated.rating}`
)
} catch (error) {
console.log(error)
}
}
})
await Promise.all(promises)
return changes
}
(The diff, for convenience:
9,10c9,10
< let jsonObj = require("../sample.json")
< Object.keys(jsonObj.items.item).forEach(async item => {
---
> const jsonObj = require("../sample.json")
> const promises = Object.keys(jsonObj.items.item).map(async item => {
33a34
> await Promise.all(promises)
)
EDIT: a further refactoring would be to use that array of promises for the change descriptions themselves. Basically changePromises is an array of Promises that resolve to a string or null (if there was no change), so a .filter with the identity function will filter out the falsy values.
This method also has the advantage that changes will be in the same order as the keys were iterated over; with the original code, there's no guarantee of order. That may or may not matter for your use case.
I also flipped the if/elses within the map function to reduce nesting; it's a matter of taste really.
Ps. That await Game.find({}) will be a problem when you have a large collection of games.
const parseJSON = async xmlData => {
const games = await Game.find({});
const gameObjects = games.map(game => new GameObject(game.name, game.id, game));
const jsonGames = require("../sample.json").items.item;
const changePromises = Object.keys(jsonGames).map(async item => {
const game = jsonGames[item];
const gameID = game["#_objectid"];
const rating = game.stats.rating["#_value"];
if (rating === "N/A") {
// Rating from data is N/A, we don't need to update anything.
return null;
}
const gameObject = await gameObjects.find(game => game.bgg === parseInt(gameID));
if (!(gameObject && gameObject.rating !== parseInt(rating))) {
// Game not found or its rating is already correct; no change.
return null;
}
try {
const updated = await Game.findOneAndUpdate(
{ _id: gameObject.id },
{ rating: rating },
{ new: true },
).exec();
return `${updated.name}: ${gameObject.rating} -> ${updated.rating}`;
} catch (error) {
console.log(error);
}
});
// Await for the change promises to resolve, then filter out the `null`s.
return (await Promise.all(changePromises)).filter(c => c);
};

Complex query on more than one child in cloud functions

Here is the structure of my firebase database:
/UserData
/DeviceMgmt
/Counters
/NumberOfAll:
/NumberOfSelected
/TotalDownloaded
...
/Devices
/pushId1
/uid
/signOutTime
/toSelect=true (optional)
/downloaded
/lastDownload
/pushId2
/pushId3
...
And this is my cloud function:
exports.markDevicesForDownload = functions.database.ref('/UserData/DeviceMgmt/Counters/NumberOfSelected').onUpdate( (change) => {
const changeRef = change.after.ref;
const deviceMgmtRef = changeRef.parent.parent; // /UserData/DeviceMgmt
if (change.after.val() === 0 ) { //NumberOfSelected gets 0 value
return deviceMgmtRef.once('value')
.then((snap) => {
const devicesRef = snap.child('Devices').ref;
var average;
var numberOfAllDevices;
var totalDownloaded;
numberOfAllDevices = snap.child('Counters/NumberOfAll').val();
totalDownloaded = snap.child('Counters/TotalDownloaded').val();
average = Math.round(totalDownloaded/numberOfAllDevices);
return devicesRef
.orderByChild('signOutTime')
.equalTo(0)
.once('value',(devices) => {
return devices.ref
.orderByChild('downloaded')
.endAt(average)
.once('value',(devices) => {
devices.forEach((device) => {
device.child('toSelect').ref.set(true);
});
});
});
});
} else {
return false;
}
});
The function triggers when the counter NumberOfSelected = 0;
This happens when under any of device pushId there is no child toSelect. Then the query on downloaded child makes all devices with downloaded less than average set toSelect=true.
I wanted to limit the devices only to those which have signOutTime equal 0.
Somehow that query does not work and all devices are considered.
What I did wrong???
I would push all async tasks into a promise array and then return them all when all tasks complete:
exports.markDevicesForDownload = functions.database.ref('/UserData/DeviceMgmt/Counters/NumberOfSelected').onUpdate((change) => {
const changeRef = change.after.ref;
const deviceMgmtRef = changeRef.parent.parent; // /UserData/DeviceMgmt
if (change.after.val() === 0) { //NumberOfSelected gets 0 value
return deviceMgmtRef.once('value')
.then((snap) => {
const promises = [];
const devicesRef = snap.child('Devices').ref;
var average;
var numberOfAllDevices;
var totalDownloaded;
numberOfAllDevices = snap.child('Counters/NumberOfAll').val();
totalDownloaded = snap.child('Counters/TotalDownloaded').val();
average = Math.round(totalDownloaded / numberOfAllDevices);
const dR = devicesRef
.orderByChild('signOutTime')
.equalTo(0)
.once('value', (devices) => {
const dW = devices.ref
.orderByChild('downloaded')
.endAt(average)
.once('value', (devices) => {
devices.forEach((device) => {
if (device.child("signOutTime").val() === 0){
promises.push(device.child('toSelect').ref.set(true));
}
});
});
promises.push(dW);
});
promises.push(dR);
return Promise.all(promises);
});
} else {
return false;
}
});

SYSTEM_ERROR: system encountered unexpected error. Function killed. Cloud functions

When I looked through logs from Firebase cloud functions, I noticed that some functions have an error in logs with a description of "SYSTEM_ERROR: system encountered unexpected error. Function killed.", Until yesterday, this behavior was not, here is one of these functions. How can I fix it?
const functions = require('firebase-functions');
const admin = require('firebase-admin');
module.exports = functions.database.ref('/cards/{cardID}/interestedUsers').onWrite(event => {
const cardID = event.params.cardID;
console.log("interestedUsers", event.data.val(), cardID);
var currentInterestedUsers = [];
currentInterestedUsers = event.data.val();
var previousInterestedUsers = [];
previousInterestedUsers = event.data.previous.val();
if (event.data.previous.val() && currentInterestedUsers) {
var isNewPendingRequest = false;
if (currentInterestedUsers.length < previousInterestedUsers.length) {
currentInterestedUsers.forEach((interestedUser, index) => {
const interestedUserVal = interestedUser.val();
const isApproved = interestedUser["isApproved"];
console.log("result", interestedUser.val(), interestedUserVal);
if (isApproved == false) {
isNewPendingRequest = true;
}
});
}
if (isNewPendingRequest == false) {
const cardRef = admin.database().ref("cards").child(cardID);
const setupIsNewPendingRequestPromise = cardRef.update({
"isNewPendingRequest": false
});
return Promise.all([setupIsNewPendingRequestPromise]);
};
return console.log("interestedUsers deleting");
};
if (event.data.val() == null) {
console.log("event.data.val() == null");
const cardRef = admin.database().ref("cards").child(cardID);
// check card
const cardCheckPromise = cardRef.once("value", function(cardCheckPromiseSnap, error){
if (error) {
return console.log("cardCheckPromise", error);
};
if (cardCheckPromiseSnap.val()) {
const checkCardID = cardCheckPromiseSnap.val()["id"];
if (checkCardID) {
console.log("checkCardID", checkCardID);
const setupIsNewPendingRequestPromise = cardRef.update({
"isNewPendingRequest": false
});
return Promise.all([setupIsNewPendingRequestPromise]);
} else {
return console.log("checkCardID == null");
}
} else {
return console.log("cardCheckPromiseSnap.val() == null");
};
});
return Promise.all([cardCheckPromise]).catch(function(cardCheckPromiseError){
console.log("cardCheckPromise error", cardCheckPromiseError.message, cardCheckPromiseError.messageId)
});
}
return console.log("just update or adding a new interested user");
});
You can try re-deploying your app.
Likely a GCF outage. There was one today and yesterday. These errors were being thrown during the outage.
Solution
Take a break and wait for the cloud to heal. Contact GCP Support if you've purchased support.

Resources