Get all documents in collection using Cloud Firestore - node.js

I read several documentation but I don't understand why I should use an extra layer(foreach) in my code when I read all of the data inside a collection using Firebase (Cloud Firestore).
Here is the original documentation:
https://firebase.google.com/docs/firestore/query-data/get-data#get_all_documents_in_a_collection
Here is my code:
async loadUsers(): Promise<User[]> {
const users = new Array<User>();
const snapshot = await this.firestore.collection('users').get();
snapshot.forEach((collection) => {
collection.docs.forEach(doc => {
users.push(doc.data() as User);
});
});
return users;
}
As I understand it should work like this:
async loadUsers(): Promise<User[]> {
const users = new Array<User>();
const snapshot = await this.firestore.collection('users').get();
snapshot.forEach(doc => {
users.push(doc.data() as User);
});
return users;
}
Error message:
"Property 'data' does not exist on type 'QuerySnapshot'."

.collection().get() does NOT return an array; it returns a QuerySnapshot, which has a property .docs, which is an array of QueryDocumentSnapshot, each of which has a property .data, which is the data read from the document.
Documentation
https://firebase.google.com/docs/reference/js/firebase.firestore.CollectionReference

In new modular firebase firestore(version 9.+) it should be like this:
import { getFirestore, collection, query, getDocs } from 'firebase/firestore/lite'
async readAll() {
const firestore = getFirestore()
const collectionRef = collection(firestore, '/users')
let q = query(collectionRef, orderBy('createTimestamp', 'desc'))
const querySnapshot = await getDocs(q)
const items = []
querySnapshot.forEach(document => {
items.push(document.data())
})
return items
}
I could not find any parameter on querySnapshot directly that is something like .docs was and included whole array before. So it is kinda like onSnapshot is and was.

Based on #LeadDreamer answer, I could manage to simplify the code
async loadUsers(): Promise<User[]> {
const users = new Array<User>();
await this.firestore.collection('users').get().subscribe(querySnapshot => {
querySnapshot.docs.forEach(doc => {
users.push(doc.data() as User);
});
});
return users;
}

There seems to be no other way but to iterate.
const q = query(collection(db, "item"));
getDocs(q).then( response => {
const result = response.docs.map(doc=>({
id: doc.id,
...doc.data(),
}))
console.log(result);
}).catch(err=>console.log(err))

Related

Google Firestore array_contains query doesn't work

I am trying to run a simple query to find other document that contains some ID. Here is how it looks, and here is what I am trying to get. I don't see a reason for it to work this way. I tried this code for Firestore Functions but it doesn't work.
I tried this code:
exports.updateDietDaysWhenMealChanges = functions.firestore
.document("Posilek/{posilekId}")
.onUpdate((change, context) => {
const posilekId = context.params.posilekId;
const posilekAfter = change.after.data();
return db.collection("DietDays")
.where("Meals", "array-contains", { ID: posilekId })
.get()
.then(snapshot => {
if (snapshot.empty) {
functions.logger.log("No matching DietDay found");
return null;
} else {
return Promise.all(snapshot.docs.map(dietDayDoc => {
const dietDayId = dietDayDoc.id;
const meals = dietDayDoc.data().Meals;
const mealIndex = meals.findIndex(meal => meal.ID === posilekId);
meals[mealIndex] = { ID: posilekId, Portions: posilekAfter.Portions };
functions.logger.log(`Editing meal in DietDay with ID: ${dietDayId}`);
return dietDayDoc.ref.update({ Meals: meals });
}));
}
});
});
And I tried manual query.
The array-contains operator can only check for exact matches between items in the array and the value you pass. So in code:
.where("Meals", "array-contains", { ID: posilekId, Portions: 12.5 })
There is no way to do a partial match.
The common workaround is to add an additional field (e.g. MealIDs) that contains just the value you want to filter on:
MealIDs: ["ohN....", "..."]
With that additional array, you can then filter with:
.where("MealIDs", "array-contains", posilekId)

Caching in NODEJS

I am trying to cache the results of the Twitter query as they have rate limiting and I cannot seem to figure it out.
I am following this tutorial here.
The Cache.js file looks like so (taken from tutorial):
import NodeCache from 'node-cache';
class Cache {
constructor(ttlSeconds) {
this.cache = new NodeCache({ stdTTL: ttlSeconds, checkperiod: ttlSeconds * 0.2, useClones: false });
}
get(key, storeFunction) {
const value = this.cache.get(key);
if (value) {
return Promise.resolve(value);
}
return storeFunction().then((result) => {
this.cache.set(key, result);
return result;
});
}
del(keys) {
this.cache.del(keys);
}
delStartWith(startStr = '') {
if (!startStr) {
return;
}
const keys = this.cache.keys();
for (const key of keys) {
if (key.indexOf(startStr) === 0) {
this.del(key);
}
}
}
flush() {
this.cache.flushAll();
}
}
export default Cache;
My Twitter query file looks like so :
import TwitterCount from "../models/TwitterModel.js";
import { TwitterApi } from 'twitter-api-v2';
import dotenv from "dotenv";
dotenv.config();
import CacheService from '../middleware/Cache.js';
const twitterClient = new TwitterApi(process.env.TWITTER_API_BEARER_TOKEN);
const readOnlyClient = twitterClient.readOnly;
const ttl = 60 * 60 * 1; //cache for 1 hour
const cache = new CacheService(ttl);
export const getCount = async(req, res) => {
try {
const twit = await TwitterCount.findOne({
attributes:['followersCount']
});
const key = `Twit_${String(twit.dataValues.followersCount)}`;
const twitterFollowers = await readOnlyClient.v2.followers('1563787278857785350'); //HejOfficial account Twitter
const results = cache.get( key , () => twitterFollowers.data.length );
if (twit.dataValues.followersCount === results) {
console.log('same results');
} else {
await TwitterCount.create({
followersCount: results
});
console.log("Twitter Data added Successful");
}
res.json(twit);
cache.del(key);
} catch (error) {
console.log(error);
}
}
I presume that I am not using the key properly. Please assist if you have encountered this issue before, or provide a better caching alternative.
Thank you.
Try to use the account id as the key, rather than follower count. This way you are caching the accounts followers for an hour.
Something like this should get you going with the cache, it doesn't have the database inserts, but they can be handled now in the getCount method.
export const getCount = async(req, res) => {
const result = await getCountFromAPI("1563787278857785350") //HejOfficial account Twitter
res.json(result);
};
const getCountFromAPI = async(account) => {
const key = `Twit_${account}`;
return cache.get(key, async() => {
const twitterFollowers = await readOnlyClient.v2.followers(account);
return twitterFollowers;
});
};
Note that if you save the results to your database every time, it defeats the purpose of the cache. I'd suggest only using the API for this data, and if you want to use database as well, maybe the cache is unnessesary? Save the follower count and account id with time-updated value, and if time updated is more than one hour ago, then query the value again from Twitter.

async function doesn't wait of inside await in nodejs

I am implementing function monthlyRevenue.
Simply, it will return total monthly revenue,and it takes arguments of station array which will make revenues, month and year.
Problem
Inside of this function I have getStationPortion which will fetch the revenue portion of user's.
So I would like to make it return object like this.
stationsPortion = {station1 : 30, station2 : 20}
In the monthlyRevenue
const stationPortions = await getStationPortions(stations)
console.log("portion map", stationPortions //it will be shown very beginning with empty
getStationPortions
const getStationPortions = async (stations) => {
let stationPortions = {}
stations.map(async (value) => {
const doc = await fdb.collection('Stations').doc(value).get()
if (!doc.exists) {
console.log("NO DOC")
} else {
stationPortions[value] = doc.data().salesPortion
console.log(stationPortions) //it will be shown at the last.
}
})
return stationPortions
}
I thought that async function should wait for the result, but it does not.
I am kind of confusing if my understanding is wrong.
Thank you
(by the way, fdb is firebase admin(firestore)
Working code
const getStationPortions = async (stations) => {
let stationPortions = {}
await Promise.all(stations.map(async (value) => {
const doc = await fdb.collection('Stations').doc(value).get()
if (!doc.exists) {
console.log("NO DOC")
} else {
stationPortions[value] = doc.data().salesPortion
console.log(stationPortions)
}
}))
return stationPortions
}
module.exports = router;

How to use MongoDB locally and directline-js for state management in Bot Framework using NodeJs and Mongoose?

I am maintaining the bot state in a local MongoDB storage. When I am trying to hand-off the conversation to an agent using directline-js, it shows an error of BotFrameworkAdapter.sendActivity(): Missing Conversation ID. The conversation ID is being saved in MongoDB
The issue is arising when I change the middle layer from Array to MongoDB. I have already successfully implemented the same bot-human hand-off using directline-js with an Array and the default Memory Storage.
MemoryStorage in BotFramework
const { BotFrameworkAdapter, MemoryStorage, ConversationState, UserState } = require('botbuilder')
const memoryStorage = new MemoryStorage();
conversationState = new ConversationState(memoryStorage);
userState = new UserState(memoryStorage);
Middle Layer for Hand-Off to Agent
case '#connect':
const user = await this.provider.connectToAgent(conversationReference);
if (user) {
await turnContext.sendActivity(`You are connected to
${ user.userReference.user.name }\n ${ JSON.stringify(user.messages) }`);
await this.adapter.continueConversation(user.userReference, async
(userContext) => {
await userContext.sendActivity('You are now connected to an agent!');
});
}
else {
await turnContext.sendActivity('There are no users in the Queue right now.');
}
The this.adapter.continueConversation throws the error when using MongoDB.
While using Array it works fine. The MongoDB and Array object are both similar in structure.
Since this works with MemoryStorage and not your MongoDB implementation, I'm guessing that there's something wrong with your MongoDB implementation. This answer will focus on that. If this isn't the case, please provide your MongoDb implementation and/or a link to your repo and I can work off that.
Mongoose is only necessary if you want to use custom models/types/interfaces. For storage that implements BotState, you just need to write a custom Storage adapter.
The basics of this are documented here. Although written for C#, you can still apply the concepts to Node.
1. Install mongodb
npm i -S mongodb
2. Create a MongoDbStorage class file
MongoDbStorage.js
var MongoClient = require('mongodb').MongoClient;
module.exports = class MongoDbStorage {
constructor(connectionUrl, db, collection) {
this.url = connectionUrl;
this.db = db;
this.collection = collection;
this.mongoOptions = {
useNewUrlParser: true,
useUnifiedTopology: true
};
}
async read(keys) {
const client = await this.getClient();
try {
var col = await this.getCollection(client);
const data = {};
await Promise.all(keys.map(async (key) => {
const doc = await col.findOne({ _id: key });
data[key] = doc ? doc.document : null;
}));
return data;
} finally {
client.close();
}
}
async write(changes) {
const client = await this.getClient();
try {
var col = await this.getCollection(client);
await Promise.all(Object.keys(changes).map((key) => {
const changesCopy = { ...changes[key] };
const documentChange = {
_id: key,
document: changesCopy
};
const eTag = changes[key].eTag;
if (!eTag || eTag === '*') {
col.updateOne({ _id: key }, { $set: { ...documentChange } }, { upsert: true });
} else if (eTag.length > 0) {
col.replaceOne({ _id: eTag }, documentChange);
} else {
throw new Error('eTag empty');
}
}));
} finally {
client.close();
}
}
async delete(keys) {
const client = await this.getClient();
try {
var col = await this.getCollection(client);
await Promise.all(Object.keys(keys).map((key) => {
col.deleteOne({ _id: key });
}));
} finally {
client.close();
}
}
async getClient() {
const client = await MongoClient.connect(this.url, this.mongoOptions)
.catch(err => { throw err; });
if (!client) throw new Error('Unable to create MongoDB client');
return client;
}
async getCollection(client) {
return client.db(this.db).collection(this.collection);
}
};
Note: I've only done a little testing on this--enough to get it to work great with the Multi-Turn-Prompt Sample. Use at your own risk and modify as necessary.
I based this off of a combination of these three storage implementations:
memoryStorage
blobStorage
cosmosDbStorage
3. Use it in your bot
index.js
const MongoDbStorage = require('./MongoDbStorage');
const mongoDbStorage = new MongoDbStorage('mongodb://localhost:27017/', 'testDatabase', 'testCollection');
const conversationState = new ConversationState(mongoDbStorage);
const userState = new UserState(mongoDbStorage);

How to make Mongoose update work with await?

I'm creating a NodeJS backend where a process reads in data from a source, checks for changes compared to the current data, makes those updates to MongoDB and reports the changes made. Everything works, except I can't get the changes reported, because I can't get the Mongoose update action to await.
The returned array from this function is then displayed by a Koa server. It shows an empty array, and in the server logs, the correct values appear after the server has returned the empty response.
I've digged through Mongoose docs and Stack Overflow questions – quite a few questions about the topic – but with no success. None of the solutions provided seem to help. I've isolated the issue to this part: if I remove the Mongoose part, everything works as expected.
const parseJSON = async xmlData => {
const changes = []
const games = await Game.find({})
const gameObjects = games.map(game => {
return new GameObject(game.name, game.id, game)
})
let jsonObj = require("../sample.json")
Object.keys(jsonObj.items.item).forEach(async item => {
const game = jsonObj.items.item[item]
const gameID = game["#_objectid"]
const rating = game.stats.rating["#_value"]
if (rating === "N/A") return
const gameObject = await gameObjects.find(
game => game.bgg === parseInt(gameID)
)
if (gameObject && gameObject.rating !== parseInt(rating)) {
try {
const updated = await Game.findOneAndUpdate(
{ _id: gameObject.id },
{ rating: rating },
{ new: true }
).exec()
changes.push(
`${updated.name}: ${gameObject.rating} -> ${updated.rating}`
)
} catch (error) {
console.log(error)
}
}
})
return changes
}
Everything works – the changes are found and the database is updated, but the reported changes are returned too late, because the execution doesn't wait for Mongoose.
I've also tried this instead of findOneAndUpdate():
const updated = await Game.findOne()
.where("_id")
.in([gameObject.id])
.exec()
updated.rating = rating
await updated.save()
The same results here: everything else works, but the async doesn't.
As #Puneet Sharma mentioned, you'll have to map instead of forEach to get an array of promises, then await on the promises (using Promise.all for convenience) before returning changes that will then have been populated:
const parseJSON = async xmlData => {
const changes = []
const games = await Game.find({})
const gameObjects = games.map(game => {
return new GameObject(game.name, game.id, game)
})
const jsonObj = require("../sample.json")
const promises = Object.keys(jsonObj.items.item).map(async item => {
const game = jsonObj.items.item[item]
const gameID = game["#_objectid"]
const rating = game.stats.rating["#_value"]
if (rating === "N/A") return
const gameObject = await gameObjects.find(
game => game.bgg === parseInt(gameID)
)
if (gameObject && gameObject.rating !== parseInt(rating)) {
try {
const updated = await Game.findOneAndUpdate(
{ _id: gameObject.id },
{ rating: rating },
{ new: true }
).exec()
changes.push(
`${updated.name}: ${gameObject.rating} -> ${updated.rating}`
)
} catch (error) {
console.log(error)
}
}
})
await Promise.all(promises)
return changes
}
(The diff, for convenience:
9,10c9,10
< let jsonObj = require("../sample.json")
< Object.keys(jsonObj.items.item).forEach(async item => {
---
> const jsonObj = require("../sample.json")
> const promises = Object.keys(jsonObj.items.item).map(async item => {
33a34
> await Promise.all(promises)
)
EDIT: a further refactoring would be to use that array of promises for the change descriptions themselves. Basically changePromises is an array of Promises that resolve to a string or null (if there was no change), so a .filter with the identity function will filter out the falsy values.
This method also has the advantage that changes will be in the same order as the keys were iterated over; with the original code, there's no guarantee of order. That may or may not matter for your use case.
I also flipped the if/elses within the map function to reduce nesting; it's a matter of taste really.
Ps. That await Game.find({}) will be a problem when you have a large collection of games.
const parseJSON = async xmlData => {
const games = await Game.find({});
const gameObjects = games.map(game => new GameObject(game.name, game.id, game));
const jsonGames = require("../sample.json").items.item;
const changePromises = Object.keys(jsonGames).map(async item => {
const game = jsonGames[item];
const gameID = game["#_objectid"];
const rating = game.stats.rating["#_value"];
if (rating === "N/A") {
// Rating from data is N/A, we don't need to update anything.
return null;
}
const gameObject = await gameObjects.find(game => game.bgg === parseInt(gameID));
if (!(gameObject && gameObject.rating !== parseInt(rating))) {
// Game not found or its rating is already correct; no change.
return null;
}
try {
const updated = await Game.findOneAndUpdate(
{ _id: gameObject.id },
{ rating: rating },
{ new: true },
).exec();
return `${updated.name}: ${gameObject.rating} -> ${updated.rating}`;
} catch (error) {
console.log(error);
}
});
// Await for the change promises to resolve, then filter out the `null`s.
return (await Promise.all(changePromises)).filter(c => c);
};

Resources