How do I query Firestore all documents today date? - node.js

I am trying to generate report at a fixed schedule. But I am having a problem where I cannot retrieve the date for today onwards until the current time where the function will run.
exports.generateReport = functions.pubsub.schedule('36 15 * * *').onRun(async (context) => {
console.log(context);
const currentTime = admin.firestore.Timestamp.now();
const shopSnapshot = await db.collection("shops").get();
let shopDoc = shopSnapshot.docs.map(doc => doc.data());
const promises = [];
let transactionList = [];
let reportList = [];
let i = 0;
console.log(shopDoc);
shopDoc = shopDoc.filter(shop => !!shop.key);
console.log(shopDoc);
for(var j=0; j<shopDoc.length; j++){
console.log("Enter shop ID:"+shopDoc[j].key);
promises.push(db.collection("shops").doc(shopDoc[j].key).collection("transactions").get());
}
const snapshotArrays = await Promise.all(promises);
snapshotArrays.forEach(snapArray => {
snapArray.forEach(snap => {
//console.log(snap.data());
transactionList.push({data: snap.data(), key: shopDoc[i].key});
})
i++;
});
for(var k=0; k<shopDoc.length; k++){
let amount = 0;
for (var l=0; l<transactionList.length; l++){
if(shopDoc[k].key === transactionList[l].key){
console.log("get date");
if (transactionList[l].data.createAt < currentTime){
amount += transactionList[l].data.amount;
console.log(amount);
}
}
}
reportList.push({amount: amount, key: shopDoc[k].key});
}
console.log(reportList);
console.log(transactionList);
});
I tried using new Date() also compared with a string of date exactly the same as Firestore Timestamp format but still all transaction appear before the time or no transaction is included at this time.

If I correctly understand that you want to generate a daily report for all the transactions that occurred yesterday by running a schedule Cloud Function today at 00:05 for example, here is a possible approach using the moment.js library:
const functions = require('firebase-functions');
const admin = require('firebase-admin');
const moment = require('moment');
admin.initializeApp();
exports.generateReport = functions.pubsub.schedule('05 00 * * *').onRun(async (context) => {
let m1 = moment();
let m2 = moment();
m1.add(-1, 'days');
m2.add(-1, 'days');
m1.startOf('day');
m2.endOf('day');
const shopSnapshot = await db.collection("shops").get();
let shopDoc = shopSnapshot.docs.map(doc => doc.data());
const promises = [];
let transactionList = [];
let reportList = [];
let i = 0;
shopDoc = shopDoc.filter(shop => !!shop.key);
console.log(shopDoc);
for(var j=0; j<shopDoc.length; j++){
console.log("Enter shop ID:"+shopDoc[j].key);
promises.push(
db.collection("shops")
.doc(shopDoc[j].key)
.collection("transactions")
.orderBy("createAt")
.where("createAt", ">", m1.toDate())
.where("createAt", "<=", m2.toDate())
.get()
);
}
const snapshotArrays = await Promise.all(promises);
snapshotArrays.forEach(snapArray => {
snapArray.forEach(snap => {
//console.log(snap.data());
transactionList.push({data: snap.data(), key: shopDoc[i].key});
})
i++;
});
for(var k=0; k<shopDoc.length; k++){
let amount = 0;
for (var l=0; l<transactionList.length; l++){
if(shopDoc[k].key === transactionList[l].key){
amount += transactionList[l].data.amount;
}
}
reportList.push({amount: amount, key: shopDoc[k].key});
}
//.....
});
So, what do we do in this code?
First we create two moment objects and we set their date to yesterday. Then, with startOf() and endOf() we adjust the time of the first one to yesterday at 12:00:00.000 am (i.e. 00:00:00, see here) and the second one to yesterday at 11:59:59.999 pm (i.e. 23:59:59).
With these two dates, for each transactions collection, we adapt the query as follows, calling the toDate() method:
db.collection("shops")
.doc(shopDoc[j].key)
.collection("transactions")
.orderBy("createAt")
.where("createAt", ">", m1.toDate())
.where("createAt", "<=", m2.toDate())
.get();
And that's it. The big advantage here, is that the filtering is done in the Firestore database (in the back-end), not in the front-end as you did in your question (if (transactionList[l].data.createAt < currentTime){...})

Related

Batch 500 writes into firestore loop from json file

Using some inspiration I got from this thread and reply I tried to get my loop working which is to write into firestore in batches. But somehow I only can only update 1 document even if I can see I iterate through different values from my array. I load data into an array and work from there.
const db = admin.firestore();
const jsonStream = StreamArray.withParser();
let arr = []
jsonStream.on('data', ({ key, value }) => {
arr.push(value);
});
jsonStream.on('end', () => {
var counter = 0;
var commitCounter = 0;
var batches = [];
arr.forEach((a, ind) => {
batches[commitCounter] = db.batch();
if (counter <= 498) {
var thisRef = db.collection('Testing').doc(a.id);
console.log("id")
console.log(a.id);
batches[commitCounter].set(thisRef, { ...a });
counter = counter + 1;
} else {
counter = 0;
commitCounter = commitCounter + 1;
batches[commitCounter] = db.batch();
}
})
for (var i = 0; i < batches.length; i++) {
if(i==0)
{
console.log(batches[0])
}
batches[i].commit().then(function () {
console.count('wrote batch');
});
}
});
const filename = path.join(__dirname, 'mydata.json');
fs.createReadStream(filename).pipe(jsonStream.input);
Following line gets executed on each iteration, which essentially "resets" your batch on each round:
batches[commitCounter] = db.batch();
So at the end each of your batches will only contain one document write.

How to handle a long request with Express

I'm working on a simple function I have for a specific GET request triggered in the browser. The objective of this request is to make multiple queries to a mongodb (mongoose) database and then perform some calculation and structure formating on the results to send it back to the browser.
The only problem is that everything takes too long and it results in an error in the browser:
net::ERR_EMPTY_RESPONSE
to give an example of part of the function I'm trying to build here it goes:
async function getPriceByMake(makes, id) {
return new Promise(async (resolve, reject) => {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
await Listing.find({ catFrom: id, model: currModel }, 'year asking', (err, docs) => {
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
});
}
}
resolve(pMakes);
});
}
In this function, if I leave the async / await out, I get an empty {} on the other end. Which is obviously not the objective.
I've been searching the web a little and was able to find an article pointing to this scheme:
Browser:
Initiates request
displays progress
Show result
WebServer:
Submit event
Checks for completion
Return result
BackEndApp:
Picks up event
Runs task
Returns results
My question is the following:
How can I do that with NodeJS and Express?
In this code:
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
await Listing.find({ catFrom: id, model: currModel }, 'year asking', (err, docs) => {
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
});
}
Your await isn't working because you're passing a callback to Listing.find(). When you do that, it does NOT return a promise and therefore the await does nothing useful. You get the empty response because the await doesn't work and thus you call resolve() before there's any actual data there.
Change the code to this:
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}
And, then the await will work properly.
You also should remove the return new Promise() wrapper. You don't want that. Just make the function async and use await and it will already return a promise.
Here's your function with the unnecessary promise wrapper removed:
async function getPriceByMake(makes, id) {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}
}
return pMakes;
}
Then, keep in mind that whatever code sends your actual response needs to use .then() or await when calling this async function in order to get the final result.
Your best bet to speed up this code would be to refactor either your queries or your database structure or both to not have to do N * M separate queries to get your final result. That's likely where your slowness is coming from. The biggest performance gains will probably come from reducing the number of queries you have to run here to far fewer.
Depending upon your database configuration and capabilities, it might speed things up to run the inner loop queries in parallel as shown here:
async function getPriceByMake(makes, id) {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
await Promise.all(modelsArr.map(async currModel => {
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}));
}
return pMakes;
}

node wait for iteration to complete before callback

I have a lambda function in node.js to send a push notification.
In that function I need to iterate through my users sending a notification for each one prior to the callback.
Ideally I would like the iteration to perform in parallel.
What would be the best way to do this?
My code is currently as follows but it does not work as expected because the last user is not always the last to be handled:
var apnProvider = new apn.Provider(options);
var iterationComplete = false;
for (var j = 0; j < users.length; j++) {
if (j === (users.length - 1)) {
iterationComplete = true;
}
var deviceToken = users[j].user_device_token;
var deviceBadge = users[j].user_badge_count;
var notification = new apn.Notification();
notification.alert = message;
notification.contentAvailable = 1;
notification.topic = "com.example.Example";
apnProvider.send(notification, [deviceToken]).then((response) => {
if (iterationComplete) {
context.succeed(event);
}
});
}
Use Promise.all instead - map each user's associated apnProvider.send call to a Promise in an array, and when all Promises in the array are resolved, call the callback:
const apnProvider = new apn.Provider(options);
const userPromises = users.map((user) => {
const deviceToken = user.user_device_token;
const deviceBadge = user.user_badge_count;
const notification = new apn.Notification();
notification.alert = message;
notification.contentAvailable = 1;
notification.topic = "com.example.Example";
return apnProvider.send(notification, [deviceToken]);
})
Promise.all(userPromises)
.then(() => {
context.succeed(event);
})
.catch(() => {
// handle errors
});

making node wait for db call to get completed

I just started writing node.js code.
I'm writing a code that extracts data from a pdf file, cleans it up and stores it in a database (using couchdb and accessing that using nano library).
The problem is that the calls are being made asynchronously... so the database get calls (i make some get calls to get a few affiliation files during the clean up) get completed only after the program runs resulting in variables being undefined. is there any way around this?
I've reproduced my code below
const fs = require('fs');
const os = require('os');
var couchDB = require('couch-db').CouchDB;
var pdf_table_extractor = require('pdf-table-extractor');
const filename = "PQ-PRI-0005-1806-01-0000_quoteSlipForLIBVIDGI1.pdf"
var nano = require('nano')('https://couchadmin:difficulttoguessmypassword#dbdev.perilwise.com');
var server = new couchDB('https://db.url.com');
server.auth("admin","admin");
var db = nano.db.use('pwfb');
var temp = [];
//New callView function
async function callView(){
try{
const doc = await view('liabilitymdm','pi');
for (var i =0; i<doc.rows.length;i++){
tmp.push(doc.rows[i]);
};
return doc;
} catch(e){
console.log(e);
};
};
function suc(result){
let ttmp = [];
console.log(result);
var pageTables = result.pageTables;
var firstPageTables = pageTables[0].tables;
ttmp = callView();
//this console log shows Promise { <pending> }
console.log(ttmp)
for (var k = 0; k < firstPageTables.length; k++) {
var temp = firstPageTables[k];
if (temp.length > 0) {
dump.push(temp);
}
}
// console.log(dump);
var insurer = filename.substr(37,8);
read_quote_slip(insurer,dump);
}
var read_quote_slip = (insurer,data) => {
console.log("read_quote_slip correctly entered");
var finOut = {};
if (insurer === "LIBVIDGI"){
finOut.insurer = insurer;
finOut.policyType = data[2][0].replace(/Quotation for/g,"");
finOut.natureOfWork = data[13][3];
let dedpos = indexGetter(data, "Deductible")[0];
finOut.deductible = data[dedpos+1][0];
let cov = indexGetter(data, "Coverage Territory and Jurisdiction")[0];
finOut.coverageTerritory = data[cov+1][0].replace(/Territory/g,"");
finOut.coverageJurisdiction = data[cov+2][0].replace(/Jurisdiction/g,"");
let ext = indexGetter(data,"Extensions")[0];
finOut.coverage = data[ext+1][0].split(/\r?\n/);
let majexc = indexGetter(data,"Major Exclusions")[0];
finOut.exclusions = data[majexc+1][0].split(/\r?\n/);
let prdtl = indexGetter(data,"Description")[0];
let prm = premiumcompute(data,prdtl,dedpos);
finOut.premium = prm;
finCleaned = libvidgi_cleaned(finOut);
// console.log(finCleaned);
}
}
var indexGetter = (words,toFind) => {
var finindex = [];
for (var i = 0; i < words.length; i++){
for (var j = 0; j < words[i].length; j++){
if(words[i][j].indexOf(toFind) >=0 ){
finindex.push(i);
}
}
}
return finindex;
}
var premiumcompute = (data, from, to) => {
let finprem = [];
let numbop = to - from - 2;
let incr = 0;
for (var i = from+2; i < to; i++){
let pr = {};
pr.option = incr+1;
pr.sumInsured = data[i][2].replace(/ /g,"");
pr.premium = data[i][data[i].length - 1].replace(/ /g,"");
finprem.push(pr);
incr +=1;
}
return finprem;
}
var libvidgi_cleaned = (finOut) => {
return finOut;
}
var fal = (result) => {
console.log(result);
console.log("there was an error");
}
var readPDFFile = function(filename){
//Decide which insurer from the filename
// console.log(filename);
console.log(filename.substr(37,8)+"Printed on line 38");
insurer = filename.substr(37,8)
pdf_table_extractor(filename, (result) => {suc(result)} , fal);
}
var libvidgi_data_extract = (data) => {
console.log(data);
let arr = data.pageTables.tables;
for (var i = 0; i <= arr.length; i++ ){
console.log(arr[i]);
}
}
readPDFFile(filename);
This answer assumes you are using Node.js > v7.6
Since db.view accepts a callback, and you wish to wait for it to finish, one solution will be to promisify it - meaning to turn it into a promise which can be awaited. You can use a library like Bluebird or you can even use Node's builtin promisify util. Then you can rewrite callViews:
const {promisify} = require('util');
const view = promisify(db.view);
async function callView() {
try {
const doc = await view('liabilitymdm', 'pi');
// the async operation is now guaranteed to be done
// (if there is an error it will be caught by the catch clause)
for (var i = 0; i < doc.rows.length; i++) {
temp.push(doc.rows[i]);
}
console.log(temp);
} catch (e) {
}
}
If you are not using Node.js > v7.6 (and cannot use async\await you can still utilize promises, by using their then method:
const {promisify} = require('util');
const view = promisify(db.view);
function callView() {
view('liabilitymdm', 'pi')
.then(doc => {
for (var i = 0; i < doc.rows.length; i++) {
temp.push(doc.rows[i]);
}
console.log(temp);
return temp;
})
.then(temp => {
console.log(temp);
})
.catch(e => {});
}
Notice how the first then is returning something which is used in a later then.
To make Node run asynchronously, you can use the keywords async and await.
They work like this:
async function doSomething () {
const formattedData = formatData();
const result = await db.postToDatabase(formattedData);
// the below will not happen until the above line is finished
doSomethingElse(result);
}
It's pretty simple in Node to get functions to execute asynchronously. Just put the async keyword at the beginning of the function definition and then put await in front of anything that you want to block execution until completed.

Firebase Functions - NodeJS: Asynchronious Array handling

I´ve checked numerous posts but could not solve the issue.
I want to return the array before going on. I tried using a function with a callback but that did not work either.
My code looks as following:
exports.GetHelmets = functions.database.ref('onTrack/{userID}').onCreate(event => {
var helmets = [];
let userID = event.params.userID;
let friendRef = admin.database().ref("friends").child(userID);
friendRef.once("value").then(snapshot => {
snapshot.forEach(function(snapshot2){
let rider = snapshot2.val();
let riderID = rider.id;
let rhRef = admin.database().ref("User").child(riderID);
rhRef.once("value", function(snapshot3){
let rider2 = snapshot3.val();
let helmetID = rider2.helmet;
if (helmetID != "STANDARD"){
if(helmets.indexOf(helmetID) < 0){
helmets.push(helmetID);
};
};
});
});
return helmets;
}).then(helmets => {
//WORK WITH ARRAY
});
I hope you can help me, thanks!
You want the last then() to get all the inner data, each of which requires its own call to once(). In such a case, you'll want to use Promise.all() to wait for all the onces.
exports.GetHelmets = functions.database.ref('onTrack/{userID}').onCreate(event => {
let userID = event.params.userID;
let friendRef = admin.database().ref("friends").child(userID);
friendRef.once("value").then(snapshot => {
var promises = []l
snapshot.forEach(function(snapshot2){
let rider = snapshot2.val();
let riderID = rider.id;
let rhRef = admin.database().ref("User").child(riderID);
promises.push(rhRef.once("value");
});
});
return Promise.all(promises);
}).then(snapshots => {
var helmets = [];
snapshots.forEach((snapshot) => {
let rider2 = snapshot.val();
let helmetID = rider2.helmet;
if (helmetID != "STANDARD"){
if(helmets.indexOf(helmetID) < 0){
helmets.push(helmetID);
};
};
});
// WORK WITH helmets
});

Resources