Firebase Cloud Functions Cron Async Func Multiple Calls - node.js

I have a cloud function. It is triggered by an App Engine Cron job. It triggers my Firebase Cloud Function every hour with a Google Cloud Pub/Sub. I fetch my Firebase Realtime Database once, and loop for every value. The problem is my main.async function called multiple times. I use an i variable for loop and my console logs more i count than my database length. I mean that if my database length is 4, but for loop runs 8 or 15 or 23 times. This values change randomly. I want to loop for my each database value,fetch some data on internet,and when it is done, go for next value. Here is the code:
use strict';
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp(functions.config().firebase);
var request = require('request-promise').defaults({ encoding: null });
var fs = require('fs');
// Get a reference to the Cloud Vision API component
const Vision = require('#google-cloud/vision');
const vision = new Vision.ImageAnnotatorClient();
var os = require("os");
var databaseArray = [];
var uniqueFilename = require('unique-filename')
exports.hourly_job = functions.pubsub
.topic('hourly-job')
.onPublish((event) => {
console.log("Hourly Job");
var db = admin.database();
var ref = db.ref("myData")
ref.once("value").then(function(allData) {
allData.forEach(function(deviceToken) {
deviceToken.forEach(function(firebaseIDs) {
var deviceTokenVar = deviceToken.key;
var firebaseIDVar = firebaseIDs.key;
var firstvalue = firebaseIDs.child("firstvalue").val();
var secondvalue = firebaseIDs.child("secondvalue").val();
var items = [deviceTokenVar, firebaseIDVar, firstvalue, secondvalue];
databaseArray.push(items);
});
});
return databaseArray;
}).then(function (databasem) {
main().catch(console.error);
});
return true;
});
const main = async () => {
var i;
for (i = 0; i < databaseArray.length; i++) {
console.log("Database Arrays " + i + ". elements: ");
if (databaseArrayfirst != "") {
var apiUrl = "http://api.blabla;
try {
const apiBody = await request.get(apiUrl);
///////////////////////////vison start//////////////////////
const visionResponseBody = await vision.documentTextDetection(apiBody)
var visionResponse = visionResponseBody[0].textAnnotations[0].description;
...some logic here about response...
/////////////////////////////////////////////////
var getdatabasevar = await admin.database().ref("myData/" + databaseArrayDeviceToken + "/" + databaseArrayFirebaseID);
await getdatabasevar.update({
"firstvalue": visionResponse
});
/////////////////////////////////////////////////
} catch (error) {
console.error(error);
}
///////////////////////////vison end//////////////////////
}
};
return true;
};
Thank you.

Related

Puppeteer to convert html to pdf using Nodejs in Durable functions(fan out fan in)

I'm working on a small project to convert a large xml to several formatted pdf documents. The large xml contains multiple similar format xmls. So I'm using a single html template for printing all the documents. After producing all the pdf documents I also need to produce a metadata file with some basic info on each document that was printed.
I thought using the fan out fan in scenario of durable functions is a perfect for my use case. I'm working with Nodejs. I setup all my code and it seems to be working fine locally. The Orchestration function looks like the below.
const df = require("durable-functions");
module.exports = df.orchestrator(function* (context) {
var xmldata = yield context.df.callActivity("DurablegetblobJS1","");
var tasks = [];
for (file of xmldata) {
tasks.push(context.df.callActivity("Durableactivityjs2", file));
}
const outputs = yield context.df.Task.all(tasks);
var finalout = "";
for (out of outputs){
console.log('I am done1 :' + out );
finalout = finalout + out;
}
return finalout;
});
DurablegetblobJS1 : Fetches the entire xmls and splits it into multiple smaller xmls(1 per document).
Durableactivityjs2 : Fetches the html template, extracts the different values from the individual xmls and applies them to the html and finally prints out the pdf into an azure storage. It returns the name of the pdf document that was printed for creation of the metadata file. The code for this is below.
var fs = require('fs');
var xml2js = require('xml2js');
var html_to_pdf = require('html-pdf-node');
var parser = new xml2js.Parser();
module.exports = async function (context) {
//console.log("Hello from activity :")
var xmldict = {}
var xmltext = context.bindings.name;
//Extract the nodes and attributes
metadata(xmltext,xmldict);
report(xmltext,xmldict);
context.log(xmldict)
const { BlobServiceClient } = require("#azure/storage-blob");
// Load the .env file if it exists
require("dotenv").config();
const AZURE_STORAGE_CONNECTION_STRING = process.env.STORAGE_CONNECTION_STRING || "";
const blobServiceClient = BlobServiceClient.fromConnectionString(
AZURE_STORAGE_CONNECTION_STRING
);
var containerClient = blobServiceClient.getContainerClient('test');
var blobname = 'comb_template.html';
var blockBlobClient = containerClient.getBlockBlobClient(blobname);
var downloadBlockBlobResponse = await blockBlobClient.download(0);
var html_template = await streamToText(downloadBlockBlobResponse.readableStreamBody);
let options = { format: 'A4'};
let file = { content: html_template};
const x = await writepdf1(file, options,blobServiceClient,xmldict);
console.log("Written Blob PDF");
return x;
};
async function writepdf1(file, options,blobServiceClient,xmldict){
const pdfBuffer = await html_to_pdf.generatePdf(file, options);
const containerClient = blobServiceClient.getContainerClient('test2');
const targetblob = xmldict['OU'] + '/' + xmldict['ReportName'] + '/' + xmldict['OU'] + '_' + xmldict['ReportName'] + '_' + xmldict['DocumentID'] + '_' + '.pdf';
console.log('Blob name :' + targetblob);
const blockBlobClient_t = containerClient.getBlockBlobClient(targetblob);
const uploadBlobResponse = await blockBlobClient_t.upload(pdfBuffer, pdfBuffer.length);
return targetblob;
}
async function streamToText(readable) {
readable.setEncoding('utf8');
let data = '';
for await (const chunk of readable) {
data += chunk;
}
return data;
}
function metadata(xmltext,xmldict){
parser.parseString(xmltext, function (err, result) {
var test1 = result['HPDPSMsg']['DocumentRequest'][0]['MetaData'][0];
Object.entries(test1).forEach(([key, value]) => {
xmldict[key] = value[0];
});
});
}
function report(xmltext,xmldict){
parser.parseString(xmltext, function (err, result) {
var test2 = result['HPDPSMsg']['DocumentRequest'][0]['Report'][0]['$'];
Object.entries(test2).forEach(([key, value]) => {
xmldict[key] = value;
});
});
}
However, when I deploy the entire project into a azure premium function(EP1 - Windows), I see some errors in app insights when I try and execute my function and the pdfs are never generated.
Activity function 'Durableactivityjs2' failed: Could not find browser
revision 818858. Run "PUPPETEER_PRODUCT=firefox npm install" or
"PUPPETEER_PRODUCT=firefox yarn install" to download a supported
Firefox browser binary
I'm a bit clueless how I'm supposed to resolve this. Any help or suggestions would be appreciated.

AWS lambda sending sns message to phone number

I have a lambda function running node.js The function makes database calls and sends a text message via sns. It has the following structure
the functions -> index.js file looks like this
async function sendTextMessage(message, phoneNumber) {
try {
console.log("hitSendFunction");
const sns = new AWS.SNS();
const params = {
Message: message,
MessageStructure: "string",
PhoneNumber: phoneNumber
};
//remember phone number must have +1 before it.
return await sns.publish(params).promise();
} catch (error) {
console.log(error);
}
}
module.exports = {
sendTextMessage
};
that function is than called in the main index.js file:
const database = require("./db");
const { sendTextMessage } = require("./functions");
const AWS = require("aws-sdk");
AWS.config.region = "us-east-1";
exports.handler = async function (event, context) {
try {
console.log("hit");
const result = await database.query("call getTicker()");
const data = result[0][0];
const currentProjectEndDate = new Date(
data.currentProjectEndDate
).getTime();
const now = new Date().getTime();
console.log("data.currentProjectEndDate", data.currentProjectEndDate);
const runningHot =
data.jobsInQueue > 0 && currentProjectEndDate <= now && data.textSent < 1;
if (runningHot) {
const numbers = ["+1435994****"];
for (let i = 0; i < numbers.length; i++) {
let number = numbers[i];
console.log("number", number);
let messageResult = await sendTextMessage(
"The CE Bot is running hot from lambda",
number
);
console.log("messageResult", messageResult);
}
await database.query("call insertIntoTextSent()");
console.log("yes");
}
} catch (error) {
console.log("this was the error", error);
}
The database calls work correctly but the textMessage function just hangs and times out. The lambda function has the following permissions attached to it:
Finally even though I do not think it is needed as the db code is working here is what the db -> index.js file looks like:
const mysql = require("mysql");
const util = require("util");
const awsConfig = {
host: process.env.RDS_HOST,
user: process.env.RDS_USER,
password: process.env.RDS_PASSWORD,
database: process.env.RDS_DATABASE
};
const connection = mysql.createConnection(awsConfig);
connection.query = util.promisify(connection.query.bind(connection));
connection.end = util.promisify(connection.end.bind(connection));
module.exports = connection;
I am not quite sure where I am going wrong here. Can someone point me in the right direction?
You can just send it via sms in sns just type in the phone number

Using classes in firebase functions?

I am writing code for a firebase function, the problem is that I need to use a class but when I call the class method the firebase function log shows this error:
ERROR:
ReferenceError: Proverbi is not defined
at exports.getProverbio.functions.https.onRequest (/srv/index.js:48:26)
at cloudFunction (/srv/node_modules/firebase-functions/lib/providers/https.js:49:16)
at /worker/worker.js:783:7
at /worker/worker.js:766:11
at _combinedTickCallback (internal/process/next_tick.js:132:7)
at process._tickDomainCallback (internal/process/next_tick.js:219:9)
Here's the "index.js" code:
//firebase deploy --only functions
// The Cloud Functions for Firebase SDK to create Cloud Functions and setup triggers.
const functions = require('firebase-functions');
// The Firebase Admin SDK to access the Firebase Realtime Database.
const admin = require('firebase-admin');
admin.initializeApp();
// Take the text parameter passed to this HTTP endpoint and insert it into the
// Realtime Database under the path /messages/:pushId/original
exports.addStanza = functions.https.onRequest(async (req, res) => {
// Grab the text parameter.
const nome = req.query.text;
// Push the new message into the Realtime Database using the Firebase Admin SDK.
const snapshot = await admin.database().ref('/stanze').push({giocatori: {giocatore:{nome:nome,punteggio:0}}});
// Redirect with 303 SEE OTHER to the URL of the pushed object in the Firebase console.
//res.redirect(200, nome.toString());
var link = snapshot.toString().split('/');
res.json({idStanza:link[4]});
});
// Listens for new messages added to /messages/:pushId/original and creates an
// uppercase version of the message to /messages/:pushId/uppercase
exports.addFirstPlayer = functions.database.ref('/stanze/{pushId}/giocatori/giocatore/nome')
.onCreate((snapshot, context) => {
// Grab the current value of what was written to the Realtime Database.
const nome = snapshot.val();
// const snapshot3 = snapshot.ref('/stanza/{pushId}/giocatori/giocatore').remove();
const snapshot2 = snapshot.ref.parent.parent.remove();
return snapshot.ref.parent.parent.push({nome:nome,punteggio:0});
});
exports.addPlayer = functions.https.onRequest(async (req, res) => {
// Grab the text parameter.
const nome = req.query.text;
const idStanza = req.query.id;
// Push the new message into the Realtime Database using the Firebase Admin SDK.
const snapshot = await admin.database().ref('/stanz/'+idStanza+"/giocatori").push({nome:nome,punteggio:0 });
// Redirect with 303 SEE OTHER to the URL of the pushed object in the Firebase console.
//res.redirect(200, nome.toString());
res.json({success:{id:idStanza}});
});
exports.getProverbio = functions.https.onRequest(async (req, res) => {
const difficolta = req.query.difficolta;
var proverbioClass = new Proverbi(2);
var p = proverbioClass.getProverbio();
var proverbio = p.split('-');
var inizio = proverbio[0];
var fine = proverbio[1];
res.json({proverbio:{inizio:inizio,fine:fine,difficolta:difficolta}});
});
Here's the code that's causing the problem:
exports.getProverbio = functions.https.onRequest(async (req, res) => {
const difficolta = req.query.difficolta;
var proverbioClass = new Proverbi(2);
var p = proverbioClass.getProverbio();
var proverbio = p.split('-');
var inizio = proverbio[0];
var fine = proverbio[1];
res.json({proverbio:{inizio:inizio,fine:fine,difficolta:difficolta}});
});
Here's the "Proverbi.class" code:
class Proverbi{
constructor(n) {
this.magicNumber = n;
}
getProverbio() {
var text = "";
switch (magicNumber) {
case 1:
text += ("Chip");
text += ("-Chop");
break;
}
return text;
}
}
How can I use the "Proverbi" class inside the "index.js"?
You need to add the definition of your Proverbi Class to the index.js file.
If you are just going to use this Class in the getProverbio Cloud Function, do as follows:
exports.getProverbio = functions.https.onRequest(async (req, res) => {
class Proverbi {
constructor(n) {
this.magicNumber = n;
}
getProverbio() {
var text = "";
switch (this.magicNumber) {
case 1:
text += ("Chip");
text += ("-Chop");
break;
}
return text;
}
}
const difficolta = req.query.difficolta;
var proverbioClass = new Proverbi(2);
var p = proverbioClass.getProverbio();
var proverbio = p.split('-');
var inizio = proverbio[0];
var fine = proverbio[1];
res.json({ proverbio: { inizio: inizio, fine: fine, difficolta: difficolta } });
});
If you want to use the Class in other functions, just declare it as follows:
class Proverbi {
constructor(n) {
console.log(n);
}
getProverbio() {
console.log(this.magicNumber);
console.log(this.magicNumber);
var text = "";
switch (this.magicNumber) {
case 1:
text += ("Chip");
text += ("-Chop");
break;
}
return text;
}
}
exports.getProverbio = functions.https.onRequest(async (req, res) => {
const difficolta = req.query.difficolta;
var proverbioClass = new Proverbi(2);
var p = proverbioClass.getProverbio();
var proverbio = p.split('-');
var inizio = proverbio[0];
var fine = proverbio[1];
res.json({ proverbio: { inizio: inizio, fine: fine, difficolta: difficolta } });
});

Firebase Database Get All Value In Order Cloud Functions

I develop for Firebase Cloud Functions. I have a Firebase Realtime Database like this:
----- myData
-------eqewrwrepere (this one is a device token)
---------Lta+sde-fer (this one is a firebase id)
firstvalue : "a"
secondvalue : "b"
----------Qrgd+ad-qdda (this one is second firebase id)
firstvalue : "c"
secondvalue : "d"
-------eqwerSAsdqe (this one is another device token)
---------Lta+sde-fer (this one is a firebase id)
firstvalue : "x"
secondvalue : "y"
----------Qrgd+ad-qdda (this one is second firebase id)
firstvalue : "z"
secondvalue : "t"
I fetch these data by this code. With this code i fetch all data and put them an array. And when fetching done, i loop this array for finding items. I am an iOS developer, so i am a newbie for NodeJS. Here is what i want to do:
Get firstvalue for each database data.
Make a api request with firstvalue of each database data.
Api returns an image.
Write image temp directory.
Process this image for visionApi.
Extract text.
Update database.
Send notification for deviceToken
Now i am able to retrieve database items in my array. When i make a request in for loop, request called async. So for loop continues, but request response or writing file and vision processing executed only once.
In for loop, get databasearray[0], make request, write file, process it with vision api, update database and go for next databasearray[1] item.
I read about Promises on different pages. But i did not understand.
Thank you.
'use strict';
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp(functions.config().firebase);
var request = require('request');
var fs = require('fs');
//var fs = require("fs");
// Get a reference to the Cloud Vision API component
const Vision = require('#google-cloud/vision');
const vision = new Vision.ImageAnnotatorClient();
// Imports the Google Cloud client library
//const {Storage} = require('#google-cloud/storage');
var fs = require("fs");
var os = require("os");
var databaseArray = [];
exports.hourly_job = functions.pubsub
.topic('hourly-job')
.onPublish((event) => {
console.log("Hourly Job");
var db = admin.database();
var ref = db.ref("myData")
ref.once("value").then(function(allData) {
allData.forEach(function(deviceToken) {
deviceToken.forEach(function(firebaseIDs) {
var deviceTokenVar = deviceToken.key;
var firebaseIDVar = firebaseIDs.key;
var firstvalue = firebaseIDs.child("firstvalue").val();
var secondvalue = firebaseIDs.child("secondvalue").val();
var items = [deviceTokenVar, firebaseIDVar, firstvalue, secondvalue];
databaseArray.push([...items]);
});
});
return databaseArray;
}).then(function(databasem) {
var i;
for (i = 0; i < databaseArray.length; i++) {
var databaseArrayDeviceToken = databaseArray[i][0];
console.log("DeviceToken: " + databaseArrayDeviceToken);
var databaseArrayFirebaseID = databaseArray[i][1];
console.log("FirebaseID: " + databaseArrayFirebaseID);
var databaseArrayfirstvalue = databaseArray[i][2];
console.log("firstval: " + databaseArrayfirstvalue);
var databaseArraysecondval = databaseArray[i][3];
console.log("Second: " + databaseArraysecondval);
var url = "http://api.blabla" + databaseArrayfirstvalue;
/////////////here make a request, pause loop, process returned image, but how //////////////////////
request.get({
url: url,
encoding: 'binary'
}, function(error, httpResponse, body) {
if (!error && httpResponse.statusCode == 200) {
fs.writeFileSync('/tmp/processed.jpg', body, 'binary')
console.log("file written");
})
}
});
return true;
});
I found solution with Mocas helps. Here is the solution. I use async/await functions in code. Now for loop waits for the function response. But now I have different problems. I think main async function hangs because of awaits. And then next hourly trigger, it runs again. So console log shows 15-16-17 or more ‘i’ values in for loop. I have 4 element in database array but console log shows more than this every hour. And it increases every time. So I guess that I should cancel this await functions after a timeout. But I don’t know how. Here is code:
use strict';
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp(functions.config().firebase);
var request = require('request-promise').defaults({ encoding: null });
var fs = require('fs');
// Get a reference to the Cloud Vision API component
const Vision = require('#google-cloud/vision');
const vision = new Vision.ImageAnnotatorClient();
var os = require("os");
var databaseArray = [];
var uniqueFilename = require('unique-filename')
exports.hourly_job = functions.pubsub
.topic('hourly-job')
.onPublish((event) => {
console.log("Hourly Job");
var db = admin.database();
var ref = db.ref("myData")
ref.once("value").then(function(allData) {
allData.forEach(function(deviceToken) {
deviceToken.forEach(function(firebaseIDs) {
var deviceTokenVar = deviceToken.key;
var firebaseIDVar = firebaseIDs.key;
var firstvalue = firebaseIDs.child("firstvalue").val();
var secondvalue = firebaseIDs.child("secondvalue").val();
var items = [deviceTokenVar, firebaseIDVar, firstvalue, secondvalue];
databaseArray.push([...items]);
//console.log(databaseArray);
//return true;
});
//return true;
});
return databaseArray;
}).then(function (databasem) {
main().catch(console.error);
});
return true;
});
const main = async () => {
var i;
for (i = 0; i < databaseArray.length; i++) {
console.log("Database Arrays " + i + ". elements: ");
var databaseArrayDeviceToken = databaseArray[i][0];
console.log("DeviceToken: " + databaseArrayDeviceToken);
var databaseArrayFirebaseID = databaseArray[i][1];
console.log("FirebaseID: " + databaseArrayFirebaseID);
var databaseArrayfirst = databaseArray[i][2];
console.log("first: " + databaseArrayfirst);
var databaseArraysecond = databaseArray[i][3];
console.log("second: " + databaseArraysecond);
if (databaseArrayfirst != "") {
var apiUrl = "http://api.blabla;
try {
const apiBody = await request.get(apiUrl);
///////////////////////////vison start//////////////////////
const visionResponseBody = await vision.documentTextDetection(apiBody)
var visionResponse = visionResponseBody[0].textAnnotations[0].description;
console.log("Vision response text " + visionResponse );
...some logic here about response...
/////////////////////////////////////////////////
var getdatabasevar = await admin.database().ref("myData/" + databaseArrayDeviceToken + "/" + databaseArrayFirebaseID);
await getdatabasevar.update({
"firstvalue": visionResponse
});
/////////////////////////////////////////////////
var getanotgerdatabasevar = await admin.database().ref("myData/" + databaseArrayDeviceToken + "/" + databaseArrayFirebaseID + "/" + "secondvalue");
await getanotgerdatabasevar.once("value")
.then(function(var) {
..some logic..
//send notification
});
} catch (error) {
console.error(error);
}
///////////////////////////vison end//////////////////////
}
};
return true;
};

NodeJS Script Runs on Local Machine but Not in Firebase Cloud Functions?

My script written in NodeJS connects to Firebase, checks my Firebase Database and even successfully sends notifications when results from my database return true... However, it only works when I run it from my local machine. I deploy it to Firebase and it will not work. Could someone please advise? Thank you.
I hate asking on here because I'm a newbie but I have spent hours tonight trying to find an answer...
INDEX.JS
// Firebase Functions
const functions = require('firebase-functions');
var admin = require("firebase-admin");
// Default admin firebase configuration
admin.initializeApp(functions.config().firebase);
// var serviceAccount = require("xxxxxx-80xxxxd-firebase-adminsdk- xxxxxxx.json");
var moment = require('moment');
var FCM = require('fcm-push');
var dateTime = require('node-datetime');
var serverKey = 'xxxxxxxxxxxxxpSELZBjQYwpZgmxxxxxxxxxxx';
var fcm = new FCM(serverKey);
//Initial function call:
exports.CheckDates = functions.https.onRequest((req, response) => {
// Get a database reference to our posts
var db = admin.database();
var ref = db.ref("records");
var userToken = '';
var itemExpires = '';
var itemName = '';
var reminded = '';
var itemCount = 0;
var counter = 1;
var itemFoundCount = 0;
var dt = dateTime.create();
var formatted = dt.format('m-d-Y');
ref.once("value", function (recordsSnapshot) {
recordsSnapshot.forEach(function (recordsSnapshot) {
var mainKey = recordsSnapshot.key;
recordsSnapshot.forEach(function (child) {
var key = child.key;
var value = child.val();
if (key == 'Account') {
userToken = value.userToken;
}
if (key == 'Items') {
recordsSnapshot.child("Items").forEach(function (itemsSnapshot) {
counter++;
if (itemFoundCount === 0) {
itemFoundCount = itemsSnapshot.numChildren();
}
var itemsChildkey = itemsSnapshot.key;
var itemsChildvalue = itemsSnapshot.val();
itemExpires = itemsChildvalue.itemExpires;
itemName = itemsChildvalue.itemName;
reminded = itemsChildvalue.reminded;
moment().format('YYYY-MM-DD');
var currentDate = moment();
var otherTime = moment(reminded);
if (typeof reminded !== 'undefined') {
if (currentDate.diff(otherTime, 'days') >= 30) {
if (currentDate.diff(itemExpires, 'days') <= 90) {
itemCount++;
console.log("Expire date is less than " +
currentDate + " by 90 days = " + (currentDate.diff(otherTime, 'days') <=
90));
db.ref("records/" + mainKey + "/Items/" +
itemsChildkey + '/reminded').set(formatted);
}
}
} else {
itemCount++;
db.ref("records/" + mainKey + "/Items/" + itemsChildkey +
`enter code here`'/reminded').set(formatted);
}
if (counter == itemFoundCount && itemCount > 0) {
console.log(itemFoundCount);
var message = {
to: userToken, // required fill with device token or
topics
notification: {
title: 'Item Expire Notification',
body: itemCount + ' is about to expire.'
}
};
//callback style
fcm.send(message, function (err, response) {
if (err) {
console.log("Something has gone wrong!");
} else {
console.log("Successfully sent with response: ",
response);
}
});
itemCount = 0;
itemFoundCount = 0;
counter = 1;
}
});
}
});
});
});
response.send(200, "ok");
}) // END exports.CheckDates
Obviously, I remove the Exports.CheckDates lines when I run it locally but it wasn't showing up at all without the exports on Firebase's console. It returns warnings locally and on Firebase console but it works on one and not the other.
Please disregard this question. The script adds the date that the last time a notification was sent to the item's "Reminded" key in the database... this prevents notifications from going out every day for the same thing..
It ran on my local machine, and did it's job but I couldn't figure out why it wouldn't run again... well, ha! At least we know that part works.

Resources