Basically, I'm trying to export the SendGrid event data to GCP storage using a cloud function in Node.js 16 using SendGrid's event Webhook functionality
I'm currently running (with example naming conventions);
const {BigQuery} = require('#google-cloud/bigquery');
const projectId = 'my_project';
const dataset = 'my_dataset';
const tableName = 'sendgrid_events';
//Initialize bigquery object
const bigQuery = new BigQuery({
projectId: projectId
});
exports.helloWorld = (req, res) => {
let message = req.query.message || req.body.message || 'Hello World!';
var timestamp = new Date().getTime()+'';
var data = req.body;
var rows = [];
for(var i=0; i< data.length; i++){
rows.push({
'hit_timestamp' : timestamp,
'email': data[i]['email'],
'event': data[i]['event'],
'category': data[i]['category']+ "",
'event_id': data[i]['sg_event_id'],
'event_payload' : JSON.stringify(data[i]),
});
}
send_to_bq(rows);
res.status(200).send(message);
};
function send_to_bq(rows){bigQuery
.dataset(dataset)
.table(tableName)
.insert(rows)
.then((data) => {
console.log(`Inserted rows : `, rows.length);
//console.log(data);
})
.catch(err => {
console.error('BQ ERROR:', err);
console.error('BQ ERROR:', err.response.insertErrors[1].error);
});
}
However, the data doesn't seem to pass into the table.
Any help is appreciated.
The solution is authentication, when deploying the cloud function setting it as unauthenticated means this boiler plate code works. However, if you require authentication then the permissions variable need to be added so google cloud can verify the data is being passed from the correct place.
Related
I have been given an assignment to fetch a JSON API, and also parse an XML feed URL and store their responses inside separate Firestore collections. I am not really good at cloud functions, but after lots of research, I have written the cloud function code below for the JSON API and it works well.
const functions = require("firebase-functions");
const axios = require("axios");
const admin = require("firebase-admin");
const api_token = "XXXXXXX";
const includes = "XXXXXX";
const url = "https://XXXXXXXXXXXXXX.com/?api_token=" + api_token + includes;
exports.allLeagues = functions.region('europe-west1').https.onRequest(async (req, res) => {
try {
let response = await axios.get(url);
var data = response.data.data;
for (let leagueData of data) {
await admin.firestore().collection("leagues").doc(leagueData.id.toString()).collection("all_data").doc(leagueData.id.toString()).set({
id : leagueData.id,
name : leagueData.name,
logo_path : leagueData.logo_path,
is_cup : leagueData.is_cup
});
}
console.log("Table complete...");
console.log("successful");
return res.status(200).json({ message: "successful" });
} catch(error) {
console.log("Error encountered: "+error);
return res.status(500).json({ error });
}
});
I am through with the JSON API. But for the XML feed, I don't know where to start. I have done lots of research to no avail. I found this on Stackoverflow but it doesn't address my need. Assuming this is my feed: https://www.feedforall.com/sample.xml , please how do I parse it and save the items inside Firestore?
Kindly help.
Thank you.
You can use rss-parser that can be used to fetch data from RSS feeds or parse from XML strings as shown below:
// npm install rss-parser
const Parser = require("rss-parser");
const parser = new Parser();
exports.rssFeedParser = functions.https.onRequest(
async (request, response) => {
const rssUrl = "https://www.feedforall.com/sample.xml";
const feed = await parser.parseURL(rssUrl);
const { items } = feed;
const batch = db.batch();
items.forEach((item) => {
const docRef = db.collection("rss").doc();
// restructure item if needed
batch.set(docRef, item);
});
await batch.commit();
response.send("Done");
}
);
Do note that you can add up to 500 documents only using Batched Writes as in the answer above. If your feed can return more than that, then you should create multiple batches of 500 or add them individually.
I have used the code from the Firebase documentation to schedule a backup of the data in my Firestore project in a bucket every 6 hours. See the link and the code here:
https://firebase.google.com/docs/firestore/solutions/schedule-export
const functions = require('firebase-functions');
const firestore = require('#google-cloud/firestore');
const client = new firestore.v1.FirestoreAdminClient();
// Replace BUCKET_NAME
const bucket = 'gs://BUCKET_NAME';
exports.scheduledFirestoreExport = functions.pubsub
.schedule('every 24 hours')
.onRun((context) => {
const projectId = process.env.GCP_PROJECT || process.env.GCLOUD_PROJECT;
const databaseName =
client.databasePath(projectId, '(default)');
return client.exportDocuments({
name: databaseName,
outputUriPrefix: bucket,
// Leave collectionIds empty to export all collections
// or set to a list of collection IDs to export,
// collectionIds: ['users', 'posts']
collectionIds: []
})
.then(responses => {
const response = responses[0];
console.log(`Operation Name: ${response['name']}`);
})
.catch(err => {
console.error(err);
throw new Error('Export operation failed');
});
});
Everything works well, my data is saved like I want to but nevertheless I am getting an error:
Error serializing return value: TypeError: Converting circular structure to JSON
Can someone tell me what I should change? Would be glad to get a hint.
I've been playing with the google sheets API in NodeJS for the past couple of days and I'm having some trouble. I took one of the examples from their docs and edited it to suit my needs, however at one point it started returning an error. The console logs:
The API returned an error: SyntaxError: Unexpected token in JSON at position 0
Here's my code:
function getApplies(auth) {
const sheets = google.sheets({version: 'v4', auth});
sheets.spreadsheets.values.get({
spreadsheetId: '1g5kYIIQy5f-UTSmOatTBPOto13ccfPvsJKQpxT6lhjE',
range: 'A2:I',
}, (err2, res) => {
if (err2) return console.log('The API returned an error: ' + err2);
const rows = res.data.values;
if (rows.length) {
//saving data here...
}
});
}
Any idea what I could be doing wrong here? Thanks!
From this link:
Go to Google Console
Create a new Credential of type service account key
Download your json file
npm install google-spreadsheet#2.0.3
var GoogleSpreadsheet = require('google-spreadsheet');
var creds = require('./client_secret.json');
var doc = new GoogleSpreadsheet('*spreadsheet ID*');
doc.useServiceAccountAuth(creds, function (err) {
doc.getRows(1, function (err, rows) {
if (err) {
console.log(err);
} else {
console.log(rows.length);
console.log(rows);
for (var i=0; i < rows.length; i++) {
console.log(rows[i].TITLE-NAME-OF-YOUR-FIRST-COLUMN, rows[i].TITLE-NAME-OF-YOUR-SECOND-COLUMN);
}
}
});
});
Replace the spreadsheet ID with the ID found in the spreadsheet URL.
Share the sheet with the service account email. It is in your json file.
Hope this helps!!
For v4 of googleapi use:
const { google } = require("googleapis");
const key = require("service_account_credentials.json");
const client = new google.auth.JWT(key.client_email, null, key.private_key, [
"https://www.googleapis.com/auth/spreadsheets",
]);
client.authorize(function (err, tokens) {
const gsapi = google.sheets({ version: "v4", auth: client });
const opt = {
spreadsheetId: "spreadsheetId",
range: "Sheet1!A:T",
};
gsapi.spreadsheets.values.get(opt).then(res => console.log(res));
});
You need to parse your json file like this, first:
const rows = JSON.parse(res.data.values);
I have been trying to get this to work, but am new to NodeJS. I suspect the issue is due to async, but am not familiar with how it works.
The idea behind this code is that it monitors a firebase database change and sends an email to the users. I am getting everything from the change snapshot, and using the values to check another table for user data. The request is not returning before the email gets sent and I am unsure why.
Edit I should specify that the email function sgMail is firing off before I get the results from the requests. I've tried putting a delay, but I am still not getting the result to return in time.
Here's my index.js
// The Cloud Functions for Firebase SDK to create Cloud Functions and setup triggers.
const functions = require('firebase-functions');
var requestify = require('requestify');
//SendGrid
const SENDGRID_API_KEY = functions.config().sendgrid.key;
const sgMail = require('#sendgrid/mail');
sgMail.setApiKey(SENDGRID_API_KEY);
// The Firebase Admin SDK to access the Firebase Realtime Database.
const admin = require('firebase-admin');
admin.initializeApp(functions.config().firebase);
exports.packingListEmail = functions.database.ref('Order/{orderID}')
.onUpdate(event => {
// Grab the current value of what was written to the Realtime Database.
const eventSnapshot = event.data;
//Here You can get value through key
var shipperInfo = eventSnapshot.child("fk_shipper_id").val();
var travelerInfo = eventSnapshot.child("fk_traveler_id").val();
//Print value of string
console.log(shipperInfo);
//Get Shipper Info
const shipperPath = 'https://shlep-me-f516e.firebaseio.com/User/'+shipperInfo+'.json';
requestify.get(shipperPath)
.then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
shipperResult = response.getBody();
console.log(shipperResult.email);
return shipperResult;
});
function getTravelerData() {
return new Promise(resolve => {
requestify.get('https://shlep-me-f516e.firebaseio.com/User/' + travelerInfo + '.json')
.then(function (response) {
resolve(response.getBody())
});
});
}
var TravelD = getTravelerData();
//Send an email
const msg = {
to: 'andrew#shlepme.com',
from: 'support#shlepme.com',
subject: 'New Follower',
// text: `Hey ${toName}. You have a new follower!!! `,
// html: `<strong>Hey ${toName}. You have a new follower!!!</strong>`,
// custom templates
templateId: 'd1ccfeb9-2e2d-4979-a3ca-c53975fe486e',
substitutionWrappers: ['%', '%'],
substitutions: {
'%shipper_name%': "Test",
'traveler_name': TravelD.name
// and other custom properties here
}
};
console.log('Sending email');
console.log(TravelD);
return sgMail.send(msg)
});
Any ideas? I have been trying to figure this out.
It seems that you need to understand about Promises first.
When you start using promises you will need to ALWAYS use them and chain one with the other.
So I would rewrite your code like this: (not tested)
// The Cloud Functions for Firebase SDK to create Cloud Functions and setup triggers.
const functions = require("firebase-functions");
var requestify = require("requestify");
//SendGrid
const SENDGRID_API_KEY = functions.config().sendgrid.key;
const sgMail = require("#sendgrid/mail");
sgMail.setApiKey(SENDGRID_API_KEY);
// The Firebase Admin SDK to access the Firebase Realtime Database.
const admin = require("firebase-admin");
admin.initializeApp(functions.config().firebase);
exports.packingListEmail = functions.database
.ref("Order/{orderID}")
.onUpdate(event => {
// Grab the current value of what was written to the Realtime Database.
const eventSnapshot = event.data;
//Here You can get value through key
var shipperInfo = eventSnapshot.child("fk_shipper_id").val();
var travelerInfo = eventSnapshot.child("fk_traveler_id").val();
//Print value of string
console.log(shipperInfo);
//Get Shipper Info
const shipperPath = "https://shlep-me-f516e.firebaseio.com/User/" + shipperInfo + ".json";
requestify.get(shipperPath)
.then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
var shipperResult = response.getBody();
console.log(shipperResult.email);
return shipperResult;
})
.then(function (shipperResult) {
//Send an email
const msg = {
to: "andrew#shlepme.com",
from: "support#shlepme.com",
subject: "New Follower",
// text: `Hey ${toName}. You have a new follower!!! `,
// html: `<strong>Hey ${toName}. You have a new follower!!!</strong>`,
// custom templates
templateId: "d1ccfeb9-2e2d-4979-a3ca-c53975fe486e",
substitutionWrappers: ["%", "%"],
substitutions: {
"%shipper_name%": "Test",
traveler_name: shipperResult.name
// and other custom properties here
}
};
console.log("Sending email");
console.log(shipperResult);
return sgMail.send(msg);
});
});
I am trying to do push notification through Functions in Firebase.
Here is my code in Node.JS
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp(functions.config().firebase);
exports.sendPushNotification = functions.database.ref('Received Downs/{owner}/{propID}')
.onCreate(event => {
// get the owner name and propID
var owner = event.params.owner;
var propID = event.params.propID;
// Log it
console.log('Owner: ' + owner + ' Property ID: ' + propID);
// Get the list of device notification tokens.
return admin.database().ref(`/users/${owner}`).once('value', snapshot => {
var ownerID = snapshot.val();
// This will find requester ID
return admin.database().ref(`/Received Downs/${owner}/${propID}`).once('value', snapshot => {
// First will find the property the requester downed
var property = snapshot.val();
// Find the requester's name
return admin.database().ref('/Users Info/' + property.downedBy).once('value', snapshot => {
// Requester's ID
var downedBy = snapshot.val();
// Notification details.
const payload = {
notification: {
title: 'You have a new request!',
body: `${downedBy.name} is now following you.`,
sound: 'default'
}
};
// Listing all tokens. (the function save the keys of given variable)
// const tokens = Object.keys(getDeviceTokens.val());
// var fcmToken = "dzJLM-JdIt8:APA91bHBJJP6t3Z0_T7kEFDrLLsu5T_NpYsR6QmJz2EJhpK88SV1ZfemoyCtC_6hl3_0sCPdzkvlQFoAFhlWn4xTQOY3k5P8JMvdYFyeNBN1lHceQtytE0y-9oTP6qgKspi9p9E8V9dB";
// Send to all tokens of a device
admin.messaging().sendToDevice(ownerID.token, payload)
.then(response => {
console.log("Successfully sent message:", response);
}).catch(function(error) {
console.log("Error sending message:", error);
});
})
})
})
})
And here is what I got in LOGS at Firebase Functions
When I used a variable that has fem token , typed, it works fine, but not when i fetched from Firebase Realtime Database. Anyone could tell me why?
The problem I had the wrong path return admin.database().ref(/users/${owner})