I have a simple get request built using node, express to retrieve data from datastore. I am not able to get back the results. 'get' request async call is stuck. Not sure what is happening.
const express = require('express');
const {Datastore} = require('#google-cloud/datastore');
const app = express();
// Your Google Cloud Platform project ID
const projectId = 'xxx';
// Creates a client
const datastore = new Datastore({
projectId: projectId,
keyFilename: '/Masters-LUC/spring-2019/internship/service-keys/xxx.json'
});
const query = datastore
.createQuery('approvals')
.filter('status', '=', 'yes');
app.get("/api/get", (req, res, next) => {
query.run().then(([documents]) => {
documents.forEach(doc => console.log(doc));
});
});
module.exports = app;
I re-wrote the same using async function. The below is working. Why not the above?
// Retrieve data from datastore
async function quickStart() {
// Your Google Cloud Platform project ID
const projectId = 'xxx';
// Creates a client
const datastore = new Datastore({
projectId: projectId,
keyFilename: '/Masters-LUC/spring-2019/internship/service-
keys/xxx.json'
});
const query = datastore
.createQuery('approvals')
.filter('status', '=', 'yes');
const [approvals] = await datastore.runQuery(query);
console.log('Tasks:');
approvals.forEach(task => console.log(task));
}
quickStart().catch(console.error);
The two things I notice that is different between your two functions. In the first you reuse the query object across function invocations. Query objects should not be reused.
The second thing I notice is that you don't use res that's passed into your function parameter to app.get().
Modified working code -
app.get("/api/approvals/", (req, res, next) => {
const query = datastore
.createQuery('approvals');
query.run().then((approvals) => {
approvals.forEach(appr => console.log(appr)); // This is used to log results on console for verification
// loading results on the response object to be used later by client
res.status(200).json(
{
message: "Request was processed successfully!",
approvals : approvals
}
);
})
})
Related
I have been given an assignment to fetch a JSON API, and also parse an XML feed URL and store their responses inside separate Firestore collections. I am not really good at cloud functions, but after lots of research, I have written the cloud function code below for the JSON API and it works well.
const functions = require("firebase-functions");
const axios = require("axios");
const admin = require("firebase-admin");
const api_token = "XXXXXXX";
const includes = "XXXXXX";
const url = "https://XXXXXXXXXXXXXX.com/?api_token=" + api_token + includes;
exports.allLeagues = functions.region('europe-west1').https.onRequest(async (req, res) => {
try {
let response = await axios.get(url);
var data = response.data.data;
for (let leagueData of data) {
await admin.firestore().collection("leagues").doc(leagueData.id.toString()).collection("all_data").doc(leagueData.id.toString()).set({
id : leagueData.id,
name : leagueData.name,
logo_path : leagueData.logo_path,
is_cup : leagueData.is_cup
});
}
console.log("Table complete...");
console.log("successful");
return res.status(200).json({ message: "successful" });
} catch(error) {
console.log("Error encountered: "+error);
return res.status(500).json({ error });
}
});
I am through with the JSON API. But for the XML feed, I don't know where to start. I have done lots of research to no avail. I found this on Stackoverflow but it doesn't address my need. Assuming this is my feed: https://www.feedforall.com/sample.xml , please how do I parse it and save the items inside Firestore?
Kindly help.
Thank you.
You can use rss-parser that can be used to fetch data from RSS feeds or parse from XML strings as shown below:
// npm install rss-parser
const Parser = require("rss-parser");
const parser = new Parser();
exports.rssFeedParser = functions.https.onRequest(
async (request, response) => {
const rssUrl = "https://www.feedforall.com/sample.xml";
const feed = await parser.parseURL(rssUrl);
const { items } = feed;
const batch = db.batch();
items.forEach((item) => {
const docRef = db.collection("rss").doc();
// restructure item if needed
batch.set(docRef, item);
});
await batch.commit();
response.send("Done");
}
);
Do note that you can add up to 500 documents only using Batched Writes as in the answer above. If your feed can return more than that, then you should create multiple batches of 500 or add them individually.
I have a Rest API where a user can create a list and then put list items in it (for a quiz). My schema structure is this:
const verbListSchema = {
title: String,
verbs: [{verb: String}]
};
Here are the url endpoints I have so far:
/lists/ (gets back all the lists)
/lists/verbs (gets all the verbs from all the lists)
My question is - I want to get, post, patch and delete a specific list using its id, like /lists?list_id=123/verbs or /lists/123/verbs and then one step further to get individual verbs I want to do something like /lists/123/verbs/124 or /lists?list_id=123/verbs?verb_id=124 the last doesn't work because it counts the last endpoint as a query param.
In terms of best practice what's the best way to do this. I could do something like this (I use express.js)?
app.[request-type]("/lists") {...}
app.[request-type]("/lists/:list_id") {...}
app.[request-type]("/lists/:list_id/verbs") {...}
app.[request-type]("/lists/:list_id/verbs/:verb_id") {...}
and then if I want to retrieve all the lists, not just a specific one I can check if the list_id is "all" like, /lists/all/verbs?
And here is my code so far:
const express = require("express");
const verbRouter = require("./verbRoutes");
const router = express.Router();
const VerbList = require("../../verb-list-db");
const isOriginal = async (req,res,next) => {
const listExists = await VerbList.find({title: req.body.listTitle})
if (listExists.length > 0 ) return res.status(400).json({message: "list already exists"});
next();
};
router.route("/")
.get(async (req,res,next) => {
try {
const listId = req.query.list_id;
if (listId) return res.json(await VerbList.find({_id: listId}));
const lists = await VerbList.find({});
res.json(lists);
} catch(err) {next(err)}
})
.post(isOriginal, async (req,res,next) => {
const newList = new VerbList({ // creates a new list
title: req.body.listTitle
})
newList.save()
.then(() => {return res.send("list successfully added!")})
.catch(err => next(err));
})
.patch(isOriginal, async (req,res,next) => {
try {
const listId = req.query.list_id;
if (!listId) throw new Error("you must have a list_id to patch!")
res.json(await VerbList.updateOne({_id: req.query.list_id}, {title: req.body.listTitle}))
} catch(err) {next(err)}
})
.delete(async (req,res,next) => {
try {
const listId = req.query.list_id;
if (!listId) throw new Error("you must have a list_id to delete!");
res.json(await VerbList.deleteOne({_id: req.query.list_id}))
} catch(err) {next(err)}
})
Any suggestions would be appreciated :)
You can try to modularize your express code by separating your /lists routes from your main server.js (or index.js) file.
index.js
const express = require('express');
const app = express();
//Now lets route all the API requests that start with '/list' to a file called lists.js
app.use('/lists', require('/path/to/lists.js')
app.listen(3000, () => console.log(`\nServer started on port 3000`))
lists.js
const express = require('express');
const router = express.Router();
// now you know all the requests in this file would be for /list so lets implement a router for fetching all the lists
router.get('/', async(req, res) => {
*** add all your logic for /lists endpoints here**
res.status(200).json(lists_json_response); //send a response back to your client
})
//you can create as many endpoints as you want in this file for endpoints that start with '/lists'
router.[request-method]("/lists/:list_id") {...} // endpoint for requesting a specific list
router.[request-method]("/:list_id/verbs") {...} //endpoint for requesting all the verbs for a specific list
router.[request-method]("/lists/all/verbs") {...} // all the verbs in all the lists
module.exports = router;
Also you cant put query parameters in the middle of an endpoint. if it is going to be a variable that you need and its not in the end of the URL, you have to pass it as a param, e.g.:
instead of doing /lists?list_id=123/verbs?verb_id=124, you can do something like /lists/123/verbs/124 to look for the verb with id 124 in a list with id 123.
so to listen to a request to this endpoint, you can design another endpoint in your lists.js file like this:
router[request-method].('/:list_id/verb/:verb_id', async(req, res)=> {
var list_id = req.params.list_id
var verb_id = req.params.verb_id
***
now use the list_id and verb_id to query the requested data and send a response back to the client
***
})
I have used the code from the Firebase documentation to schedule a backup of the data in my Firestore project in a bucket every 6 hours. See the link and the code here:
https://firebase.google.com/docs/firestore/solutions/schedule-export
const functions = require('firebase-functions');
const firestore = require('#google-cloud/firestore');
const client = new firestore.v1.FirestoreAdminClient();
// Replace BUCKET_NAME
const bucket = 'gs://BUCKET_NAME';
exports.scheduledFirestoreExport = functions.pubsub
.schedule('every 24 hours')
.onRun((context) => {
const projectId = process.env.GCP_PROJECT || process.env.GCLOUD_PROJECT;
const databaseName =
client.databasePath(projectId, '(default)');
return client.exportDocuments({
name: databaseName,
outputUriPrefix: bucket,
// Leave collectionIds empty to export all collections
// or set to a list of collection IDs to export,
// collectionIds: ['users', 'posts']
collectionIds: []
})
.then(responses => {
const response = responses[0];
console.log(`Operation Name: ${response['name']}`);
})
.catch(err => {
console.error(err);
throw new Error('Export operation failed');
});
});
Everything works well, my data is saved like I want to but nevertheless I am getting an error:
Error serializing return value: TypeError: Converting circular structure to JSON
Can someone tell me what I should change? Would be glad to get a hint.
I am trying to get users data from firebase but I keep receive 'CANNOT GET /' error.
//firebase init
const functions = require('firebase-functions');
const admin = require("firebase-admin");
admin.initializeApp();
//express and cors init
const express = require('express');
const cors = require('cors');
//middleware init
const app = express();
app.use(cors());
const database = admin.database();
app.get("/users", function (request, response) {
return database.ref('/users').on("value", snapshot => {
return response.status(200).send(snapshot.val());
}, error => {
console.error(database);
return response.status(500).send(err);
})
});
exports.users = functions.https.onRequest(app);
I also try to use another function from another website I refer to get at least the username but it returns error.
exports.users = functions.https.onRequest((req, res) => {
database().ref('/merchants').once('value').then(function(snapshot) {
var username = snapshot.val().username;
res.status(200).send(username);
});
});
Problem 1: Exporting express applications
When you export an express app through a Cloud Function, the paths become relative to the exported function name. You exported your app as exports.users which sets the root path of your function to /users and to call it, you would visit https://us-central1-<project-id>.cloudfunctions.net/users.
However, because you defined a route handler for /users (using app.get("/users", ...)) as well, you added a handler for https://us-central1-<project-id>.cloudfunctions.net/users/users instead.
The error Cannot GET / is thrown because when you call the function at https://us-central1-<project-id>.cloudfunctions.net/users, the relative URL is set as "/", which you haven't configured a handler for (using app.get("/", ...)).
So to fix your code above, change
app.get("/users", function (request, response) {
to
app.get("/", function (request, response) {
Problem 2: Username queries
The issue could be as simple as that you call database() instead of admin.database().
However, the purpose of this query is unclear, so I will assume that you have a merchant's ID and you are trying to get the username of the owner of that particular merchant.
This would mean a data structure similar to:
{
"users": {
"somePerson": { ... },
"otherPerson: { ... }
},
"merchants": {
"reallyGreatGuyInc": {
"username": "somePerson",
...
},
"realShadyPeopleInc": {
"username": "otherPerson",
...
}
}
}
If you exported a function called getMerchantUsername where you pass in the merchant ID via a GET parameter and call it at the URL https://us-central1-<project-id>.cloudfunctions.net/getMerchantUsername?id=reallyGreatGuyInc, you would define it using the following code:
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
exports.getMerchantUsername = functions.https.onRequest((req, res) => {
if (!req.query.id) {
// '?id=' is required
return res.status(400).send("Missing 'id' parameter");
} else if (/[\u0000-\u001F\u007F\.\$#/\[\]\\]/.test(req.query.id)) {
// check id for invalid characters
return res.status(400).send("Invalid 'id' parameter");
}
// run query
admin.database().ref(`/merchants/${req.query.id}/username`).once('value')
.then((snapshot) => {
var username = snapshot.val();
res.status(200).send(username);
})
.catch((error) => {
console.error(error);
res.status(500).send('Request failed.');
});
});
Notes:
The test for invalid characters is based on the list in the RTDB: Structure data documentation. This somewhat prevents looking at other database values.
If you only need the value of username, rather than request the entire merchant object and parse it, request only the username as in the example above where I used `/merchants/${req.query.id}/username`.
I have a question for dialogflow. I want to know if it's possible to have the agent create new fields or tables in the firebase database (firestore or realtime) All the code I find is about changing the values of a table and not creating them.
I do not know where to start, I've done integrations with the server and everything is working.
function writeToDb (agent) {
const databaseEntry = agent.parameters.databaseEntry;
const dialogflowAgentRef = db.collection('dialogflow').doc('agent');
return db.runTransaction(t => {
t.set(dialogflowAgentRef, {entry: databaseEntry});
return Promise.resolve('Write complete');
I need a explication to create new tables or fields by the agent
Google's Dialogflow Firestore sample on Github demonstrates how to connect Dialogflow to the Firestore database.
Check out the writeToDb() function below, and remember to require the same dependencies:
'use strict';
const functions = require('firebase-functions');
const admin = require('firebase-admin');
const {WebhookClient} = require('dialogflow-fulfillment');
process.env.DEBUG = 'dialogflow:*'; // enables lib debugging statements
admin.initializeApp(functions.config().firebase);
const db = admin.firestore();
exports.dialogflowFirebaseFulfillment = functions.https.onRequest((request, response) => {
const agent = new WebhookClient({ request, response });
function writeToDb (agent) {
// Get parameter from Dialogflow with the string to add to the database
const databaseEntry = agent.parameters.databaseEntry;
// Get the database collection 'dialogflow' and document 'agent' and store
// the document {entry: "<value of database entry>"} in the 'agent' document
const dialogflowAgentRef = db.collection('dialogflow').doc('agent');
return db.runTransaction(t => {
t.set(dialogflowAgentRef, {entry: databaseEntry});
return Promise.resolve('Write complete');
}).then(doc => {
agent.add(`Wrote "${databaseEntry}" to the Firestore database.`);
}).catch(err => {
console.log(`Error writing to Firestore: ${err}`);
agent.add(`Failed to write "${databaseEntry}" to the Firestore database.`);
});
}
function readFromDb (agent) {
// Get the database collection 'dialogflow' and document 'agent'
const dialogflowAgentDoc = db.collection('dialogflow').doc('agent');
// Get the value of 'entry' in the document and send it to the user
return dialogflowAgentDoc.get()
.then(doc => {
if (!doc.exists) {
agent.add('No data found in the database!');
} else {
agent.add(doc.data().entry);
}
return Promise.resolve('Read complete');
}).catch(() => {
agent.add('Error reading entry from the Firestore database.');
agent.add('Please add a entry to the database first by saying, "Write <your phrase> to the database"');
});
}
// Map from Dialogflow intent names to functions to be run when the intent is matched
let intentMap = new Map();
intentMap.set('ReadFromFirestore', readFromDb);
intentMap.set('WriteToFirestore', writeToDb);
agent.handleRequest(intentMap);
});