Node.js Firestore forEach collection query cannot populate associative array - node.js

In this simplified example, associative array A cannot be populated in a Node.js Firestore query---it's as if there is a scoping issue:
var A = {};
A["name"] = "nissa";
firestore.collection("magic: the gathering")
.get()
.then(function(query) {
query.forEach(function(document) {
A[document.id] = document.id;
console.log(A);
});
})
.catch(function(error) {
});
console.log(A);
Console output:
{ name: 'nissa' } < last console.log()
{ name: 'nissa', formats: 'formats' } < first console.log() (in forEach loop)
{ name: 'nissa', formats: 'formats', releases: 'releases' } < second console.log() (in forEach loop)
Grateful for any assistance, please request for further detail if needed.

Data is loaded from Firestore asynchronously, and while that is happening, your main code continues to run.
It's easiest to see what that means by placing a few logging statements:
console.log("Starting to load data");
firestore.collection("magic: the gathering")
.get()
.then(function(query) {
console.log("Got data");
});
console.log("After starting to load data");
When you run this code, it prints:
Starting to load data
After starting to load data
Got data
This is probably not the order that you expected the logging to be in. But it is actually working as intended, and explains the output you see. By the time your last console.log(A); runs, the data hasn't been loaded yet, so A is empty.
The solution is simple, but typically takes some time to get used to: all code that needs the data from the database must be inside the callback, or be called from there.
So something like this:
var A = {};
A["name"] = "nissa";
firestore.collection("magic: the gathering")
.get()
.then(function(query) {
query.forEach(function(document) {
A[document.id] = document.id;
});
console.log(A);
})
Also see:
Array of JSON object is not empty but cannot iterate with foreach, show zero length
NodeJS, Firestore get field
Unable to add Google markers inside a loop, a more complex problem, calling multiple asynchronous API
scope issue in javascript between two Functions, which also shows using the more modern async and await keywords, instead of then()
How to get data from firestore DB in outside of onSnapshot, which uses an onSnapshot listener instead of get()

Related

Read whole node from Firebase Database

Categories:
Database:
I am trying to read the whole selected nodes (categories) and then filter through them and return them in a cloud callable function. How can I get all the data and then filter through it? When I try to log the array, it is empty.
exports.getRecipes = functions.region('europe-west1').https.onCall((data, context) => {
categories = data.categories;
eventsData = [];
for (let i = 0; i < categories.length; i++) {
admin.database().ref(categories[i]).once('value', (data) => {
eventsData.push(data.val());
});
}
console.log(eventsData);
return "hello";
});
Is there any other way getting the whole node with admin.database().ref(), without .once()?
Data is loaded from Firebase (and most modern cloud APIs) asynchronously, and while the data is being loaded the rest of your code continues to run. This is easiest to see if you add some logging to your code:
console.log("Before starting to load data")
for (let i = 0; i < categories.length; i++) {
admin.database().ref(categories[i]).once('value', (data) => {
console.log("Got data")
});
}
console.log("After starting to load data")
When you run this code, the output is:
Before starting to load data
After starting to load data
Got data
Got data
...
This is probably the order that you expected the output to be in, but it explains why your console.log(eventsData) shows an empty array: by the time you log the array, none of the data has been loaded yet and eventsData.push(data.val()) hasn't run.
The solution for this is always the same: any code that needs the data from the asynchronous call, needs to either be directly inside the callback, be called from there, or be otherwise synchronized.
Since you're loading multiple nodes we'll use Promise.all here to wait for all of those nodes to have been loaded.
exports.getRecipes = functions.region('europe-west1').https.onCall((data, context) => {
const categories = data.categories;
const eventsData = Promise.all(categories.map((category) => {
return admin.database().ref(categories[i]).once('value').then((snapshot) => {
return snapshot.val();
});
});
return eventsData;
});
Since we now return a promise, Cloud Functions will wait for that promise to resolve, and then return the resulting value to the caller.
I recommend learning more about promises and asynchronous behavior at:
The Firebase documentation on terminating functions: Sync, async, and promises.
Doug's video series on Learn JavaScript Promises (Pt.1) with HTTP Triggers in Cloud Functions
The MDN pages on Asynchronous JavaScript

Promise inside loop in nodejs isn't working as expected

I have array of events, for each value there may/may not query fires to get data.
var eventTypes = [["write","write","write","write"],["write","write","read","write"]];
_.each(eventTypes,function(obj){
gettime(obj);
});
gettime=function(events){
var resultArray = [];
_.each(events,function(event){
if(event === "read"){
resultArray.push(makesqlquery(event));
}else{
resultArray.push({"time":current_time})
}
});
q.all(resultArray).then(function(finalResult){
insertIntoPostgreSQL(finalResult);
});
}
makesqlquery = function(event){
var deferred = q.defer();
sql.query("select time from events where eventtype ="+ event,
function(result,error){
if(error){
deferred.reject(error);
}else{
deferred.resolve({time:result.time});
}
});
return deferred.promise;
}
In the above code I'm able to push 1st set of data(["write","write","write","write"]) into postgresql database but not 2nd set(["write","write","read","write"]). Whenever I get read event in a set, I'm getting empty object. What would be the problem? For the above example I should have 8 records in postgresql, but I see only first array's four data.
More Info: insertIntoPostgreSQL() function will get list of objects and insert each object into database. This operation is workin fine.
I tried use two console stmt as
console.log("sql result:"result);
deferred.resolve({time:result.time});
and
console.log("Before Insert:"JSON.stringigy(resultArray));
q.all(resultArray).then(function(finalResul‌​t){
I get result as in the following order.
Before insert:[{"source":{}},{"source":{}},{"source":{}},{"source":{}}]
Before insert:[{}]
sql result:{time:"2015-07-10 00:00:00"}
As Roamer-1888 mentioned in the comments, definitely add an error handler for your Q.all. The basic structure of your promise seems to be fine, the error is somewhere else.
It looks like the result in your sql.query callback is not quite what you expect, as it is read as undefined. Because of an error there, Q.all is not getting resolved, therefore nothing gets added to your database.

node.js for loop execution in a synchronous manner

I have to implement a program in node.js which looks like the following code snippet. It has an array though which I have to traverse and match the values with database table entries. I need to wait till the loop ends and send the result back to the calling function:
var arr=[];
arr=[one,two,three,four,five];
for(int j=0;j<arr.length;j++) {
var str="/^"+arr[j]+"/";
// consider collection to be a variable to point to a database table
collection.find({value:str}).toArray(function getResult(err, result) {
//do something incase a mathc is found in the database...
});
}
However, as the str="/^"+arr[j]+"/"; (which is actually a regex to be passed to find function of MongoDB in order to find partial match) executes asynchronously before the find function, I am unable to traverse through the array and get required output.
Also, I am having hard time traversing through array and send the result back to calling function as I do not have any idea when will the loop finish executing.
Try using async each. This will let you iterate over an array and execute asynchronous functions. Async is a great library that has solutions and helpers for many common asynchronous patterns and problems.
https://github.com/caolan/async#each
Something like this:
var arr=[];
arr=[one,two,three,four,five];
asych.each(arr, function (item, callback) {
var str="/^"+item+"/";
// consider collection to be a variable to point to a database table
collection.find({value:str}).toArray(function getResult(err, result) {
if (err) { return callback(err); }
// do something incase a mathc is found in the database...
// whatever logic you want to do on result should go here, then execute callback
// to indicate that this iteration is complete
callback(null);
});
} function (error) {
// At this point, the each loop is done and you can continue processing here
// Be sure to check for errors!
})

Asynchronous Database Queries with PostgreSQL in Node not working

Using Node.js and the node-postgres module to communicate with a database, I'm attempting to write a function that accepts an array of queries and callbacks and executes them all asynchronously using the same database connection. The function accepts a two-dimensional array and calling it looks like this:
perform_queries_async([
['SELECT COUNT(id) as count FROM ideas', function(result) {
console.log("FUNCTION 1");
}],
["INSERT INTO ideas (name) VALUES ('test')", function(result) {
console.log("FUNCTION 2");
}]
]);
And the function iterates over the array, creating a query for each sub-array, like so:
function perform_queries_async(queries) {
var client = new pg.Client(process.env.DATABASE_URL);
for(var i=0; i<queries.length; i++) {
var q = queries[i];
client.query(q[0], function(err, result) {
if(err) {
console.log(err);
} else {
q[1](result);
}
});
}
client.on('drain', function() {
console.log("drained");
client.end();
});
client.connect();
}
When I ran the above code, I expected to see output like this:
FUNCTION 1
FUNCTION 2
drained
However, the output bizarrely appears like so:
FUNCTION 2
drained
FUNCTION 2
Not only is the second function getting called for both requests, it also seems as though the drain code is getting called before the client's queue of queries is finished running...yet the second query still runs perfectly fine even though the client.end() code ostensibly killed the client once the event is called.
I've been tearing my hair out about this for hours. I tried hardcoding in my sample array (thus removing the for loop), and my code worked as expected, which leads me to believe that there is some problem with my loop that I'm not seeing.
Any ideas on why this might be happening would be greatly appreciated.
The simplest way to properly capture the value of the q variable in a closure in modern JavaScript is to use forEach:
queries.forEach(function(q) {
client.query(q[0], function(err, result) {
if(err) {
console.log(err);
} else {
q[1](result);
}
});
});
If you don't capture the value, your code reflects the last value that q had, as the callback function executed later, in the context of the containing function.
forEach, by using a callback function isolates and captures the value of q so it can be properly evaluated by the inner callback.
A victim of the famous Javascript closure/loop gotcha. See my (and other) answers here:
I am trying to open 10 websocket connections with nodejs, but somehow my loop doesnt work
Basically, at the time your callback is executed, q is set to the last element of the input array. The way around it is to dynamically generate the closure.
It will be good to execute this using async module . It will help you to reuse the code also . and will make the code more readable . I just love the auto function provided by async module
Ref: https://github.com/caolan/async

Returning an Array using Firebase

Trying to find the best-use example of returning an array of data in Node.js with Q library (or any similar library, I'm not partial) when using Firebase .on("child_added");
I've tried using Q.all() but it never seems to wait for the promises to fill before returning. This is my current example:
function getIndex()
{
var deferred = q.defer();
deferred.resolve(new FirebaseIndex( Firebase.child('users').child(user.app_user_id).child('posts'), Firebase.child('posts') ) );
return deferred.promise;
}
function getPost( post )
{
var deferred = q.defer();
deferred.resolve(post.val());
return deferred.promise;
}
function getPosts()
{
var promises = [];
getIndex().then( function (posts) {
posts.on( 'child_added', function (_post) {
promises.push( getPost(_post) );
});
});
return q.all(promises);
}
The problem occurs in getPosts(). It pushes a promise into your array inside an async function--that won't work since q.all is called before the promise objects have been added.
Also, child_added is a real-time event notification. You can't use that as a way to grab "all of the data" because there is no such thing as "all"; the data is constantly changing in real-time environments. FirebaseIndex is also using child_added callbacks internally, so that's not going to work with this use case either.
You can grab all of the posts using the 'value' callback (but not a specific subset of records) as follows:
function getPosts() {
var def = q.defer();
Firebase.child('users').once('value', function(snap) {
var records = [];
snap.forEach(function(ss) {
records.push( ss.val() );
});
def.resolve(records);
});
return def.promise;
}
But at this point, it's time to consider things in terms of real-time environments. Most likely, there is no reason "all" data needs to be present before getting to work.
Consider just grabbing each record as they come in and appending them to whatever DOM or Array where they need to be stored, and working from an event driven model instead of a GET/POST centered approach.
With luck, you can bypass this use case entirely.

Resources