Firestore Real Time updates connection in NodeJS - node.js

I'm developing a NodeJS web app to receive Real Time updates from Firestore DB through Admin SDK.
This is the init code for the Firestore object. It's executed just once, when the app is deployed (on AWS Elastic Beanstalk):
const admin = require('firebase-admin');
var serviceAccount = require('./../key.json');
admin.initializeApp({
credential: admin.credential.cert(serviceAccount)
});
var db = admin.firestore();
FUNC.firestore = db;
Then I use this firestore object in a websocket comunication to send realtime updates to browser. The idea is to use the server as a proxy between browser and Firestore.
socket.on('open', function (client) {
var query = FUNC.firestore.collection("notifications").doc(client.user.id.toString()).collection("global");
query.onSnapshot(querySnapshot => {
querySnapshot.docChanges().forEach(change => {
client.send({ id: change.doc.id, body: change.doc.data(), type: change.type });
});
}, err => {
console.log(`Encountered error: ${err}`);
});
});
socket.on('close', function (client) {
var unsub = FUNC.firestore.collection("notifications").doc(client.user.id.toString()).collection("global").onSnapshot(() => {
});
unsub();
});
It works well for a while, but after few hours the client stop receiving onSnapshot() updates, and after a while the server log the error: Encountered error: Error: 10 ABORTED: The operation was aborted.
What's wrong? Should I initialized firestore on each connection? Is there some lifecycle mistake?
Thank you
EDIT (A very bad solution)
I've tried to create a single firebase-admin app instance for each logged user and changed my code in this way
const admin = require('firebase-admin');
var serviceAccount = require('./../key.json');
admin.initializeApp({
credential: admin.credential.cert(serviceAccount)
});
FUNC.getFirestore = function (user) {
try {
user.firebase = admin.app(user.id.toString());
return user.firebase.firestore();
} catch(e) {
//ignore
}
var app = admin.initializeApp({
credential: admin.credential.cert(serviceAccount)
}, user.id.toString());
user.firebase = app;
return user.firebase.firestore();
}
FUNC.removeFirebase = function (user) {
if (user.firebase) {
user.firebase.delete();
}
}
And then socket listeners:
self.on('open', function (client) {
var query = FUNC.getFirestore(client.user).collection("notifications").doc(client.user.id.toString()).collection("global");
query.onSnapshot(querySnapshot => {
querySnapshot.docChanges().reverse();
querySnapshot.docChanges().forEach(change => {
client.send({ id: change.doc.id, body: change.doc.data(), type: change.type });
});
}, err => {
console.log(`Encountered error: ${err}`);
});
});
self.on('close', function (client) {
var unsub = FUNC.getFirestore(client.user).collection("notifications").doc(client.user.id.toString()).collection("global").onSnapshot(() => {
});
unsub();
FUNC.removeFirebase(client.user);
});
So when a client disconnect for a reason the server removes its firebase app, it works, but I've noticed a huge memory leak on server, I need some help

UPDATED ANSWER
After many reaserch I've understand that this kind of approach is wrong. Of course, the old answer could be a workaround but is not the real solution of the problem, because Firestore was not designed to do something like: Firestore <--(Admin SDK)--> Server <--(WebSocket)--> Client.
In order to create the best comunication I have understand and applied Firestore Security Rules (https://firebase.google.com/docs/firestore/security/get-started) together with Custom token generation (https://firebase.google.com/docs/auth/admin/create-custom-tokens). So the correct flow is:
Client login request --> Server + Admin SDK generate custom auth token and return to client
Then, the real time comunication will be only between Client and Firestore itself, so: Client + Custom Auth Token <--(Firebase JS SDK)--> Firestore DB
As you can see, the server is not involved anymore in real-time comunication, but client receive updates directly from Firestore.
OLD ANSWER
Finally I can answer from myself. First of all the second solution I've tried is a very bad one, because each new app created through Admin SDK is stored in RAM, with 20/30 users the app reaches more then 1GB of RAM, absolutely unacceptable.
So the first implementation was the better solution, anyway I've wrong the register/unregister onSnapshot listener lifecycle. Each onSnapshot() call returns a different function, even if called on the same reference. So, instead of close the listener when socket close, I opened another one. This is how should be:
socket.on('open', function (client) {
var query = FUNC.firestore.collection("notifications").doc(client.user.id.toString()).collection("global");
client.user.firestoreUnsub = query.onSnapshot(querySnapshot => {
querySnapshot.docChanges().forEach(change => {
client.send({ id: change.doc.id, body: change.doc.data(), type: change.type });
});
}, err => {
console.log(`Encountered error: ${err}`);
});
});
socket.on('close', function (client) {
client.user.firestoreUnsub();
});
After almost 48h, listeners still works without problems and no memory leaks occurs.

Related

Function execution took 540029 ms, finished with status: 'timeout'

I have created a cloud function that connects to an MQTT broker I have used a third-party MQTT broker (Mosquito MQTT broker), and sends the data to the Firebase real-time database every time the MQTT broker receives data from the machine. I am using the GCP console for writing and deploying the function. I successfully deployed the function without any errors, however, when I test it from the GCP console, it starts sending data but stops after the time specified in the timeout. I have tried timeout values from 60 to 540 seconds, but it still stops after the specified time. I have also increased the allocated memory, but it hasn't resolved the issue and I keep getting the same timeout error.
This is my code
const Admin = require("firebase-admin");
const functions = require("firebase-functions");
const mqtt = require('mqtt');
const clientId = 'mqtt_googleserver_********7'
const topic = '#'
const serviceAccount = require("./service.json");
Admin.initializeApp({
credential: Admin.credential.cert(serviceAccount),
databaseURL: "https://***************firebaseio.com/"
});
exports.rtdb_mains = functions.https.onRequest((_request, _response) => {
const client = mqtt.connect('mqtt://**.**.**.****.***',{
clientId,
clean: true,
connectTimeout: 4000,
username: '******',
password: '********',
reconnectPeriod: 1000,
});
const db = Admin.database();
client.addListener('connect', () => {
console.log('Connected');
client.subscribe([topic], { qos: 1 });
console.log(`Subscribe to topic '${topic}'`);
});
client.on('message', async (topic, payload) => {
console.log('Received Message:', topic, payload.toString());
if (payload.toString() !== "" && topic !== "") {
const ref = db.ref("All_machines");
const childref = ref.child(topic.toString());
await childref.set(payload.toString());
const topicDetails = topic.split("/");
const machineId = topicDetails[1];
const machineParameter = topicDetails[2];
if (machineParameter === "BoardID") {
const ref = db.ref(machineParameter);
await ref.set(machineId);
}
}
});
});
can anyone please help me with this problem.
You don't need to specify a service.json if you push the CF on firebase. You can directly use the default configuration.
You can do directly this :
admin.initializeApp();
Secondly, the way you use your MQTT implementation and the cloud function are not correct.
You are listenning and waiting for a message in a function that is trigger only by a POST or GET request.
I suggest to use the pub/sub api for doing such a thing and have a good implementation for sending / receiving messages.
In case of you really need to listen for message in your MQTT implementation, you will need another provider than Cloud Function or calling the native MQTT of Cloud Function
https://cloud.google.com/functions/docs/calling/pubsub
https://www.googlecloudcommunity.com/gc/Serverless/Can-a-cloud-function-subscribe-to-an-MQTT-topic/m-p/402965

Able to connect to redis but set/get times out

I'm trying to do a get() from my AWS Lambda (NodeJS) on ElastiCache Redis using node_redis client. I believe that I'm able to connect to redis but I'm getting Time out (Lambda 60 sec time out) when I'm trying to perform a get() operation.
I have also granted my AWS lambda Administrator access just to be certain that it's not a permissions issue. I'm hitting lambda by going to AWS console and clicking the Test button.
Here is my redisClient.js:
const util = require('util');
const redis = require('redis');
console.info('Start to connect to Redis Server');
const client = redis.createClient({
host: process.env.ElastiCacheEndpoint,
port: process.env.ElastiCachePort
});
client.get = util.promisify(client.get);
client.set = util.promisify(client.set);
client.on('ready',function() {
console.log(" subs Redis is ready"); //Can see this output in logs
});
client.on('connect',function(){
console.log('subs connected to redis'); //Can see this output in logs
})
exports.set = async function(key, value) {
console.log("called set!");
return await client.set(key, value);
}
exports.get = async function(key) {
console.log("called get!"); //Can see this output in logs
return await client.get(key);
}
Here's my index.js which calls the redisClient.js:
const redisclient = require("./redisClient");
exports.handler = async (event) => {
const params = event.params
const operation = event.operation;
try {
console.log("Checking RedisCache by calling client get") // Can see this output in logs
const cachedVal = await redisclient.get('mykey');
console.log("Checked RedisCache by calling client get") // This doesn't show up in logs.
console.log(cachedVal);
if (cachedVal) {
return {
statusCode: 200,
body: JSON.stringify(cachedVal)
}
} else {
const setCache = await redisclient.set('myKey','myVal');
console.log(setCache);
console.log("*******")
let response = await makeCERequest(operation, params, event.account);
console.log("CE Request returned");
return response;
}
}
catch (err) {
return {
statusCode: 500,
body: err,
};
}
}
This is the output (time out error message) that I get:
{
"errorMessage": "2020-07-05T19:04:28.695Z 9951942c-f54a-4b18-9cc2-119eed65e9f1 Task timed out after 60.06 seconds"
}
I have tried using Bluebird (changing get to getAsync()) per this: https://github.com/UtkarshYeolekar/promisify-redis-client/blob/master/redis.js but still got the same behavior.
I also changed the port to use a random value (like 8088) that I'm using to create client (to see the behavior of connect event for a failed connection) - in this case I still see a Timed Out error response but I don't see the subs Redis is ready and subs connected to redis in my logs.
Can anyone please point me in the right direction? I don't seem to understand why I'm able to connect to redis but the get() request times out.
I figured out the issue and posting here in case it helps anyone in future as the behavior wasn't very intuitive for me.
I had enabled AuthToken param while setting up my redis. I was passing the param to lambda with the environment variables but wasn't using it while sending the get()/set() requests. When I disabled the AuthToken requirement from redis configuration - Lambda was able to hit redis with get/set requests. More details on AuthToken can be found here: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticache-replicationgroup.html#cfn-elasticache-replicationgroup-authtoken

Socket.io: Other client only updates when being interacted

I'm trying to set up a realtime application using socket.io in Angular and node.js, which is not working as intended.
Whenever a client is making a new post, the other clients won't update until you interact with the client (e.g. clicking somewhere on the page, or clicking on the browsers tab).
However, having console open in the browser, I can see the new post in the console when I log the posts/objects - without the need to interact with the clients.
Angular:
import io from 'socket.io-client';
const socket = io('http://localhost:3000');
posts: Post[] = [];
...
// Inside ngOnInit:
socket.on('data123', (res) => {
console.log('Updating list..', res);
this.postService.getPosts();
this.postsSub = this.postService.getPostUpdateListener()
.subscribe((posts: Post[]) => {
this.posts = posts;
});
});
Displaying in the template:
<... *ngFor="let item of posts">
Inside PostsService:
getPosts() {
this.http.get<{ message: string, posts: Post[] }>('http://localhost:3000/api/posts')
.subscribe((postData) => {
this.posts = postData.posts;
this.postsUpdate.next([...this.posts]);
});
}
Node.js - this socket.io solution is not yet sending the actual list:
const io = socket(server);
io.sockets.on('connection', (socket) => {
console.log(`new connection id: ${socket.id}`);
sendData(socket);
})
function sendData(socket){
socket.emit('data123', 'TODO: send the actual updated list');
setTimeout(() => {
console.log('sending to client');
sendData(socket);
}, 3000);
}
What worked as intended:
Using setInterval instead "socket.on(..)" on the front-end gave the intended result, meaning the clients will update automatically without the need of interacting. I'm fully aware this solution is horrible, but I assume this pinpointing that it's something wrong with socket solution above in Angular part.
wait, every time when socket.on('data123', (res) => {... you are creating new subscribe? it's wrong way...you must create subscribe in your socket connect feature

node-soap service that uses database (dependency issues)

First of all, this is one of my first projects in Node.js so I'm very new to it.
I have a project I want to make that is a SOAP (I know, SOAP... backwards compatibility, huh?) interface that connects to an Oracle database.
So I have a WSDL describing what these functions look like (validation for addresses and stuff) and I have a connection to the database.
Now when using the SOAP npm module, you need to create a server and listen using a service that allows you to respond to requests. I have a separate file that contains my SOAP service but this service should do queries on the database to get its results.
How would I go about sort of 'injecting' my database service into my SOAP service so that whenever a SOAP call is done, it orchestrates this to the correct method in my database service?
This is what my code looks like:
databaseconnection.js
var oracledb = require('oracledb');
var dbConfig = require('../../config/development');
var setup = exports.setup = (callback) => {
oracledb.createPool (
{
user : dbConfig.user,
password : dbConfig.password,
connectString : dbConfig.connectString
},
function(err, pool)
{
if (err) { console.error(err.message); return; }
pool.getConnection (
function(err, connection)
{
if (err) {
console.error(err.message);
return callback(null);
}
return callback(connection);
}
);
}
);
};
databaseservice.js
var DatabaseService = function (connection) {
this.database = connection;
};
function doSomething(callback) {
if (!this.database) { console.log('Database not available.'); return; }
this.database.execute('SELECT * FROM HELP', function(err, result) {
callback(result);
});
};
module.exports = {
DatabaseService: DatabaseService,
doSomething: doSomething
};
soapservice.js
var myService = {
CVService: {
CVServicePort: {
countryvalidation: function (args, cb, soapHeader) {
console.log('Validating Country');
cb({
name: args
});
}
}
}
};
server.js
app.use(bodyParser.raw({type: function(){return true;}, limit: '5mb'}));
app.listen(8001, function(){
databaseconnection.setup((callback) => {
var temp = databaseservice.DatabaseService(callback);
soapservice.Init(temp);
var server = soap.listen(app, '/soapapi/*', soapservice.myService, xml);
databaseservice.doSomething((result) => {
console.log(result.rows.length, ' results.');
});
});
console.log('Server started');
});
How would I go about adding the databaseservice.doSomething() to the countryvalidation soap method instead of 'name: args'?
Also: I feel like the structure of my code is very, very messy. I tried finding some good examples on how to structure the code online but as for services and database connections + combining them, I didn't find much. Any comments on this structure are very welcome. I'm here to learn, after all.
Thank you
Dieter
The first thing I see that looks a little off is the databaseconnection.js. It should be creating the pool, but that's it. Generally speaking, a connection should be obtained from the pool when a request comes in and release when you're done using it to service that request.
Have a look at this post: https://jsao.io/2015/02/real-time-data-with-node-js-socket-io-and-oracle-database/ There are some sample apps you could have a look at that might help. Between the two demos, the "employees-cqn-demo" app is better organized.
Keep in mind that the post is a little dated now, we've made enhancements to the driver that make it easier to use now. It's on my list to do a post on how to build a RESTful API with Node.js and Oracle Database but I haven't had a chance to do it yet.

How do I ignore redis if it is not available?

I want my application (lets say a simple node file for now) to work as it is even if redis is not available. I'm not able to do it the correct way. This is what I've tried.
var redis = require('redis');
var redisClient = null;
var getRedisClient = function(){
if(redisClient){
return redisClient;
}
try {
redisClient = redis.createClient({connect_timeout : 5000, max_attempts : 1});
redisClient.on("error", function(err) {
console.error("Error connecting to redis", err);
redisClient = null;
});
return redisClient;
} catch(ex){
console.log("error initialising redis client " + ex);
return null;
}
};
try {
var client = getRedisClient();
console.log("done!");
} catch (ex){
console.log("Exception");
}
However, with this code my application exits if redis is not available (it shouldn't because i've not given a process.exit() command).
How can I solve this?
Checking for Successful Connection on Start
Using a promise, you could guarantee that at least initially, you were able to connect to redis without error within a specified time period:
const redis = require('redis');
const Promise = require('bluebird');
function getRedisClient(timeoutMs){
return new Promise((resolve, reject) => {
const redisClient = redis.createClient();
const timer = setTimeout(() => reject('timeout'), timeoutMs);
redisClient.on("ready", () => {
clearTimeout(timer);
resolve(redisClient);
});
redisClient.on("error", (err) => {
clearTimeout(timer);
reject(err);
});
});
};
const redisReadyTimeoutMs = 10000;
getRedisClient(redisReadyTimeoutMs)
.then(redisClient => {
// the client has connected to redis sucessfully
return doSomethingUseful();
}, error => {
console.log("Unable to connect to redis", error);
});
You Need Proper Error Handling
The redis client being non-null does NOT guarantee using it won't throw an error.
you could experience infrastructure misfortune e.g. crashed redis process, out of memory or network being down.
a bug in your code could cause an error e.g. invalid or missing arguments to a redis command.
You should be handling redis client errors as a matter of course.
DON'T null the Redis Client on Error
It won't give you much but it will force you to check for null every time you try and use it.
The redis client also has inbuilt reconnect and retry mechanisms that you'll miss out on if you null it after the first error. See the redis package docs, look for retry_strategy.
DO Wrap your redis client code with try .. catch ... or use .catch in your promise chain.
DO Make use of a retry_strategy.

Resources