Mongoose Connection is not Restored when DB becomes accessible again - node.js

The problem is Mongoose loses its connection in the following scenario:
Node App is started, all seems fine (DB accessible, etc.)
Mongo server process (version 2.6.10) running locally is stopped, then started after 15 seconds, all seems fine (DB accessible, etc.)
Mongo process is stopped.
a request to Express is made, trying to go to DB (using Mongoose) - inaccessible as expected.
Mongo process is started.
The same request is made, DB is inaccessible even though Mongo is up.
If I restart the Node JS app, all works fine.
Why is a failing query (using Mongoose version 4.4.5) to Mongo prevents the connection from being restored when the DB process is up again?
Should we implement a retry mechanism to try and restore the connection until DB becomes accessible?
I've tried the configuration mentioned here but it didn't work.
Any help would be appreciated.
Bellow is a sample code of our connection helper:
'use strict';
const _ = require('lodash');
const mongoose = require('mongoose');
const config = require('../config');
const logger = require('../logger');
const _instance = Symbol('instance');
const _enforcer = Symbol('enforcer');
const _members = Symbol('members');
/**
* Singleton implementation of ConnectionHelper module
* This module is responsible to reusing common db connections in the application
* #type {ConnectionsHelper}
*/
module.exports = class ConnectionsHelper {
constructor(enforcer) {
if (enforcer !== _enforcer) {
throw new Error('invalid singleton instantiation');
}
initConnectionFromConfig.call(this);
}
/**
* The single instance
* #returns {*}
*/
static get instance() {
if (!this[_instance]) {
this[_instance] = new ConnectionsHelper(_enforcer);
}
return this[_instance];
}
/**
* method retrieves connection by its name
* #param connectionKey
* #returns {*}
*/
getConnection(connectionKey) {
return this[_members][connectionKey];
}
/**
* method disconnects all underlying connections
* #returns {void|MongooseThenable}
*/
closeConnections() {
return mongoose.disconnect();
}
};
function initConnectionFromConfig() {
this[_members] = {};
const dbsConnections = config.get('dbsConnections');
_.forEach(dbsConnections, (connection, connectionName) => {
const protocol = connection.protocol;
const repSetPath = connection.mongoPath.join(',');
const options = connection.options;
options.server = {auto_reconnect: true, socketOptions: {keepAlive: 1, connectTimeoutMS: 30000 }};
options.replset = {socketOptions: {keepAlive: 1, connectTimeoutMS: 30000 }};
this[_members][connectionName] = mongoose.createConnection(protocol + repSetPath, options);
addConnectionEvents.call(this, connectionName);
});
}
function initConnectionIfNeeded() {
this[_members] = {};
const dbsConnections = config.get('dbsConnections');
_.forEach(dbsConnections, (connection, connectionName) => {
const protocol = connection.protocol;
const repSetPath = connection.mongoPath.join(',');
const options = connection.options;
//options.server = {auto_reconnect: true, socketOptions: {keepAlive: 1, connectTimeoutMS: 30000 }};
//options.replset = {socketOptions: {keepAlive: 1, connectTimeoutMS: 30000 }};
this[_members][connectionName] = mongoose.createConnection(protocol + repSetPath, options);
addConnectionEvents.call(this, connectionName);
});
}
function addConnectionEvents(connectionName) {
const connection = this[_members][connectionName];
connection.on('connected', () => {
logger.debug(`Mongoose connection open`);
});
connection.on('error', (err) => {
logger.debug(`Mongoose connection error: ${err}`);
});
connection.on('exception', (err) => {
logger.debug(`Mongoose connection exception: ${err}`);
});
connection.on('disconnected', () => {
logger.debug(`Mongoose connection disconnected`);
});
connection.on('reconnected', () => {
logger.debug(`Mongoose connection reconnected`);
});
connection.on('open', () => {
logger.debug(`Mongoose connection is open`);
});
// If the Node process ends, close the Mongoose connection
process.on('SIGINT', () => {
connection.close(() => {
logger.debug(`Mongoose default connection disconnected through app termination`);
process.exit(0);
});
});
}

I had the same problem. What I did is to restart mongoose in here:
connection.on('disconnected', () => {
logger.debug(`Mongoose connection disconnected`);
mongoose.connect(path);
});
And I also use setTimeout() so that it tries every 3 secs.
mongoose.connection.on('disconnected', function () {
console.log('Mongoose default connection disconnected');
console.log("Trying to reconnect");
function connect(){
mongoose.connect(dbPath, function(err){
if(err) console.log("Error while trying to reconnect to MongoDB");
});
}
setTimeout(connect, 3000);
});
And I also use ForeverJS.

Related

node-media-server: session.reject() not working

I am trying to create an RTMP-server with the npm package: http://github.com/illuspas/Node-Media-Server. So the server works fine but I need to implement authentication in it. I am trying to check the authentication on "prePublish" event. I am querying the database and retrieving the user if the user was found then I want to let the user stream otherwise rejected. But the problem is, it doesn't leave it instead disconnects and then the stream automatically reconnected to it then it disconnects again and the loop goes on. How do I fix this problem?
Here is the code for the event:
const NodeMediaServer = require('node-media-server');
const config = require('./config').rtmp_server;
const db = require('./db');
const nms = new NodeMediaServer(config);
const getStreamKeyFromStreamPath = (path) => {
const parts = path.split('/');
return parts[parts.length - 1];
};
nms.on('prePublish', async (id, StreamPath, args) => {
const session = nms.getSession(id);
try {
const streamKey = getStreamKeyFromStreamPath(StreamPath);
const validStream = (
await db.query('SELECT * FROM public."People" WHERE stream_key = $1', [streamKey])
).rows[0];
console.log(validStream);
if (validStream) {
// do stuff
} else {
session.reject((reason) => {
console.log(reason);
});
}
console.log(
'[NodeEvent on prePublish]',
`id=${id} StreamPath=${StreamPath} args=${JSON.stringify(args)}`
);
} catch (err) {
session.reject();
}
});
module.exports = nms;
Here is the code of the entry point of the server:
require("dotenv").config();
const db = require("./db");
const nms = require("./nms");
// database connection
db.connect()
.then(() => {
console.log("Connected to database");
// start the rtmp server
nms.run();
})
.catch((err) => console.log(err.message));
Here is the db file:
const { Pool } = require('pg');
const connectionString = process.env.PG_CONNECTION_STRING;
const poolOptions = {
host: process.env.PG_HOST,
user: process.env.PG_USER,
port: process.env.PG_PORT,
password: process.env.PG_PASSWORD,
database: process.env.PG_DATABASE,
};
const pool = new Pool(process.env.NODE_ENV === 'production' ? connectionString : poolOptions);
module.exports = pool;
My procedures to solve that problem:
Instead of the async function, I tried to handle the database query using a callback but it didn't work.
Before I was calling session.reject() now I am passing a callback there but the behavior is still the same
If you have any solution for that, please let me know.
Thanks in advance

How to properly connect to MongoDB using Cloud functions?

I would like to connect to my Atlas cluster only once per instance running Cloud Functions.
Here is my code for an instance :
const MongoClient = require("mongodb").MongoClient;
const client = new MongoClient("myUrl", {
useNewUrlParser: true,
useUnifiedTopology: true,
});
exports.myHttpMethod = functions.region("europe-west1").runWith({
memory: "128MB",
timeoutSeconds: 20,
}).https.onCall((data, context) => {
console.log("Data is: ", data);
client.connect(() => {
const testCollection = client.db("myDB").collection("test");
testCollection.insertOne(data);
});
});
And i would like to avoid the client.connect() in each function call that seems to be really too much.
I would like to do something like this :
const MongoClient = require("mongodb").MongoClient;
const client = await MongoClient.connect("myUrl", {
useNewUrlParser: true,
useUnifiedTopology: true,
});
const db = client.db("myDB");
exports.myHttpMethod = functions.region("europe-west1").runWith({
memory: "128MB",
timeoutSeconds: 20,
}).https.onCall((data, context) => {
console.log("Data is: ", data);
const testCollection = db.collection("test");
testCollection.insertOne(data);
});
But i can't await like this.
In my AWS Lambda functions (running in python) i have not this issue and i am able to connect only once per instance, so i guess there is an equivalent but i don't know much JS / Node JS.
You can store your database client as a global variable. From the documentation,
Cloud Functions often recycles the execution environment of a previous invocation. If you declare a variable in global scope, its value can be reused in subsequent invocations without having to be recomputed.
Try refactoring the code as shown below:
import * as functions from "firebase-functions";
import { MongoClient } from "mongodb";
let client: MongoClient | null;
const getClient = async () => {
if (!client) {
const mClient = new MongoClient("[MONGODB_URI]", {});
client = await mClient.connect();
functions.logger.log("Connected to MongoDB");
} else {
functions.logger.log("Using existing MongoDB connection");
}
functions.logger.log("Returning client");
return client;
};
export const helloWorld = functions.https.onRequest(
async (request, response) => {
const db = (await getClient()).db("[DATABASE]");
const result = await db.collection("[COLLECTION]").findOne({});
response.send("Hello from Firebase!");
}
);
This should reuse the connection for that instance.

reconnecting to nodejs websocket on failure

This is my first practice after reading some tutorials and videos. Basically, I need message to be sent from the server (nodejs) to the client (Angular 6). At first tho, when client app is booted, it sends user id to the server for authentication. The server then will send data based on that user.
Now my problem is on first load and a few calls, the connection does work. But then on refresh or so, the connection drops. My client does console out "retrying" but it never succeeds. It works only if I manually restart the server and reload the client so a new connection could be established.
How can I maintain a fairly stable connection throughout the lifetime of a client? At times the readyState stays at 3 on the server i.e. connecting, which I am confused with because the client does try to reconnect...just fails.
My Server is simple. index.js (Tried to put it up on stackblitz but failed...would appreciate if someone can figure out the dependency file: nodejs websocket server)
const express = require('express');
const bodyParser = require('body-parser');
const pg = require ('pg');
var ws = require('./ws')
var app = express()
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
app.listen(3000, function () {
console.log('We are running on port 3000!')
})
ws.js:
const winston = require('winston');
const logger = winston.createLogger({
transports: [
new winston.transports.File({ filename: 'error.log'})
]
});
var WebSocketServer = require("ws").Server,
wss = new WebSocketServer({
port: 40511
});
let data = {
'packet': ['amy1', 'amy2', 'amy3']
}
const mx = 2;
const mn = 0;
wss.on("connection", function(ws) {
ws.on("message", function(user) {
// client has called now. If the connection
// fails, the client does try to connection again and again -- no limit but it simply doesn't seem to have effect. When connecting, it simply sends user name
console.log("received: %s", user);
setInterval(function(){
var random = Math.floor(Math.random() * (mx - mn + 1) + mn);
if (ws.readyState == ws.OPEN){
ws.send(data['packet'][random]);
}
}, 3000);
});
});
My front end is: service.ts
import { Observable} from 'rxjs';
export class WebSocketService {
socket: WebSocket;
constructor() { }
initConnection(): void {
if(!this.socket){
this.socket = new WebSocket('ws://localhost:40511');
// when connection is open, send user id
this.socket.onopen = () => this.socket.send(2);
}
}
manageState() : Observable<any>{
const vm = this;
return new Observable(observer => {
this.socket.onerror = (e) => {
// close it
this.socket.close();
observer.next('web socket communication closed due to error')
};
this.socket.onclose = (e) => {
//socket closed for any reason
setTimeout(function() {
console.log('try to connect')
vm.initConnection();
observer.next('still trying')
}, 1000);
}
});
}
onMessage(): Observable<any> {
// when message arrives:
return new Observable(observer => {
this.socket.onmessage = (e) => {
console.log(e.data);
observer.next(e.data)
};
});
}
}
component.ts:
// initialize the connection
this.service.initConnection();
this.service.onMessage().subscribe(
data => {
// we have got data
console.log('data came ', data)
},
err => {
console.log("error websocking service ", err);
}
);
// track state of the communication, letting the service to reconnect if connection is dropped
this.service.manageState().subscribe(data => {
console.log(data);
});

WebRTC Video sharing app doesn't work

I have been trying to share a video stream between two clients over WebRTC for about a week now and I have no idea how to proceed any further. I'm frustrated and could really use some help from the more experienced. Please help me get this running.
I am using Websockets and NodeJS. I will post all of my code below:
Server Code ( on NodeJS )
"use strict";
/** Requires **/
var webSocketServer = require('websocket').server,
expr = require("express"),
xpress = expr(),
server = require('http').createServer(xpress);
// Configure express
xpress.configure(function() {
xpress.use(expr.static(__dirname + "/public"));
xpress.set("view options", {layout: false});
});
// Handle GET requests to root directory
xpress.get('/', function(req, res) {
res.sendfile(__dirname + '/public/index.html');
});
// WebSocket Server
var wsServer = new webSocketServer({
httpServer: server
});
// Set up the http server
server.listen(8000, function(err) {
if(!err) { console.log("Listening on port 8000"); }
});
var clients = [ ];
/** On connection established */
wsServer.on('request', function(request) {
// Accept connection - you should check 'request.origin' to make sure that client is connecting from your website
var connection = request.accept(null, request.origin);
var self = this;
// We need to know client index to remove them on 'close' event
var index = clients.push(connection) - 1;
// Event Listener for when Clients send a message to the Server
connection.on('message', function(message) {
var parsedMessage = JSON.parse(message.utf8Data);
if ( parsedMessage.kind == 'senderDescription' ) {
wsServer.broadcastUTF(JSON.stringify({ kind:'callersDescription', callerData: parsedMessage }));
}
});
});
Index.html loads and immediately runs VideoChatApp.js
function VideoChatApp() {
this.connection = null;
this.runConnection();
}
_p = VideoChatApp.prototype;
/** Initialize the connection and sets up the event listeners **/
_p.runConnection = function(){
// To allow event listeners to have access to the correct scope
var self = this;
// if user is running mozilla then use it's built-in WebSocket
window.WebSocket = window.WebSocket || window.MozWebSocket;
// if browser doesn't support WebSocket, just show some notification and exit
if (!window.WebSocket) { return; }
/** Where to make the connection **/
var host = location.origin.replace(/^http/, 'ws');
console.log(host);
this.connection = new WebSocket(host);
/** Once the connection is established **/
this.connection.onopen = function () {
console.log("Web Socket Connection Established");
self.onConnectionEstablished();
};
/** If there was a problem with the connection */
this.connection.onerror = function (error) {
console.log("ERROR with the connection *sadface*");
};
}; // end runConnection
_p.onConnectionEstablished = function() {
// My connection to the nodejs server
var websocketConnection = this.connection;
// Some local variables for use later
var mediaConstraints = {
optional: [],
mandatory: {
OfferToReceiveVideo: true
}
};
var offerer, answerer;
this.theLocalStream = null;
var amITheCaller = false;
var localVideoTag = document.getElementById('localVideoTag');
var remoteVideoTag = document.getElementById('remoteVideoTag');
window.RTCPeerConnection = window.mozRTCPeerConnection || window.webkitRTCPeerConnection;
window.RTCSessionDescription = window.mozRTCSessionDescription || window.RTCSessionDescription;
window.RTCIceCandidate = window.mozRTCIceCandidate || window.RTCIceCandidate;
navigator.getUserMedia = navigator.mozGetUserMedia || navigator.webkitGetUserMedia;
window.URL = window.webkitURL || window.URL;
window.iceServers = {
iceServers: [{
url: 'stun:23.21.150.121'
}]
};
var callButton = document.getElementById("callButton");
callButton.onclick = callClicked;
function callClicked() {
amITheCaller = true;
setUpOffer();
}
offerer = new RTCPeerConnection(window.iceServers);
answerer = new RTCPeerConnection(window.iceServers);
/** Start Here - Set up my local stream **/
getUserMedia(function (stream) {
hookUpLocalStream(stream);
});
function getUserMedia(callback) {
navigator.getUserMedia({
video: true
}, callback, onerror);
function onerror(e) {
console.error(e);
}
}
function hookUpLocalStream(localStream) {
this.theLocalStream = localStream;
callButton.disabled = false;
localVideoTag.src = URL.createObjectURL(localStream);
localVideoTag.play();
};
/* When you click call, then we come here. Here I want to set up the offer and send it. */
function setUpOffer() {
var stream = theLocalStream;
offerer.addStream(stream);
offerer.onaddstream = function (event) {
console.log("onaddstream callback was called");
};
offerer.onicecandidate = function (event) {
if (!event || !event.candidate) return;
answerer.addIceCandidate(event.candidate);
};
offerer.createOffer(function (offer) {
offerer.setLocalDescription(offer);
console.log("------------------- What I am sending: -------------------------");
console.log(offer);
console.log(stream);
console.log("-----------------------------------------------------------------\n");
var jsonMsg = JSON.stringify( {kind:'senderDescription', streamInfo: offer, theStream: stream} );
websocketConnection.send( jsonMsg );
//answererPeer(offer, stream);
}, onSdpError, mediaConstraints);
}
/* Respond to a call */
function answererPeer(offer, stream) {
answerer.addStream(stream);
answerer.onaddstream = function (event) {
remoteVideoTag.src = URL.createObjectURL(event.stream);
remoteVideoTag.play();
};
answerer.onicecandidate = function (event) {
if (!event || !event.candidate) return;
offerer.addIceCandidate(event.candidate);
};
answerer.setRemoteDescription(offer, onSdpSucces, onSdpError);
answerer.createAnswer(function (answer) {
answerer.setLocalDescription(answer);
offerer.setRemoteDescription(answer, onSdpSucces, onSdpError);
}, onSdpError, mediaConstraints);
}
function onSdpError(e) {
console.error('onSdpError', e);
}
function onSdpSucces() {
console.log('onSdpSucces');
}
websocketConnection.onmessage = function (messageFromServer) {
console.log(" ------------------------ Message from server: -------------------- ");
var parsedMessage = JSON.parse(messageFromServer.data);
if(parsedMessage.callerData.kind = "senderDescription") {
console.log("Received a senderDescription");
console.log(parsedMessage.callerData.streamInfo);
console.log(parsedMessage.callerData.theStream);
console.log("-------------------------------------------------------------------\n");
answererPeer(parsedMessage.callerData.streamInfo, parsedMessage.callerData.theStream);
}
};
};// end onConnectionEstablished()
Finally, here are my errors:
I am not sure if this is still interesting for you, but I have some very good experience with WebRTC using PeerJS as a wrapper around it. It takes care of all the stuff you don't want to do (http://peerjs.com/). There is the client library as well as a very nice signaling server for nodejs (https://github.com/peers/peerjs-server). You can easy extend this server in your own node server.
This may not explain why your approach failed, but gets WebRTC running easily.
You can start with code that is already working and completely open. Check out easyrtc.com we have a client api, signalling server and working code. And if you have problems with that code ask us for help on Google Groups for easyrtc.

Check mongoose connection state without creating new connection

I have some tests - namely Supertest - that load my Express app. This app creates a Mongoose connection. I would like to know how to check the status of that connection from within my test.
In app.js
mongoose.connect(...)
In test.js
console.log(mongoose.connection.readyState);
How to access the app.js connection? If I connect using the same parameters in test.js will that create a new connection or look for existing one?
Since the mongoose module exports a singleton object, you don't have to connect in your test.js to check the state of the connection:
// test.js
require('./app.js'); // which executes 'mongoose.connect()'
var mongoose = require('mongoose');
console.log(mongoose.connection.readyState);
ready states being:
0: disconnected
1: connected
2: connecting
3: disconnecting
I use this for my Express Server mongoDB status, where I use the express-healthcheck middleware
// Define server status
const mongoose = require('mongoose');
const serverStatus = () => {
return {
state: 'up',
dbState: mongoose.STATES[mongoose.connection.readyState]
}
};
// Plug into middleware.
api.use('/api/uptime', require('express-healthcheck')({
healthy: serverStatus
}));
Gives this in a Postman request when the DB is connected.
{
"state": "up",
"dbState": "connected"
}
Gives this response when the database was shutdown.
{
"state": "up",
"dbState": "disconnected"
}
(The "up" in the responses represent my Express Server status)
Easy to read (no numbers to interpret)
As stated before "readyState" is good. "ping" is also good admin utility for doing so as well. It will return { ok: 1 } if it can accept commands.
const mongoose = require('mongoose')
// From where ever your making your connection
const connection = await mongoose.createConnection(
CONNECT_URI,
CONNECT_OPTS
)
async function connectionIsUp(): Promise<boolean> {
try {
const adminUtil = connection.db.admin()
const result = await adminUtil.ping()
console.log('result: ', result) // { ok: 1 }
return !!result?.ok === 1
} catch(err) {
return false
}
}
Or if you you want it short.
async function connectionIsUp(): Promise<boolean> {
try {
return await connection.db.admin().ping().then(res => !!res?.ok === 1)
} catch (err) {
return false
}
}
var dbState = [{
value: 0,
label: "disconnected"
},
{
value: 1,
label: "connected"
},
{
value: 2,
label: "connecting"
},
{
value: 3,
label: "disconnecting"
}];
mongoose.connect(CONNECTIONSTRING, {
useNewUrlParser: true
},
() => {
const state = Number(mongoose.connection.readyState);
console.log(dbState.find(f => f.value == state).label, "to db"); // connected to db
});

Resources