MongoDB watch() to observe change in Database with NodeJS and Mongoose - node.js

I am trying to watch my mongodb. whenever a change occurs I want to apply an action. This is what I have tried
var mongoose = require('mongoose');
//mongoose.connect('mongodb://localhost/test');
mongoose.Promise = global.Promise
mongoose.connect('mongodb://localhost:27017')
mongoose.connection.createCollection('people');
const Person = mongoose.model('Person', new mongoose.Schema({ name: String }));
Person.watch().
on('change', data => console.log(new Date(), data));
console.log(new Date(), 'Inserting doc');
Person.create({ name: 'john doe' });
console.log(new Date(), 'Inserted doc');
But I am getting the following error
node_modules/mongodb/lib/utils.js:132
throw err;
^
MongoError: $changeStream may not be opened on the internal admin
database
How can I fix this ?

Change streams in MongoDB requires a replica set to function.
According to Mongoose docs:
To connect to a replica set you pass a comma delimited list of hosts
to connect to rather than a single host.
mongoose.connect('mongodb://[username:password#]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]' [, options]);
Full example
const { ReplSet } = require('mongodb-topology-manager');
const mongoose = require('mongoose');
run().catch(error => console.error(error));
async function run() {
// Make sure you're using mongoose >= 5.0.0
console.log(new Date(), `mongoose version: ${mongoose.version}`);
await setupReplicaSet();
// Connect to the replica set
const uri = 'mongodb://localhost:31000,localhost:31001,localhost:31002/' +
'test?replicaSet=rs0';
await mongoose.connect(uri);
// For this example, need to explicitly create a collection, otherwise
// you get "MongoError: cannot open $changeStream for non-existent database: test"
await mongoose.connection.createCollection('Person');
// Create a new mongoose model
const personSchema = new mongoose.Schema({
name: String
});
const Person = mongoose.model('Person', personSchema, 'Person');
// Create a change stream. The 'change' event gets emitted when there's a
// change in the database
Person.watch().
on('change', data => console.log(new Date(), data));
// Insert a doc, will trigger the change stream handler above
console.log(new Date(), 'Inserting doc');
await Person.create({ name: 'Axl Rose' });
console.log(new Date(), 'Inserted doc');
}
// Boilerplate to start a new replica set. You can skip this if you already
// have a replica set running locally or in MongoDB Atlas.
async function setupReplicaSet() {
const bind_ip = 'localhost';
// Starts a 3-node replica set on ports 31000, 31001, 31002, replica set
// name is "rs0".
const replSet = new ReplSet('mongod', [
{ options: { port: 31000, dbpath: `${__dirname}/data/db/31000`, bind_ip } },
{ options: { port: 31001, dbpath: `${__dirname}/data/db/31001`, bind_ip } },
{ options: { port: 31002, dbpath: `${__dirname}/data/db/31002`, bind_ip } }
], { replSet: 'rs0' });
// Initialize the replica set
await replSet.purge();
await replSet.start();
console.log(new Date(), 'Replica set started...');
}
Full example excerpted from https://thecodebarbarian.com/stock-price-notifications-with-mongoose-and-mongodb-change-streams

You can’t, change stream cursor is not available on system collections, or any collections in the admin, local, and config databases. You could try configuring your database structure to not be an admin dB.
Mongodb changeStreams doc

Related

Mongoose connected to mongoDB but have not initialized the database

My connection code:
const mongoose = require("mongoose");
//ES6 Promise
mongoose.Promise = global.Promise;
//connect to mongoDB
mongoose.connect("mongodb://localhost/smslist", {
useNewUrlParser: true,
useUnifiedTopology: true,
});
mongoose.connection
.once("open", function () {
console.log("Connected to database successfuly");
})
.on("error", function (error) {
console.log(`DB Connection error:`, error);
});
The mongod server terminal shows that the connection is successfull
2020-04-27T15:35:24.500+0300 I NETWORK [listener] connection accepted from 127.0.0.1:61286 #1 (1 connection now open)
2020-04-27T15:35:24.506+0300 I NETWORK [conn1] received client metadata from 127.0.0.1:61286 conn1: { driver: { name: "nodejs|Mongoose", version: "3.5.6" }, os: { type: "Windows_NT", name: "win32", architecture: "x64", version: "10.0.18362" }, platform: "'Node.js v12.16.2, LE (unified)", version: "3.5.6|5.9.10" }
The node terminal also prints
Connected to database successfuly
But when I try to use a mongoDB UI like robomongo/studio 3t/mongodb compass community, I will not see a database named smslist as expected.
I also realized that if I close the db server terminal the node terminal still shows connected to database successfully and not DB Connection error:, error as expected.
any idea?
If you haven't created any Model yet in this "smslist" database then you can not see any thing about this
You need to create some Model for this "smslist" database then you can see this in your db.
I agree it depends on the code that follows the above.
I've been working through a tutorial on this.
Here is the code and my notes on what finally worked.
// from scratch tutorial
// WORKS!!!!
//require mongoose - works
const mongoose = require('mongoose');
//define constant - works
const url = 'mongodb://127.0.0.1:27017/fruit'
// connect to datbase works as evidenced by message
mongoose.connect(url, { useNewUrlParser: true });
const db = mongoose.connection
db.once('open', _ => {
console.log('Database connected:', url)
});
db.on('error', err => {
console.error('connection error:', err)
});
//above works!! but do not see new database in mongodb
//now add schema
const Schema = mongoose.Schema
// define schema
fruitSchema = new Schema({
name: String,
rating: Number,
review: String
});
// create the model - or "collection" use singular - mongoose makes it plural
// inside database
const Fruit = mongoose.model("Fruit", fruitSchema);
//now create the document from the model (note upper and lower case)
const fruit = new Fruit({
name: "Pear",
rating: 10,
review: "good with Brie"
});
fruit.save();
//close the connection works but causes an error when trying to add a document
// mongoose.connection.close();

Beginner Issue with Mongoose and MongoDB

I'm new to Node, along with Mongoose and MongoDB. I'm trying to test inserting data into one of the collections in a database on MongoDB Atlas. However, the code somehow inserts the data into the wrong database. I intend to insert data into the 'test' collection in the 'quizzard' database. However, a new collection called 'tests' was created within quizzard where the data was placed. When I tried it again, it started inserting data into another database called 'test' and created a collection called 'tests', where the data is still being placed.
var link = "mongodb+srv://<user>:<password>#quizzard-dp0b2.mongodb.net/test?retryWrites=true&w=majority";
// changed to <user> and <password> for privacy reasons
mongoose.connect(link, {
useNewUrlParser: true,
useUnifiedTopology: true
});
mongoose.connection.on('connected', () => {
console.log('Connected');
});
const Schema = mongoose.Schema;
const TestSchema = new Schema({
_id: Number,
data: String
});
//TestSchema.set('database', 'test');
//TestSchema.set('collection', 'test');
const Test = mongoose.model('Test', TestSchema);
const data = {
_id: 11,
data: "why???"
};
const newTest = new Test(data);
newTest.save((error) => {
if(error){
console.log("An error has occured");
} else {
console.log("Action performed");
}
});
You need to change the link; after the first slash, you choose which DB you want to use.
Examples
// DB NAME youinsertinheredbname
var link = "mongodb+srv://<user>:<password>#quizzard-dp0b2.mongodb.net/youinsertinheredbname?retryWrites=true&w=majority";
// DB NAME stackoverflow
var link = "mongodb+srv://<user>:<password>#quizzard-dp0b2.mongodb.net/stackoverflow?retryWrites=true&w=majority";

How to filter mongoose changeStream

I'm working on a chat system, and I try to use the changeStream of mongoDB / mongoose.
I only want to get document if the current user is the recipient, but It not works. So far I have met two cases. One never trigger and the other returned with all documents even if not the current user is the recipient.
The difference is the pipeline in an array or not.
Do you have any idea what is the right syntax?
I read all article that I found in google at the first 10 page in the last 2 days, but none of it contain how to filter. As I understand the aggregation pipe is only for manipulate the result, but its not possible to exclude documents that not pass the conditions.
here is what I have done:
const pipeline = [{
$match: {
"userId": this.recipient.id,
"recipientId": this.user.id
}
}]
const stream = MessageModel.watch(pipeline )
stream.on('change', (data: any) => {
console.log(`messages changed`);
this.socketIo.sockets.in(this.socket.id).emit(`protected/message/subscribe/${this.msg.msgId}`, data.fullDocument);
});
this might be useful
const { ReplSet } = require('mongodb-topology-manager');
const mongoose = require('mongoose');
run().catch(error => console.error(error));
async function run() {
console.log(new Date(), 'start');
const bind_ip = 'localhost';
// Starts a 3-node replica set on ports 31000, 31001, 31002, replica set
// name is "rs0".
const replSet = new ReplSet('mongod', [
{ options: { port: 31000, dbpath: `${__dirname}/data/db/31000`, bind_ip } },
{ options: { port: 31001, dbpath: `${__dirname}/data/db/31001`, bind_ip } },
{ options: { port: 31002, dbpath: `${__dirname}/data/db/31002`, bind_ip } }
], { replSet: 'rs0' });
// Initialize the replica set
await replSet.purge();
await replSet.start();
console.log(new Date(), 'Replica set started...');
// Connect to the replica set
const uri = 'mongodb://localhost:31000,localhost:31001,localhost:31002/' +
'test?replicaSet=rs0';
await mongoose.connect(uri);
// To work around "MongoError: cannot open $changeStream for non-existent
// database: test" for this example
await mongoose.connection.createCollection('people');
const Person = mongoose.model('Person', new mongoose.Schema({ name: String }));
// Create a change stream. The 'change' event gets emitted when there's a
// change in the database
Person.watch().
on('change', data => console.log(new Date(), data));
// Insert a doc, will trigger the change stream handler above
console.log(new Date(), 'Inserting doc');
await Person.create({ name: 'Axl Rose' });
console.log(new Date(), 'Inserted doc');
}

Mongodb not creating database

I cant create my database 'sms-dev' in mongodb after starting the server but in my console it was printing connected to mongo successfully.I kept my db module inside models folder as db.js and export that module in another file(config.js) inside models folder
Here is the code in db.js file
var db = {
// Connects to mongoDB
connect: function(url, options) {
mongoose.connect(url, options);
mongoose.connection.on('open', function(){
console.log("Connected to mongo successfully");
});
mongoose.connection.on('disconnect', function(){
console.log("Mongo disconnected");
});
mongoose.connection.on('error',function (err) {
console.log('Mongoose default connection error: ' + err);
});
process.on('SIGINT', function() {
mongoose.connection.close(function () {
console.log('Mongoose default connection disconnected through app termination');
process.exit(0);
});
});
}
}
module.exports = db;
and my config.js file is
exports.database = {
url: 'mongodb://127.0.0.1:27017/sms-dev',
options: {
db: { native_parser: true,safe:true },
server: { poolSize: 10 }
}
}
I connected this db in server.js as
var dbcon = process.env.MONGOLAB_URI || config.database.url;
db.connect(dbcon, config.database.options);
This line:
As soon as you create a record with that connection
From this answer.
Did it for me. In my case I had to manually:
Create the database
Create a collection
Create a record in the collection
And then everything was good again.
You should keep your db connection very simple as you are using mongoose.
mongoose.connect should only be called once. That will create the default connection pool for your application.
//db.js
// Bring Mongoose into the app
var mongoose = require( 'mongoose' );
// Create the database connection
mongoose.connect('mongodb://127.0.0.1:27017/sms-dev');
// CONNECTION EVENTS
// When successfully connected
mongoose.connection.on('connected', function () {
console.log('Mongoose default connection open to ' + 'mongodb://127.0.0.1:27017/sms-dev');
});
// If the connection throws an error
mongoose.connection.on('error',function (err) {
console.log('Mongoose default connection error: ' + err);
});
// When the connection is disconnected
mongoose.connection.on('disconnected', function () {
console.log('Mongoose default connection disconnected');
});
// If the Node process ends, close the Mongoose connection
process.on('SIGINT', function() {
mongoose.connection.close(function () {
console.log('Mongoose default connection disconnected through app termination');
process.exit(0);
});
});
You can then easily use the db connection by require mongoose in your files.
//users.js
var mongoose = require( 'mongoose' ),
Users = mongoose.model('users');
It seems that my mongodb was not locked properly so i remove the mongodb.lock file and run with -repair option
Once you ll save data in the database
,You can see the Database by running command
show dbs
const mongoose=require("mongoose")
mongoose.connect('mongodb://localhost:27017/username_db');
var db=mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function() {
console.log("connected successfully")
});
const Schema=mongoose.Schema;
const myschema=new Schema({
name:String,
},{
timestamps:true
});
var model=mongoose.model('myname',myschema);
var data=new myname({
name: 'xyz',
})
data.save((err)=>{
res.send("Error in saving to database");
})
Once you ll save data in the database
,You can see the Database by running command
show dbs
const mongoose=require("mongoose")
mongoose.connect('mongodb://localhost:27017/username_db');
var db=mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function() {
console.log("connected successfully")
});
const Schema=mongoose.Schema;
const myschema=new Schema({
name:String,
},{
timestamps:true
});
var model=mongoose.model('myname',myschema);
var data=new myname({
name: 'xyz',
})
data.save((err)=>{
res.send("Error in saving to database");
})
This is quite old Post, to which I am updating using latest packages - This solution will fix this mongo db connection issue on following version of env.
Node version - 11.5.0 (node -v)
NPM. - 6.4.1(npm -v)
Typescript. - 3.8.3(tsc -v)
Mongodb. - 5.9.15 (package.json)
Following are the steps need to take care in order to fix this -
1. First of all verify all changes in .ts file will be reflecting changes in corresponding .js file. As it was issue with my code it was not being updated.
Run following command and verify .js file
tsc --build tsconfig.json
If js file is not being updated simply delete.js file and run above command. It's pretty simple fix but some time we overlook for it.
Since it's typescript code. So need to copy past below code for verification.
Import * as m from 'mongoose';
export class UserControl {
RegisterUser(){
Const uri = "mongodb://127.0.0.1:27017/User";
m.connect(uri,
{useNewUrlPaerser:true,
useUnifiedTopology:true,
useFindAndModify:true,
useCreateIndex:true });
Let db = m.connection;
Db.once("open",async() =>.
{console.log(connected)});
Db.once("error",async() =>
{console.log(error)});
Const userSchema = new m.schema({
FirstName:string,
Last name:string
});
Const User = m.model('users', userSchema);
Const user = new User({
FName:Andy,
LName:Pat });
Const result = await user.save();
Console.log(result);
}
Run your solution by npm start.
Verify if db collection created?
If not.
First create db with name as "User" in mongodb
Using mongo db compass.
And than try. Still not able to see the collection.
Now need to start two separate console terminal.
go to folder and executive mongo
c:\program file\MongoDb\server\4.2\bin> mongo.exe
on another terminal type mongod ,
it will start your mongo Damon.
Now try. Since this above step will stable connection and show 1 connection active.
Hope this update help.
If you are using Mongoose or MongoClient to connect the mongodb database you will see the database created after you save the first document to the database.
e.g.: the below code will only show connection was successful but it does not create the database 'mydb'
const mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/mydb')
.then(() => console.log('Connected to MongoDB...'))
.catch(err => console.error('Could not connect to MongoDB...', err));
The databse 'mydb' is created only when you save first document record.
const courseSchema = new mongoose.Schema({
name: String,
author: String,
tags: [String],
date: {type: Date, default: Date.now},
isPublished: Boolean
});
const Course = mongoose.model('Course', courseSchema);
async function createCourse(){
const course = new Course({
name: 'Some Course',
author: 'My Name',
tags: ['JavaScript', 'backend'],
isPublished: true
});
const result = await course.save();
console.log(result);
}
createCourse();
This is a duplicate of:
Mongo db that does not exist but shows up in connection
If you insert data your database will be created.

Getting list of all databases with Mongoose

There are some similar questions but all of them involves using the MongoDB NodeJS driver instead of Mongoose ODM.
I read the docs but couldn't find such functionality.
You can't directly get the list from the connection provided by mongoose, but it's easy to do with the mongo Admin object as it contains a function called listDatabases:
var mongoose = require('mongoose')
, Admin = mongoose.mongo.Admin;
/// create a connection to the DB
var connection = mongoose.createConnection(
'mongodb://user:pass#localhost:port/database');
connection.on('open', function() {
// connection established
new Admin(connection.db).listDatabases(function(err, result) {
console.log('listDatabases succeeded');
// database list stored in result.databases
var allDatabases = result.databases;
});
});
A very modern approach to get list of all mongo databases using mongoose (version 6.10.*) is to Create a mongoose connection to connect to Mongo's admin database and make sure you have an admin user.
Mongoose object is a very complex object. To list the db's :
const connection = `mongodb://${encodeURIComponent(username)}:${encodeURIComponent(password)}#${hostname}:${port}/admin`
mongoose is a very complex object with promises for executing several functions. to list the db's :
mongoose.connect(connection, { useNewUrlParser: true , useUnifiedTopology: true }).then( (MongooseNode) => {
/* I use the default nativeConnection object since my connection object uses a single hostname and port. Iterate here if you work with multiple hostnames in the connection object */
const nativeConnetion = MongooseNode.connections[0]
//now call the list databases function
new Admin(nativeConnetion.db).listDatabases(function(err, results){
console.log(results) //store results and use
});
})
Result:
{ databases:
[ { name: 'admin', sizeOnDisk: 184320, empty: false },
{ name: 'config', sizeOnDisk: 73728, empty: false },
{ name: 'local', sizeOnDisk: 73728, empty: false },
{ name: 'test', sizeOnDisk: 405504, empty: false } ],
totalSize: 737280,
ok: 1 }
If someone is looking for answers from the latest version of Mongoose and Mongodb, the below code can be used.
import mongoose from 'mongoose';
mongoose.set('strictQuery', true);
mongoose.connect('mongodb://localhost:27017/mydb', {
useNewUrlParser: true,
useUnifiedTopology: true,
});
const db = mongoose.connection;
// Check DB Connection
db.once('open', () => {
(async () => {
const data = await mongoose.connection.db.admin().command({
listDatabases: 1,
});
console.log(data);
})();
console.log('Connected to MongoDB');
});
// Check for DB errors
db.on('error', (err) => {
console.log('DB Connection errors', err);
});
export default mongoose;
If you want to get the database list on your other functions, make sure the connection is established first and also make sure the user has admin access and then just do the below query. This is a sample from my API router.
// Get all databases
router.get('/database/get', async (req, res) => {
try {
const data = await mongoose.connection.db.admin().command({
listDatabases: 1,
});
if (data && data !== null) {
res.status(200).send({ data: data });
return;
}
res.status(200).send({ data: null, message: 'Data not found' });
} catch (e) {
// eslint-disable-next-line no-console
console.log(e);
res.status(500).send(e.message);
}
});
Try running this code. Original take from Gist.

Resources