How to filter mongoose changeStream - node.js

I'm working on a chat system, and I try to use the changeStream of mongoDB / mongoose.
I only want to get document if the current user is the recipient, but It not works. So far I have met two cases. One never trigger and the other returned with all documents even if not the current user is the recipient.
The difference is the pipeline in an array or not.
Do you have any idea what is the right syntax?
I read all article that I found in google at the first 10 page in the last 2 days, but none of it contain how to filter. As I understand the aggregation pipe is only for manipulate the result, but its not possible to exclude documents that not pass the conditions.
here is what I have done:
const pipeline = [{
$match: {
"userId": this.recipient.id,
"recipientId": this.user.id
}
}]
const stream = MessageModel.watch(pipeline )
stream.on('change', (data: any) => {
console.log(`messages changed`);
this.socketIo.sockets.in(this.socket.id).emit(`protected/message/subscribe/${this.msg.msgId}`, data.fullDocument);
});

this might be useful
const { ReplSet } = require('mongodb-topology-manager');
const mongoose = require('mongoose');
run().catch(error => console.error(error));
async function run() {
console.log(new Date(), 'start');
const bind_ip = 'localhost';
// Starts a 3-node replica set on ports 31000, 31001, 31002, replica set
// name is "rs0".
const replSet = new ReplSet('mongod', [
{ options: { port: 31000, dbpath: `${__dirname}/data/db/31000`, bind_ip } },
{ options: { port: 31001, dbpath: `${__dirname}/data/db/31001`, bind_ip } },
{ options: { port: 31002, dbpath: `${__dirname}/data/db/31002`, bind_ip } }
], { replSet: 'rs0' });
// Initialize the replica set
await replSet.purge();
await replSet.start();
console.log(new Date(), 'Replica set started...');
// Connect to the replica set
const uri = 'mongodb://localhost:31000,localhost:31001,localhost:31002/' +
'test?replicaSet=rs0';
await mongoose.connect(uri);
// To work around "MongoError: cannot open $changeStream for non-existent
// database: test" for this example
await mongoose.connection.createCollection('people');
const Person = mongoose.model('Person', new mongoose.Schema({ name: String }));
// Create a change stream. The 'change' event gets emitted when there's a
// change in the database
Person.watch().
on('change', data => console.log(new Date(), data));
// Insert a doc, will trigger the change stream handler above
console.log(new Date(), 'Inserting doc');
await Person.create({ name: 'Axl Rose' });
console.log(new Date(), 'Inserted doc');
}

Related

Mongoose can't find recent saved

I am trying to save the IP address of the client who connects to my script.
However, while I am not getting any errors, when I check the collection it is empty.
index.js (main app)
const Listeners = require('mongoose').model('listeners');
const userData = {"ipaddress":ip}
const Listener = new Listeners(userData);
Listener.save(function (err, userData) {
if (err) return console.error(err);
});
Mongoose index.js
const mongoose = require('mongoose');
module.exports.connect = (uri) => {
mongoose.connect(uri, {useCreateIndex: true, useFindAndModify: false , useNewUrlParser: true, useUnifiedTopology: true });
// plug in the promise library:
mongoose.Promise = global.Promise;
mongoose.connection.on('open',function() {
console.log('Mongoose connected. what did you think');
});
mongoose.connection.on('error', (err) => {
console.error(`Mongoose connection error: ${err}`);
process.exit(1);
});
// load models
require('./listener');
};
My listener file
const mongoose = require('mongoose');
// define the User model schema
const ListenerSchema = new mongoose.Schema({
ipaddress: {
type: String,
// index: { unique: true }
},
station: String,
start_time:{
type: Date
},
end_time:{
type: Date
}
}, { collection: 'listeners'});
/**
* The pre-save hook method.
*/
ListenerSchema.pre('save', function saveHook(next) {
const Listener = this;
console.log(this)
});
module.exports = mongoose.model('listeners', ListenerSchema);
when I run it I get { _id: 5e2bf98549ae2d5d6da52475, ipaddress: '127.0.0.1' }
However when I open the mongodb collection I see nothing.
There seems to not be an error, but there must be?
πŸ’‘ The only one reason why you can't save your data into your collection it's because this code in your listener file.
πŸ‘¨β€πŸ« Try to comment this code below πŸ‘‡:
/**
* The pre-save hook method.
*/
ListenerSchema.pre('save', function saveHook(next) {
const Listener = this;
console.log(this)
});
πŸ‘¨β€πŸ« or, the second option is add next function in there. So, your code will looks like this code below πŸ‘‡:
/**
* The pre-save hook method.
*/
ListenerSchema.pre('save', function saveHook(next) {
const Listener = this;
console.log(this);
// add next function below
next();
});
And now, you can try again and I'm sure, you can see the collection in your mongodb.
I hope it's can help you πŸ™.

Node.js mssql module transaction not working with async await

config = {
user: process.env.PROD_USER,
password: process.env.PROD_PASSWORD,
server: process.env.PROD_SERVER,
database: process.env.PROD_DATABASE,
options: {
abortTransactionOnError: true, // <-- SET XACT_ABORT ON
},
}
const pool = await sql.connect(config);
const transaction = new sql.Transaction();
await transaction.begin();
const result = await this.get_sp_data(
data[0],
sp.InsertTransactionMaster,
res
);
const masterId = result.recordset[0].MasterId || 0;
if (masterId) {
asyncForeach(req.body.TransactionDetail, async (value) => {
const detailData = {
MasterId: masterId,
SubServiceId: value.SubServiceId,
Rate: value.Rate,
Quantity: value.Quantity,
Amount: value.Amount,
CreatedBy: req.user.id || 0,
MemberId: value.MemberId,
SubMemberIddd: value.SubMemberId || null,
};
await this.get_sp_data(detailData, sp.InsertTransactionDetail, res);
});
}
await transaction.commit();
console.dir('Transaction commited.');
My custom code is between begin and commit to execute a stored procedure to insert master data after inserting master data master id return that id I use to insert multiple detail data using for loop and last.
I have explicitly place erroneous code in detail insert but transaction is not rolling back as a result master data inserted and giving error of detail.

MongoDB watch() to observe change in Database with NodeJS and Mongoose

I am trying to watch my mongodb. whenever a change occurs I want to apply an action. This is what I have tried
var mongoose = require('mongoose');
//mongoose.connect('mongodb://localhost/test');
mongoose.Promise = global.Promise
mongoose.connect('mongodb://localhost:27017')
mongoose.connection.createCollection('people');
const Person = mongoose.model('Person', new mongoose.Schema({ name: String }));
Person.watch().
on('change', data => console.log(new Date(), data));
console.log(new Date(), 'Inserting doc');
Person.create({ name: 'john doe' });
console.log(new Date(), 'Inserted doc');
But I am getting the following error
node_modules/mongodb/lib/utils.js:132
throw err;
^
MongoError: $changeStream may not be opened on the internal admin
database
How can I fix this ?
Change streams in MongoDB requires a replica set to function.
According to Mongoose docs:
To connect to a replica set you pass a comma delimited list of hosts
to connect to rather than a single host.
mongoose.connect('mongodb://[username:password#]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]' [, options]);
Full example
const { ReplSet } = require('mongodb-topology-manager');
const mongoose = require('mongoose');
run().catch(error => console.error(error));
async function run() {
// Make sure you're using mongoose >= 5.0.0
console.log(new Date(), `mongoose version: ${mongoose.version}`);
await setupReplicaSet();
// Connect to the replica set
const uri = 'mongodb://localhost:31000,localhost:31001,localhost:31002/' +
'test?replicaSet=rs0';
await mongoose.connect(uri);
// For this example, need to explicitly create a collection, otherwise
// you get "MongoError: cannot open $changeStream for non-existent database: test"
await mongoose.connection.createCollection('Person');
// Create a new mongoose model
const personSchema = new mongoose.Schema({
name: String
});
const Person = mongoose.model('Person', personSchema, 'Person');
// Create a change stream. The 'change' event gets emitted when there's a
// change in the database
Person.watch().
on('change', data => console.log(new Date(), data));
// Insert a doc, will trigger the change stream handler above
console.log(new Date(), 'Inserting doc');
await Person.create({ name: 'Axl Rose' });
console.log(new Date(), 'Inserted doc');
}
// Boilerplate to start a new replica set. You can skip this if you already
// have a replica set running locally or in MongoDB Atlas.
async function setupReplicaSet() {
const bind_ip = 'localhost';
// Starts a 3-node replica set on ports 31000, 31001, 31002, replica set
// name is "rs0".
const replSet = new ReplSet('mongod', [
{ options: { port: 31000, dbpath: `${__dirname}/data/db/31000`, bind_ip } },
{ options: { port: 31001, dbpath: `${__dirname}/data/db/31001`, bind_ip } },
{ options: { port: 31002, dbpath: `${__dirname}/data/db/31002`, bind_ip } }
], { replSet: 'rs0' });
// Initialize the replica set
await replSet.purge();
await replSet.start();
console.log(new Date(), 'Replica set started...');
}
Full example excerpted from https://thecodebarbarian.com/stock-price-notifications-with-mongoose-and-mongodb-change-streams
You can’t, change stream cursor is not available on system collections, or any collections in the admin, local, and config databases. You could try configuring your database structure to not be an admin dB.
Mongodb changeStreams doc

What is wrong with my Mongoose data seeding script?

I am trying to populate my MongoDB database with data via a script (JS) that I run through npm run.
const mongoose = require('mongoose')
const dbConf = require('../config/database.js')
const User = require('../app/models/user.js')
const Account = require('../app/models/account.js')
mongoose.connect(dbConf.url, { useMongoClient: true })
mongoose.Promise = global.Promise
const users = [
{
_id: "58c039018060197ca0b52d4c",
email: "user1#example.com",
password: "foo",
balance_cents: 100
},
{
_id: "58c03ada8060197ca0b52d52",
email: "user2#example.com",
password: "foo",
balance_cents: 50000
}
]
const accounts = [
{
name: "Postbank",
iban: "1232423423",
swift: "2444444"
},
{
name: "DKB",
iban: "1234923423",
swift: "6667898"
},
{
name: "Fidor",
iban: "909873423",
swift: "998733"
}
]
async function dropDatabase () {
console.log('Removing User collection')
await User.remove()
console.log('Success!')
}
async function seedUsers () {
console.log('Seeding users..')
try {
await User.insertMany(users)
console.log('Success!')
} catch(error) {
console.log('Error:' + e)
}
}
dropDatabase()
seedUsers()
process.exit(0)
But it doesn't seem to get past the await statements in each function. The output is:
Removing User collection
Seeding users..
And there are no objects in the database. I am using the exact same syntax as I found in a tutorial and can't understand why this is not working.
Any idea anyone? I guess it is a very stupid error that I just don't see right now. Thanks!
Your code isn't waiting for dropDatabase() and seedUsers() to complete before calling process.exit(0).
You need to wrap both of those calls in another async method that can wait for their completion.
async function doBoth() {
await dropDatabase();
await seedUsers();
process.exit(0);
}
doBoth();
However, you probably shouldn't need to call process.exit(0), but that's a separate issue.

Getting list of all databases with Mongoose

There are some similar questions but all of them involves using the MongoDB NodeJS driver instead of Mongoose ODM.
I read the docs but couldn't find such functionality.
You can't directly get the list from the connection provided by mongoose, but it's easy to do with the mongo Admin object as it contains a function called listDatabases:
var mongoose = require('mongoose')
, Admin = mongoose.mongo.Admin;
/// create a connection to the DB
var connection = mongoose.createConnection(
'mongodb://user:pass#localhost:port/database');
connection.on('open', function() {
// connection established
new Admin(connection.db).listDatabases(function(err, result) {
console.log('listDatabases succeeded');
// database list stored in result.databases
var allDatabases = result.databases;
});
});
A very modern approach to get list of all mongo databases using mongoose (version 6.10.*) is to Create a mongoose connection to connect to Mongo's admin database and make sure you have an admin user.
Mongoose object is a very complex object. To list the db's :
const connection = `mongodb://${encodeURIComponent(username)}:${encodeURIComponent(password)}#${hostname}:${port}/admin`
mongoose is a very complex object with promises for executing several functions. to list the db's :
mongoose.connect(connection, { useNewUrlParser: true , useUnifiedTopology: true }).then( (MongooseNode) => {
/* I use the default nativeConnection object since my connection object uses a single hostname and port. Iterate here if you work with multiple hostnames in the connection object */
const nativeConnetion = MongooseNode.connections[0]
//now call the list databases function
new Admin(nativeConnetion.db).listDatabases(function(err, results){
console.log(results) //store results and use
});
})
Result:
{ databases:
[ { name: 'admin', sizeOnDisk: 184320, empty: false },
{ name: 'config', sizeOnDisk: 73728, empty: false },
{ name: 'local', sizeOnDisk: 73728, empty: false },
{ name: 'test', sizeOnDisk: 405504, empty: false } ],
totalSize: 737280,
ok: 1 }
If someone is looking for answers from the latest version of Mongoose and Mongodb, the below code can be used.
import mongoose from 'mongoose';
mongoose.set('strictQuery', true);
mongoose.connect('mongodb://localhost:27017/mydb', {
useNewUrlParser: true,
useUnifiedTopology: true,
});
const db = mongoose.connection;
// Check DB Connection
db.once('open', () => {
(async () => {
const data = await mongoose.connection.db.admin().command({
listDatabases: 1,
});
console.log(data);
})();
console.log('Connected to MongoDB');
});
// Check for DB errors
db.on('error', (err) => {
console.log('DB Connection errors', err);
});
export default mongoose;
If you want to get the database list on your other functions, make sure the connection is established first and also make sure the user has admin access and then just do the below query. This is a sample from my API router.
// Get all databases
router.get('/database/get', async (req, res) => {
try {
const data = await mongoose.connection.db.admin().command({
listDatabases: 1,
});
if (data && data !== null) {
res.status(200).send({ data: data });
return;
}
res.status(200).send({ data: null, message: 'Data not found' });
} catch (e) {
// eslint-disable-next-line no-console
console.log(e);
res.status(500).send(e.message);
}
});
Try running this code. Original take from Gist.

Resources