I'm building a web app that I would like to use with two databases based on a GET query. These two databases have the same schema, the only difference is one has live data and the other is scrubbed (or test) data.
This works fine, but I'm wondering if this is the proper way to go about solving this problem.
I'm referencing a model with a schema: names_model.js
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var namesSchema = new Schema({
name: String,
createdAt: String
});
module.exports = mongoose.model('names', namesSchema);
And this is my main file. If the param query is 1 it will connect to the first db, else it will go connect to the second db.
var mongoose = require('mongoose');
var db = mongoose.createConnection('mongodb://localhost/database1');
var db2 = db.useDb('database2');
var NamesDB = require('./names_model.js');
var Connect = db.model('names', NamesDB);
var Connect2 = db2.model('names', NamesDB);
exports.getData = function(dbName, sendBack) {
console.log(dbName);
if (dbName == 1) {
var Names = Connect;
}
else {
Names = Connect2;
}
Names.find({}, function (err, docs) {
if (err) {
console.log(err)
}
else {
sendBack(docs);
}
});
};
Like I mentioned above, this does work, though I feel that I might be making extra steps for myself, but I'm not quite sure. I'm hoping someone might be able to tell me if theres an easier way.
Thanks!
T
You could use an environment variable to define if you are on "live/production" site or the development one.
Define an env variable in your systems, usually it is NODE_ENV=<env-name> and then use a condition on process.env.NODE_ENV to define what to use for each env:
var dbName;
if (process.env.NODE_ENV === 'development') {
// Define the development db
dbName = 'database1';
} else if (process.env.NODE_ENV === 'production') {
// Define the production db
dbName = 'database2';
}
var db = mongoose.createConnection('mongodb://localhost/' + dbName);
var NamesDB = require('./names_model.js');
var Connect = db.model('names', NamesDB);
Related
I want to preface by saying I have looked for this issue online and found solutions using mongoose, I do not use that so I don't know how well it translate here. Plus, I also using async/await instead of .then, so my code is different.
My question is that the API I have made in ExpressJS is super slow compared to the same API I have in Flask. I tried seeing why this is an issue, since JS is supposed to be faster than Python. I noticed it was due to me connecting to Mongo each time. I need to do this as my MongoDB has a different databases for different clients, so I cannot just have 1 constant connection to one database. Below is my Mongo code.
const {MongoClient} = require('mongodb');
const client = new MongoClient("mongodb+srv://abc#xyz.mongodb.net/<Cluster");
class MongoMethods {
constructor(company, collection){
this.company = company;
this.collection = collection;
}
async get(accessParameters){
try{
await client.connect();
const db = client.db(this.company);
const collection = db.collection(this.collection);
const val = await collection.findOne({"_id":accessParameters["at"]});
return val
} finally {
await client.close();
}
}
async putIn(putParameters, type){
try{
await client.connect();
const db = client.db(this.company);
const collection = db.collection(this.collection);
var existing = await collection.findOne({"_id":putParameters["at"]});
if(type === "array"){
var toAdd = existing !== undefined && putParameters["in"] in existing? existing[putParameters["in"]] : [];
toAdd.push(putParameters["value"]);
}else if(type === "dict"){
var toAdd = existing !== undefined && putParameters["in"] in existing? existing[putParameters["in"]] : {};
toAdd[putParameters["key"]] = putParameters["value"];
}else{
var toAdd = putParameters["value"];
}
await collection.updateOne({"_id":putParameters["at"]}, {"$set": {[putParameters["in"]]: toAdd}}, {upsert: true});
} finally {
await client.close();
}
}
async remove(removeParameters){
try{
await client.connect();
const db = client.db(this.company);
const collection = db.collection(this.collection);
if(removeParameters["key"] !== undefined){
await collection.updateOne({"_id":removeParameters["at"]}, {"$unset": {[removeParameters["in"] + "." + removeParameters["key"]] : ""}})
}else if(removeParameters["in"] !== undefined){
await collection.updateOne({"_id":removeParameters["at"]}, {"$unset": {[removeParameters["in"]] : ""}})
}else{
await collection.deleteOne({"_id":removeParameters["at"]})
}
}finally{
await client.close();
}
}
}
module.exports.MongoMethods = MongoMethods;
Below is how functions in my ExpressJS file look (I cannot post this file so this is an excerpt):
var express = require('express');
var cors = require('cors')
const bodyParser = require('body-parser');
const { validate, ValidationError } = require('express-superstruct');
const { MongoMethods } = require('./mongo.js')
let app = express()
app.use(cors())
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
app.route('/enp1/')
.get(validate({uid : "string", site : "string"}), function (req, res) {
var args = req.query
var mongoClient = new MongoMethods(args.site, "collection_1")
mongoClient.get({at : args.uid}).catch(console.dir).then(result => {
if(result == undefined){ // This may need to take into account an empty dict
res.status(404).json({"Database Error" : "UID does not exists or site is not a valid website"})
}
res.status(200).json(result)
})
})
app.route('/enp2/')
.put(validate({site : "string", uid : "string", time : "string"}), function (req, res) {
var args = req.body;
var mongoClient = new MongoMethods(args.site, "time")
mongoClient.putIn({at : args.uid, in : "time", value : parseInt(args.time)}, "int").catch(console.dir).then(result => {
res.status(200).end()
})
})
It seems like this code is connecting to Mongo each time as I have to initialize the MongoMethods object each time. Can I prevent it from trying to connect each time so that my API doesn't have slow speeds? When I compare speeds, JS endpoints without Mongo are 50% faster than their Python counterpart but when using Mongo endpoints, it is around 300ms slower.
Let me know if you need anymore clarification.
Thanks in advance!
Edit 1: I wanted to mention, the API runs on AWS API Gateway and AWS Lambda#Edge functions.
I'm looking for the easiest & performant way to make a multitenant express.js app for managing projects.
Reading several blogs and articles, I figured out that, for my application, would be nice to have a database per tenant architecture.
My first try has been to use subdomains to detect the tenant, and then map the subdomain to a mongodb database.
I came up with this express middlewares
var mongoose = require('mongoose');
var debug = require('debug')('app:middleware:mongooseInstance');
var conns [];
function mongooseInstance (req, res, next) {
var sub = req.sub = req.subdomains[0] || 'app';
// if the connection is cached on the array, reuse it
if (conns[sub]) {
debug('reusing connection', sub, '...');
req.db = conns[sub];
} else {
debug('creating new connection to', sub, '...');
conns[sub] = mongoose.createConnection('mongodb://localhost:27017/' + sub);
req.db = conns[sub];
}
next();
}
module.exports = mongooseInstance;
Then I register the models inside another middleware:
var fs = require('fs');
var debug = require('debug')('app:middleware:registerModels');
module.exports = registerModels;
var models = [];
var path = __dirname + '/../schemas';
function registerModels (req, res, next) {
if(models[req.sub]) {
debug('reusing models');
req.m = models[req.sub];
} else {
var instanceModels = [];
var schemas = fs.readdirSync(path);
debug('registering models');
schemas.forEach(function(schema) {
var model = schema.split('.').shift();
instanceModels[model] = req.db.model(model, require([path, schema].join('/')));
});
models[req.sub] = instanceModels;
req.m = models[req.sub];
}
next();
}
Then I can proceed normally as any other express.js app:
var express = require('express');
var app = express();
var mongooseInstance = require('./lib/middleware/mongooseInstance');
var registerModels = require('./lib/middleware/registerModels');
app.use(mongooseInstance);
app.use(registerModels);
app.get('/', function(req, res, next) {
req.m.Project.find({},function(err, pets) {
if(err) {
next(err);
}
res.json({ count: pets.length, data: pets });
});
});
app.get('/create', function (req, res) {
var p = new req.m.Project({ name: 'Collin', description: 'Sad' });
p.save(function(err, pet) {
res.json(pet);
});
});
app.listen(8000);
The app is working fine, I don't have more than this right now, and I'd like to get some feedback before I go on, so my questions would be:
Is this approach is efficient? Take into account that a lot will be happening here, multiple tenants, several users each, I plan to setup webhooks in order to trigger actions on each instance, emails, etc...
Are there any bottlenecks/pitfalls I'm missing? I'm trying to make this scalable from the start.
What about the model registering? I didn't found any other way to accomplish this.
Thanks!
Is this approach is efficient?
Are there any bottlenecks/pitfalls I'm missing?
This all seems generally correct to me
What about the model registering?
I agree with #narc88 that you don't need to register models in middleware.
For lack of a better term, I would use a factory pattern. This "factory function" would take in your sub-domain, or however you decide to detect tenants, and return a Models object. If a given middleware wants to use its available Models you just do
var Models = require(/* path to your Model factory */);
...
// later on inside a route, or wherever
var models = Models(req.sub/* or req.tenant ?? */);
models.Project.find(...);
For an example "factory", excuse the copy/paste
var mongoose = require('mongoose');
var fs = require('fs');
var debug = require('debug')('app:middleware:registerModels');
var models = [];
var conns = [];
var path = __dirname + '/../schemas';
function factory(tenant) {
// if the connection is cached on the array, reuse it
if (conns[tenant]) {
debug('reusing connection', tenant, '...');
} else {
debug('creating new connection to', tenant, '...');
conns[tenant] = mongoose.createConnection('mongodb://localhost:27017/' + tenant);
}
if(models[tenant]) {
debug('reusing models');
} else {
var instanceModels = [];
var schemas = fs.readdirSync(path);
debug('registering models');
schemas.forEach(function(schema) {
var model = schema.split('.').shift();
instanceModels[model] = conns[tenant].model(model, require([path, schema].join('/')));
});
models[tenant] = instanceModels;
}
return models[tenant];
}
module.exports = factory;
Aside from potential (albeit probably small) performance gain, I think it also has the advantage of:
doesn't clutter up the request object as much
you don't have to worry as much about middleware ordering
allows more easily abstracting permissions for a given set of models, i.e. the models aren't sitting on the request for all middleware to see
This approach doesn't tie your models to http requests, so you might have flexibility to use the same factory in a job queue, or whatever.
how do i get the knex object in my controllers or any other model files if i am not using waterline.
for eg.:
in my api/models/Users.js
module.exports = {
find : function(id){
// my knex query
},
insert : function(data){
// my knex query again
}
}
So in my controllers i will just do:
var result = Users.find(id);
or
var result = Users.insert({username : 'sailsjs'});
or the knex object will be available globally with out being used in the model files itself... so that i can do the knex query in the controller it self
// UsersController/index
index : function(req, res){
// my knex query
}
Thanks
Arif
//config/bootstrap.js
module.exports.bootstrap = function (cb) {
var Knex = require('knex');
var knex = Knex.initialize({
client : "mysql",
connection : {
host :'localhost',
user :'root',
database : 'sales_force',
password : '*******'
}
});
knex.instanceId = new Date().getTime();
sails.config.knex = knex;
// It's very important to trigger this callack method when you are finished
// with the bootstrap! (otherwise your server will never lift, since it's waiting on the bootstrap)
cb();
};
// in the controller
var knex = sails.config.knex
this returns the knex object. the knex.instanceId shows that the same connection is used all over.
Please suggest if this might cause any problems.
Thanks
Arif
Best Option to use Knex Js globally in Sails Js (Tested for Version 1+) is to create a file named knex.js inside config directory, like this:
/**
* Knex Js, Alternate DB Adapter, In case needed, it is handy for doing migrations
* (sails.config.knex)
*
*
* For all available options, see:
* http://knexjs.org/
*/
const developmentDBConfig = require('./datastores');
const stagingDBConfig = require('./env/staging');
const productionDBConfig = require('./env/production');
function getConnectionString() {
let dbConnectionString = developmentDBConfig.datastores.default.url;
if (process.env.NODE_ENV === 'staging') {
dbConnectionString = stagingDBConfig.datastores.default.url;
}
if (process.env.NODE_ENV === 'production') {
dbConnectionString = productionDBConfig.datastores.default.url;
}
return dbConnectionString;
}
module.exports.knex = require('knex')({
client: 'postgresql',
connection: getConnectionString()
});
Now, in any file(helpers/controllers/views etc..) you can set and use knex as:
// Now use this knex object for anything like:
let user = await sails.config.knex('user').select('*').first();
I'm trying to use a new schema in my db, but get errors while trying to instantiate it. I have two other schemas (in two different model files in the folder "models"), that works perfect, and they are shaped in the same way. What does the error message mean and what can I do different to prevent it from occur?
I don't thinks its any problem with the other code in the controller, because i've tried to instantiate another db model in the same place using the same syntax, and that works fine.
The error I get: 500 TypeError: object is not a function
at Schema.CALL_NON_FUNCTION_AS_CONSTRUCTOR (native)
Sorry for all the code below. I didn't know what I could exclude in this case.
Anyway, thanks in advance!
controller file:
module.exports = function(app, service) {
var imageModel = service.useModel('image');
app.post('/file-upload', function(req, res, next) {
// other code...
var imageAdd = new imageModel.ImgSchema();
}
}
mongodb model (models/image.js):
module.exports = function (mongoose) {
var modelObject = {};
var Schema = mongoose.Schema,
ObjectId = Schema.ObjectId;
var ImgSchema = new Schema({
name : String,
size : Number,
type : String
});
modelObject.ImgSchema = ImgSchema;
modelObject.Images = mongoose.model('Images', ImgSchema);
return modelObject;
};
For mongodb I'm using a service file (service.js):
var environment;
var mongoose = require('mongoose');
module.exports.init = function(env, mongoose) {
environment = env;
mongoose = mongoose;
};
module.exports.useModel = function (modelName) {
var checkConnectionExists = (mongoose.connection.readyState === 1 || mongoose.connection.readyState === 2);
if(!checkConnectionExists)
mongoose.connect(environment.db.URL);
return require("./models/" + modelName)(mongoose);
};
module.exports.useModule = function (moduleName) {
return require("./modules/" + moduleName);
};
The modelObject.ImgSchema is not a constructor, however, modelObject.Images is.
var imageAdd = new imageModel.Images();
I'd probably rename Images to Image
I have the following problem:
In my application I have a model for all my sessions in the mongoDb-DB.
The idea is, that the model inherits from the abstract Model...so i don`t have to write all the standart functions again...like getting the collection etc.
So that is the Code of my sessionModel:
var baseProvider = require('./abstract/model').abstractProvider;
var queryString = require('querystring');
var _collection = 'sessions';
sessionProvider = function() {
baseProvider.apply(this, [_collection]);
};
sessionProvider.prototype = new baseProvider(_collection);
sessionProvider.prototype.deleteSession = function(sessionId){
this.getCollection(function(err,sessionCollection){
sessionCollection.remove({_id: sessionId},function(err,result){
console.log(err,result);
});
});
}
exports.sessionProvider = sessionProvider;
Then in my baseProvider(my "abstract Model"):
var Db = require('mongodb').Db;
var Connection = require('mongodb').Connection;
var Server = require('mongodb').Server;
var BSON = require('mongodb').BSON;
var ObjectID = require('mongodb').ObjectID;
abstractProvider = function(collectionName) {
var dbName = process.settings['shopconfig'].db.dbName;
var dbHost = process.settings['shopconfig'].db.dbHost;
var dbPort = process.settings['shopconfig'].db.dbPort;
this.db= new Db(dbName, new Server(dbHost, dbPort, {auto_reconnect: true}, {}));
this.db.open(function(){});
this.collectionName = collectionName;
};
abstractProvider.prototype.getCollection= function(callback) {
this.db.collection(this.collectionName, function(error, abstractCollection) {
if( error ) callback(error);
else callback(null, abstractCollection);
});
};
exports.abstractProvider = abstractProvider;
Ive shortend the codeparts to the important parts, dont be irritated.
The problem is, that the removefunction is executed, the callback gets called but the err-object and the result-object are undefined.
Is it false to call remove on the collection that way?
Im a bit puzzled...cause everything else works just fine...inserting stuff, updating, etc.
Edit: SessionID is filled btw, i looked at that already, cause i thought maybe it would be empty...but isn`t. Even calling remove() without criteria fails...
thanks for your time. =)
Is sessionId an instance of ...db.bson_serializer.ObjectID or are you shoving it into a session structure that converts it to a hex string? The latter happens with Cookie and Redis stores in Express, and cookie storage in general. Might be worth checking if you have a db reference in there:
sessionId = (typeof sessionId == "string")
? new db.bson_serializer.ObjectID(sessionId)
: sessionId