I am facing some problem with my DB connection design with sequelize.js. What I want to achieve is to have a centralize connection and configuration files for my application DB connection. Therefore I have created a file name database.js as below.
const Sequelize = require("sequelize");
const dbConfig = require("../../config/database.json");
const db = {};
sequelize = new Sequelize({
dialect: dbConfig.dialect,
database: dbConfig.database,
username: dbConfig.username,
password: dbConfig.password,
host: dbConfig.host,
port: dbConfig.port,
operatorsAliases: false,
logging: false,
pool: {
max: 5,
min: 0,
acquire: 30000,
idle: 10000
}
});
db.Sequelize = Sequelize;
db.sequelize = sequelize;
module.exports = db;
If there is any scripts going to use database, I just have to require the database.js file. However, there is a problem when my script is finished, the process is not exiting (terminal hang there) because of the sequelize connection is not close.
I have tried to call the close function on the finally block but this causing others query script not working (if I call it on every query block) due to the fact that they are sharing same instant. Once the first query done, then the connection will be closed.
sequelize
.query("SELECT * FROM users WHERE id = ?", {
replacements: [userId],
type: sequelize.QueryTypes.SELECT,
model: User,
mapToModel: true
})
.then(users => {
console.log(users);
})
.finally(() => {
sequelize.close();
});
I can close the connection on the last query, but it is stupid that whenever I got a new query that will need to execute at last, I will have to move the close to the new query block.
I am looking for a clean code that can help to maintain DB connection and also able to automatic close the connection when all scripts are executed.
sequelize.close() returns a promise so use async function and call
await sequelize.close()
Related
const dotenv = require('dotenv').config();
const Op = require('sequelize').Op;
//logger here
const Product = require('../models/product');
const MainCourse = require('../models/main_course');
const Drink = require('../models/drink');
const SideDish = require('../models/side_dishes.js');
const Sequelize = require('sequelize');
var database;
var username;
var password;
var host;
if (process.env.NODE_ENV !== 'production') {
database = process.env.DB_TABLE;
username = process.env.DB_USER;
password = process.env.DB_PASS;
host = process.env.DB_HOST;
}
const sequelize = new Sequelize(
database,
username,
password,
{
dialect: 'postgres',
host: host,
pool: {
max: 40,
min: 0,
idle: 20000000,
acquire: 100000000,
},
logging: console.log
}
);
sequelize
.authenticate()
.then(async () => {
sequelize.sync({force: true, logging: console.log}).then(() => console.log('Synced DB'))
console.log('Connection with Sequelize established successfully.');
})
.catch(err => {
console.error('Unable to connect via Sequelize:', err);
});
When I run this file through the terminal my result is:
Executing (default): SELECT 1+1 AS result
Connection with Sequelize established successfully.
Executing (default): SELECT 1+1 AS result
Connection with Sequelize established successfully.
Executing (default): SELECT 1+1 AS result
Synced DB
Why is the database not actually syncing even when the model files are imported? I'm trying to create a seed file where the database is dropped entirely before data is seeded. I want to authenticate the database connection and then drop all tables before recreating them, and then adding the fake data. How do you make the database sync immediately after authenticating the connection?
I am new to Node.js. I am trying to make connection pools with multiple databases. I have successfully made connection pools (i think) with below mentioned code. I know in order to execute query operations i have to do something at "Connection pool DBX Success", but i can't seem to figure out what to do so that i am able to execute queries on desired pool say crm1.execute or crm2.execute. What can i do here to achieve this. The only way i can think of is to write execute functions for each database separately which i know is wrong and i have to work with 15 databases so it isn't possible to write functions for all 15 databases separately.
const config = require("../config/config");
const oracledb = require("oracledb");
crm1 = config.crm1;
crm2 = config.crm2;
const crm1pool = oracledb.createPool ({
user: crm1.user,
password: crm1.password,
connectString: crm1.connectString,
poolMin: 1,
poolMax: 10,
poolTimeout: 300
}, (error,pool)=>{
if (error){
console.log(error);
}
console.log("Connection Pool DB1 success")
});
const crm2pool = oracledb.createPool ({
user: crm2.user,
password: crm2.password,
connectString: crm2.connectString,
poolMin: 1,
poolMax: 10,
poolTimeout: 300
}, (error,pool)=>{
if (error){
console.log(error);
}
console.log("Connection Pool DB2 success")
});
There is a lot of node-oracledb documentation on pooling and examples. Study those first.
Then you might find that giving each pool a poolAlias will let you easily choose which to use:
await oracledb.createPool({
user: 'hr',
password: myhrpw, // myhrpw contains the hr schema password
connectString: 'localhost/XEPDB1',
poolAlias: 'hrpool'
});
await oracledb.createPool({
user: 'sh',
password: myshpw, // myshpw contains the sh schema password
connectString: 'otherhost/OTHERDB',
poolAlias: 'shpool'
});
const connection = await oracledb.getConnection('hrpool');
const result = await connection.execute(
`SELECT manager_id, department_id, department_name
FROM departments
WHERE manager_id = :id`,
[103], // bind value for :id
);
console.log(result.rows);
I am creating a multi-tenant Saas App. I was advised by many to keep my separate clients on separate databases, for better security and easier management.
How do we connect multiple databases to the Node app?
I know how to make my app run with a single database connection to mongodb, but not sure about multiple connections.
The mongoose docs mentions the following solutions for multiple connections:
export schema pattern (https://mongoosejs.com/docs/connections.html#multiple_connections)
connection pools (which has only up to 5 connections, which may not be ideal as I may have hundreds of clients in the future)
Another way which I tried (and it works!), is connecting to mongodb during a node API call and executing my logic, as shown below. The code below is a test route for registering a user with name and email. dbutils() is a function that I call to connect to mongodb, using mongoose.connect(). I am not sure if this is a good practice to connect during the API call.
router.post('/:db/register', async (req,res, next) => {
const startTime = new Date();
try {
if(!req.body.name) {
throw new Error("Name required");
}
if(!req.body.email) {
throw new Error("Email required");
}
await dbutils(req.params.db);// connect to db
const session = await mongoose.startSession();
session.startTransaction();
const newUser = new User({
name: req.body.name,
email: req.body.email,
})
await newUser.save({session});
await session.commitTransaction();
session.endSession();
const endTime = new Date();
const diff = endTime.getTime() - startTime.getTime();
return res.json({
newUser: {
email: req.body.email,
name: req.body.name
},
db: req.params.db,
timeElapsed: diff,
});
} catch(ex) {
return next(ex);
}
})
My dbutils() code
const mongoose = require('mongoose');
const mongoURI = "mongodb://PC:27017,PC:27018,PC:27019";
module.exports = async function(db) {
try {
await mongoose.connect(
`${mongoURI}/${db}`,
{
useNewUrlParser: true,
useCreateIndex: true,
useFindAndModify: false,
useUnifiedTopology: true,
}
)
} catch(ex) {
throw ex
}
}
I would be very happy for any recommendation or solution to this problem. Thank you very much in advance for your answer.
It is never a good idea to connect to your DB in an API call, you will be wasting a lot of resources, and deplaying the API responses as well.
The best way for you would be connect to multiple databases when Application starts, along with connection pooling configuration.
You can specify which schema belongs to which connection, and maintain separate DB collections.
You can use below code to work with multiple connections, and pooling:
const connection1 = mongoose.createConnection('mongodb://username:password#host1:port1[?options]',{
poolSize: 10
});
const connection2 = mongoose.createConnection('mongodb://username:password#host2:port2[?options]',{
poolSize: 10
});
Models/Schema on connection 1 can be created as below:
//User schema on connection 1
const userSchema = new Schema({ ... });
const UserModel = connection1.model('User', userSchema);
module.exports = UserModel;
Models/Schema on connection 2 can be created as below:
//Product schema on connection 2
const productSchema = new Schema({ ... });
const ProductModel = connection2.model('Product', productSchema);
module.exports = ProductModel;
For better performance, you can also have shared DB clusters for each DB, and use the cluster to connect to your database.
const conn = mongoose.createConnection('mongodb://[username:password#]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]', options);
For detailed information, Please read Mongoose Multiple Connections, and Connection Pooling
I would like to use knex as a query builder, but my project already handles its own connection pool.
I wish I could to do something like:
const { Client } = require('pg')
const client = new Client()
await client.connect()
const knex = require('knex')({
client: 'pg',
connection: client,
})
Is there any way to provide knex with the pg client object, instead of letting it to manage its own connection pool?
Workaround
I think there is a way to do that, but it can be done only with a specific query. To do that you can use the connection(pool) method that accept a connection pool instance as argument.
In my case I have required knex without passing a connection argument and then when the connection was established I saved the connection instance in my object. Then when I have to use knex as client I passed the connection instance in the query building.
Code example
Here a code example:
const knex = require('knex')({
client: 'pg' // postgreSQL or whatever
});
const poolConnection = await new Client().connect(); // or any other method to create your connection here
// when you have to query your db
const results = await knex('users')
.connection(poolConnection) // here pass the connection
.where({
email: 'test#tester.com'
})
.select('id', 'email', 'createdAt')
.offset(0)
.limit(1)
.first();
Use case with moleculer
As an example, I have used that with moleculer that provides a Database Adapter that use already itself a SQL client, so knex would build an additional connection to my db. After I retrieved the connection in my microservice I've used that inside knex in the same way described above.
"use strict";
const DbService = require("moleculer-db");
const SQLAdapter = require("moleculer-db-adapter-sequelize");
const Sequelize = require("sequelize");
// here requiring knex without an actual connection
const knex = require("knex")({
client: "pg"
});
module.exports = {
name: "users",
// implementing moleculer ORM
mixins: [DbService],
adapter: new SQLAdapter(process.env.POSTGRECONNECTIONSTRING),
model: {
name: "user",
define: {
id: {
type: Sequelize.INTEGER,
primaryKey: true,
autoIncrement: true
},
email: Sequelize.STRING,
password: Sequelize.STRING,
}
},
actions: {
findByIdRaw: {
params: {
id: "number"
},
handler(ctx) {
const { id } = ctx.params;
// use the connection pool instance
return knex("users")
.connection(this.connection)
.where({
id
})
.select("id", "email", "createdAt")
.offset(0)
.limit(1)
.first();
}
}
},
started() {
// getting the connection from the adapter
return this.adapter.db.connectionManager.getConnection()
.then((connection) => {
// saving connection
this.connection = connection;
return Promise.resolve();
});
}
};
Related documentation
Knex.js
Knex.js-connection
Related links
Moleculer
Moleculer ORM
No. Unless you write your own custom dialect and override connection fetching functionality. Writing custom dialect is described here https://github.com/tgriesser/knex/blob/master/CONTRIBUTING.md#i-would-like-to-add-support-for-new-dialect-to-knex-is-it-possible
I can't seem to find how to log into a database in my FeathersJS app.
I would prefer to specify database info and login info in the service, but I just need it to work.
In myApp/config/default.json there is the following line:
"postgres": "postgres://postgress:#localhost:5432/feathers_db",
at:
http://docs.sequelizejs.com/en/latest/docs/getting-started/
It says that a string like the above should be:
"postgres": "postgres://username:user_password#localhost:5432/feathers_db",
But this does not work. It is also not very Feathers-like as now I am locked into one postgress db for all my postgress transactions.
In services/index.js there is the following line:
const sequelize = new Sequelize(app.get('postgres'), {
dialect: 'postgres',
logging: false
});
I could customize the above line to be what Sequelize says to do in their guide and have username and password as an argument, but then why is the template not already laid out like this?
There is also this line:
app.set('sequelize', sequelize);
If I have several postgress databases what do I do? DO I make new Sequelize objects and do something like:
app.set('sequelize_db1', sequelize_db1);
app.set('sequelize_db2', sequelize_db2);
Or do I specify db info, including user info in the service's model?
What does the logging in process for Postgress look like if one is using the generic db language rather than sequelize?
So in a word "yes". Everything I asked in my question was yes.
I can connect to the db like shown in the sequelize documentation. I can also configure the config.json file to have a "postgres" configuration with my user and db name. I could also place the full path in the services/index.js file when creating the new sequelize object. The best way to check that there is a connection is to have the following code after creating the new sequelize object:
new_sequelize_object
.authenticate()
.then(function(err) {
console.log('Connection to the DB has been established successfully.');
})
.catch(function (err) {
console.log('Unable to connect to the database:', err);
});
(taken from: http://docs.sequelizejs.com/en/latest/docs/getting-started/)
one can also define several sequelize objects and set them in the app. Then when defining the model in the specific service's index.js file, place the new bound name in the app.get('new_sequelize_object').
Here is the services/index.js file with two databases defined:
'use strict';
const service1 = require('./service1');
const authentication = require('./authentication');
const user = require('./user');
const Sequelize = require('sequelize');
module.exports = function() {
const app = this;
const sequelize = new Sequelize('feathers_db1', 'u1', 'upw', {
host: 'localhost',
port: 5432,
dialect: 'postgres',
logging: false
});
const sequelize2 = new Sequelize('postgres://u1:upw#localhost:5432/feathers_db2', {
dialect: 'postgres',
logging: false
});
app.set('sequelize', sequelize);
app.set('sequelize2', sequelize2);
sequelize
.authenticate()
.then(function(err) {
console.log('Connection to sequelize has been established successfully.');
})
.catch(function (err) {
console.log('Unable to connect to the database:', err);
});
sequelize2
.authenticate()
.then(function(err) {
console.log('Connection has been established to sequelize2 successfully.');
})
.catch(function (err) {
console.log('Unable to connect to the database:', err);
});
app.configure(authentication);
app.configure(user);
app.configure(service1);
};
And here is the service1/index.js file that uses service sequelize2:
'use strict';
const service = require('feathers-sequelize');
const service1 = require('./service1-model');
const hooks = require('./hooks');
module.exports = function(){
const app = this;
const options = {
//Here is where one sets the name of the differeng sequelize objects
Model: service1(app.get('sequelize2')),
paginate: {
default: 5,
max: 25
}
};
// Initialize our service with any options it requires
app.use('/service1', service(options));
// Get our initialize service to that we can bind hooks
const service1Service = app.service('/service1');
// Set up our before hooks
service1Service.before(hooks.before);
// Set up our after hooks
service1Service.after(hooks.after);
};