Sequelize on NodeJS/ExpressJS return an error when running CLI command db:migrate - node.js

I'm following this tutorial Using PostgreSQL and Sequelize to persist our data on medium, and right now I'm stuck at the db:migrate. it's returning this error
Sequelize CLI [Node: 12.1.0, CLI: 5.4.0, ORM: 5.8.2]
Loaded configuration file "config.json".
Using environment "development".
ERROR: Error parsing url: undefined
as you can see I'm using NodeJS version 12.1.0 and Sequelize CLI version 5.4.0 and Sequelize version 5.8.2 and all of them were the latest version.
and before running sequelize db:migrate, I'm running this command first SET DATABASE_URL=postgresql://[user[:password]#][netlocation][:port][/dbname] and it does not returns any error.
but it's returning error after I ran db:migrate
I already tried to find the problem, but I can't found the answer yet.
Here is my ./Models/Index.js file.
'use strict';
require('dotenv').config();
import { readdirSync } from 'fs';
import { basename as _basename, join } from 'path';
import Sequelize from 'sequelize';
const basename = _basename(__filename);
const env = process.env.NODE_ENV || 'development';
const config = require(__dirname + '/../../config.json')[env];
const db = {};
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(config.database, config.username, config.password, config);
}
readdirSync(__dirname)
.filter(file => {
return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js');
})
.forEach(file => {
const model = sequelize['import'](join(__dirname, file));
db[model.name] = model;
});
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
db[modelName].associate(db);
}
});
db.sequelize = sequelize;
db.Sequelize = Sequelize;
export default db;
if you realize I just changed it to ES6 format which change some codes, but before I change it to ES6, it doesn't work either. and for all the rest of the files I following the tutorial.
Here are the files that I think have a connection:
.env
DATABASE_URL=postgres://postgres:admin#localhost:5432/test_app
.sequelizerc
const path = require('path');
module.exports = {
"config": path.resolve('./config.json'),
"models-path": path.resolve('./app/Models'),
"migrations-path": path.resolve('./migrations')
};
config.json
{
"development": {
"use_env_variable": "DATABASE_URL"
},
"test": {
"use_env_variable": "DATABASE_URL"
},
"production": {
"use_env_variable": "DATABASE_URL"
}
}
If there are some files that I haven't included yet please tell me, and please help me to fix find the solution for this problem. Thank you
OS: Windows 10

Basically you are unable to set environment variable DATABASE_URL successfully.
I am not a Windows guy, but this should do your job.
If you are using GitBash, then it is as simple as:
export DATABASE_URL=postgres://postgres#localhost:5432/database_name
and after that:
node_modules/.bin/sequelize db:migrate
EDIT:
I am not sure how to set this variable in gitbash and cmd.
Here is an alternate.
in config/config.json
"development": {
"username": "postgres"
"password": "postgres",
"database": "your_db_here",
"host": "127.0.0.1",
"dialect": "postgres"
},
update these variables according to your postgres db.
and run:
node_modules/.bin/sequelize db:migrate

You cannot fetch values at runtime inside config.json. It has to be static.
You should either use config.json or env variables or roll your own like mentioned in another answer.
To use env variables, you will eschew config.json. Instead, in models/index.js, set
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(config.database, config.username, config.password, config);
}
to
sequelize = new Sequelize(process.env.DATABASE_URL)

AFAIK Sequelize migrations are a different beast than the normal sequelize workflow.
It is reading config/config.json when it loads - so you cannot use system environment variables - it has to be a static json file.
What I do in my projects, is having my config.js file making sure the config file is up to date with whatever settings I have.
I do this when the main program starts and also in package.json as follows:
(make sure to add npm-run-all to your package.json)
"scripts": {
"config": "node src/config.js",
"_migrate": "sequelize db:migrate",
"_migrate:status": "sequelize db:migrate:status",
"_migrate:undo": "sequelize db:migrate:undo",
"_seed": "sequelize db:seed:all",
"migrate": "npm-run-all config _migrate",
"migrate:status": "npm-run-all config _migrate:status",
"migrate:undo": "npm-run-all config _migrate:undo",
"seed": "npm-run-all config _seed"
},
config.js simply does something similar to this at the end of the file:
// Export sequelize config/config.json for easy compatibality with sequelize-cli
const filepath = path.resolve(__dirname, '../../config');
const filename = path.join(filepath, 'config.json');
fs.ensureDir(filepath)
.then(() => fs.writeFileSync(filename, JSON.stringify(sequelizeConfig, 2) + '\n'))
.catch((err) => console.error(`Failed to write config: ${err}`));
sequelizeConfig is should be the fully generated sequelize config object. You can also have a generic one like you have now, and build upon it.

Related

Is it possible to collect coverage on code loaded with vm.runInContext with Jest?

I'm working on a legacy JS project which is not using any require/import. When deploying, the files are just concatenated and the result is sent to a server.
In order to write tests with jest, I created a custom environment to load all the JS files in the global context so that I can call the functions in the test file.
For example:
src/index.js
function sum(x, y) {
return x + y;
}
src/index.spec.js
it('should sum two numbers', () => {
expect(sum(1, 2)).toBe(3);
});
jest.config.js
module.exports = {
clearMocks: true,
collectCoverage: true,
collectCoverageFrom: [
"src/**/*.js",
],
coverageDirectory: "coverage",
coverageProvider: "v8",
testEnvironment: "./jest.env.js",
};
jest.env.js
const NodeEnvironment = require('jest-environment-node').TestEnvironment;
const fs = require('fs');
const vm = require("vm");
const path = require("path");
class CustomEnv extends NodeEnvironment {
constructor(config, context) {
super(config, context);
this.loadContext();
}
loadContext() {
const js = fs.readFileSync('./src/index.js', 'utf8');
const context = vm.createContext(this.global);
vm.runInContext(js, context, {
filename: path.resolve('./src/index.js'),
displayErrors: true,
});
Object.assign(this.global, context);
}
}
module.exports = CustomEnv;
When I run npx jest, the test is executed but the coverage is empty...
Any idea on how to fix the coverage?
I've created a minimal reproducible repo here: https://github.com/GP4cK/jest-coverage-run-in-context/tree/main. You can just clone it, run npm i and npm t.
Note: I'm happy to change v8 to babel or load the context differently if it makes it easier.

Sequelize CLI & DB migrations with Typescript

Sequelize docs claim that it works with Typescript but in order for it to be useful in production one needs to use DB migration scripts. These can only be executed using the Sequelize CLI but this CLI seems to have no regard for Typescript at all. It only generates (and apparently runs) JS files. There is a "sequelize-cli-typescript" project on NPM but it's 4 years old! And the only answer on Stack Overflow to a related question is literally 6 words long, none of them useful: Using sequelize cli with typescript
My setup is like so:
I created a basic Node app with Typescript and a simple tsconfig.js like so:
{
"compilerOptions": {
"allowJs": true,
"module": "commonjs",
"esModuleInterop": true,
"target": "es6",
"noImplicitAny": true,
"moduleResolution": "node",
"sourceMap": true,
"outDir": "dist",
"baseUrl": ".",
"paths": {
"*": [
"node_modules/*"
]
}
},
"include": [
"*"
]
}
I'm also using ESLint with the following .eslintrc:
{
"root": true,
"parser": "#typescript-eslint/parser",
"plugins": [
"#typescript-eslint"
],
"extends": [
"airbnb-base", "airbnb-typescript/base"
],
"parserOptions": {
"project": "./tsconfig.json"
}
}
I then tried running the Sequelize CLI init command, npx sequelize-cli init as described in the DB migrations documentation. This caused a file models/index.js to be created with the following content:
'use strict';
const fs = require('fs');
const path = require('path');
const Sequelize = require('sequelize');
const basename = path.basename(__filename);
const env = process.env.NODE_ENV || 'development';
const config = require(__dirname + '/../config/config.json')[env];
const db = {};
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(config.database, config.username, config.password, config);
}
fs
.readdirSync(__dirname)
.filter(file => {
return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js');
})
.forEach(file => {
const model = require(path.join(__dirname, file))(sequelize, Sequelize.DataTypes);
db[model.name] = model;
});
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
db[modelName].associate(db);
}
});
db.sequelize = sequelize;
db.Sequelize = Sequelize;
module.exports = db;
This causes my IDE (VS Code) to show an error:
Parsing error: "parserOptions.project" has been set for #typescript-eslint/parser.
The file does not match your project config: models/index.js.
The file must be included in at least one of the projects provided.
I have no idea what this is supposed to mean. I've read numerous explanations and none makes sense to me. I have allowJs set to true in tsconfig and include all. What am I missing? More importantly how can I use Typescript for DB migrations and CLI generated code?
As for generated code for connecting to DB and registering models you should convert it manually to ts (a one-time thing usually).
As for migrations I use the following workaround:
configure migrations path in .sequelizerc to a subfolder in the folder where the compiled app will be placed by TS (it's important in order to be able to apply migrations on testing and prod environments), something like:
const { resolve } = require('path');
module.exports = {
config: resolve('build/application/config.js'),
'seeders-path': resolve('build/application/seeders'),
'migrations-path': resolve('build/application/migrations'),
'models-path': resolve('application/models')
};
generate a migration file with a given name:
sequelize migration:create --name add-some-table
move the generated file to application/migrations and change its extension to .ts
replace the content of the file with some kind of a migration Typescript-template:
import { QueryInterface, DataTypes, QueryTypes } from 'sequelize';
module.exports = {
up: (queryInterface: QueryInterface): Promise<void> => queryInterface.sequelize.transaction(
async (transaction) => {
// here go all migration changes
}
),
down: (queryInterface: QueryInterface): Promise<void> => queryInterface.sequelize.transaction(
async (transaction) => {
// here go all migration undo changes
}
)
};
add necessary changes to the migration file
Basically, it's possible to write some script file to simply generate .ts migration file copying a predefined template file in application/migrations folder directly with a given name to combine steps 2-4 into a single step.
If you will come up with a better solution eventually feel free to share it in the comments or even as a separate answer here.

import runs all code from module instead of just the imported function

This is my index.js file, located in the ./src directory:
import { MongoClient } from "mongodb";
import CharacterDAO from "./dao/character";
import GearDAO from "./dao/gear";
import { startServer } from "./server";
import { seedData } from "./dataSeed";
// connect mongoDb, seed data if needed, run fastify server
export const runServer = async ({ dbUrl, dbName, environment, port }) => {
// test seed data when starting server if running a test suite
if (environment === "test") {
await seedData({
hostUrl: dbUrl,
databaseName: dbName
});
}
await MongoClient.connect(dbUrl, {
poolSize: 50,
useNewUrlParser: true,
useUnifiedTopology: true,
wtimeout: 2500
})
.then(async conn => {
const database = await conn.db(dbName);
// inject database connection into DAO objects
CharacterDAO.injectDB(database);
GearDAO.injectDB(database);
// start the fastify server
startServer(port);
})
.catch(err => {
console.log(err.stack);
// process.exit(1);
});
};
const serverArguments = process.argv.slice(2).map(arg => {
return arg.split("=")[1];
});
const serverOptions = {
dbUrl: serverArguments[0],
dbName: serverArguments[1],
environment: serverArguments[2],
port: serverArguments[3]
};
runServer({
...serverOptions
});
jestconfig.json
{
"transform": {
"^.+\\.(t|j)sx?$": "ts-jest"
},
"testEnvironment": "node",
"testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$",
"moduleFileExtensions": ["ts", "tsx", "js", "jsx", "json", "node"]
}
Test script from package.json used to run the test (db credentials are omitted)
"test": "dbUrl=mongodb+srv://sdaw-dsawdad-dsadsawd#cluster0-jopi5.mongodb.net dbName=untitled-combat-game-test environment=test port=4000 jest --config jestconfig.json"
My test file:
import { runServer } from "../index";
beforeAll(async () => {
const serverOptions = {
dbUrl: process.env.dbUrl,
dbName: process.env.dbName,
environment: process.env.environment,
port: process.env.port
};
console.log(serverOptions);
await runServer({
...serverOptions
});
});
describe("mock test", () => {
it("should run a basic test", () => {
expect(true).toBe(true);
});
});
What happens when I run the test:
the test script runs runServer
the index.js file runs runServer
This causes a invalid URI error (since the process.argv referenced in index.js does not include a valid mongodb URI). I double-checked this by commenting out the runServer call at the bottom of my index.js file - and everything runs fine.
Moving the runServer function to a different file and importing it from there also solves the issue. So importing in both index.js and the test file does not result in multiple calls.
What am I doing wrong?
Importing/requiring a file evaluates the code inside of it (read: runs the code inside of it). You're not technically doing anything wrong, but for the purpose of your tests the code as you have written it won't work.
In your index.js file you are executing runServer(). Whenever that file is imported/required, that function call is also run.
Having a start.js file or similar which will actually start your server is a common pattern. This will help you avoid the issue you're experiencing.
I would split the definition of your server and invoking your server into two different files, say server.js and index.js. I will leave the fixing up of the imports to you, but this is the idea:
server.js
// connect mongoDb, seed data if needed, run fastify server
export const runServer = async ({ dbUrl, dbName, environment, port }) => {
// test seed data when starting server if running a test suite
if (environment === "test") {
await seedData({
hostUrl: dbUrl,
databaseName: dbName
});
}
await MongoClient.connect(dbUrl, {
poolSize: 50,
useNewUrlParser: true,
useUnifiedTopology: true,
wtimeout: 2500
})
.then(async conn => {
const database = await conn.db(dbName);
// inject database connection into DAO objects
CharacterDAO.injectDB(database);
GearDAO.injectDB(database);
// start the fastify server
startServer(port);
})
.catch(err => {
console.log(err.stack);
// process.exit(1);
});
};
index.js
import { runServer } from './server';
const serverArguments = process.argv.slice(2).map(arg => {
return arg.split("=")[1];
});
const serverOptions = {
dbUrl: serverArguments[0],
dbName: serverArguments[1],
environment: serverArguments[2],
port: serverArguments[3]
};
runServer({
...serverOptions
});

Sequelize - Cannot read property 'list' of undefined

Im just learn to use sequelize for my node.js project. For summary my project is ExpressJS+Typescript with Sequelize as ORM and Webpack as module bundler.
Below is my project structure.
src
-router
-server
--config
config.json
--controllers
index.ts
User.ts
--migrations
--models
index.js
user.js
--seeders
App.ts
index.ts
(sorry can not post picture yet, new user to stackoverflow)
I have build some simple router '/user' and expect it should call the user controller and call sequelize method findAll() from my models module, but the result is its error says Cannot read property 'list' of undefined. Below is my code:
models/index.js
const fs = require('fs');
const path = require('path');
const Sequelize = require('sequelize');
const basename = path.basename(module.filename);
const env = process.env.NODE_ENV || 'development';
const config = require(`${__dirname}/../config/config.json`)[env];
const db = {};
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable]);
} else {
sequelize = new Sequelize(
config.database, config.username, config.password, config
);
}
fs
.readdirSync(__dirname)
.filter(file =>
(file.indexOf('.') !== 0) &&
(file !== basename) &&
(file.slice(-3) === '.js'))
.forEach(file => {
const model = sequelize.import(path.join(__dirname, file));
db[model.name] = model;
});
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
db[modelName].associate(db);
}
});
db.sequelize = sequelize;
db.Sequelize = Sequelize;
export default db;
models/user.js
export default function(sequelize, DataTypes) {
var user = sequelize.define('user', {
username: DataTypes.STRING,
name: DataTypes.STRING,
email: DataTypes.STRING,
password: DataTypes.STRING,
phone: DataTypes.STRING,
wallet: DataTypes.DECIMAL
}, {
classMethods: {
associate: function(models) {
// associations can be defined here
user.hasMany(models.top_up);
}
}
});
return user;
};
controllers/User.ts
let user = require('../models').user;
export default {
list(req, res) {
return user
.findAll()
.then(topUp => res.status(200).send(topUp))
.catch(error => res.status(400).send(error));
}
};
controllers/Index.ts
import users from './User'
export default {
users
}
router/router.ts
import * as express from 'express';
const userController = require('../server/controllers').users;
// Init express router
let router = express.Router();
// Setting API URL
router.get('/', (req, res, next) => {
res.json({
message: 'Hello World!'
});
});
router.get('/about',(req, res, next) => {
res.send('<p>This is about about</p>');
});
router.get('/user', userController.list());
export default router
Fyi, all of my project configuration for start express server, typescript compile and webpack bundle is fine already, and the other route for '/' and '/about' is work fine. I know there is something I'm missing, im still new to sequelize, thanks for help.
TL;DR: The server/controllers/index.ts does not export a binding named users, itexports a binding named default.
You are importing the controllers/Index.ts module using the require function. In a CommonJS environment, this imports the entire module.exports object. As currently transpiled by TypeScript, every named export of the required module is exposed as a property of the import.
An export default clause implies an export named default. As per the ES Module specification, there is a shorthand for importing the default export of a module. That shorthand is
import userController from '../server/controllers';
On the other hand, the syntax
import userController = require('../server/controllers');
or
const userController = require('../server/controllers'); // (or let or var)
imports an object with a property corresponding to each export. In this case it has the shape
{ default }
So if you use require, you need to write
import userController = require('../server/controllers').default;
or
import userController from '../server/controllers';
All ES Module style exports are named, including the default which is named default.
To illustrate this, consider the following, more verbose but semantically identical form
import {default as userController} from '../server/controllers';
If you would prefer to stick with CommonJS style exports, eschewing ES Modules when working in NodeJS, the idiomatic way to export a single object as the entire module (the object returned by require)
You may write
// ../server/controllers/index.ts
export = {
list(req, res) {
return user
.findAll()
.then(topUp => res.status(200).send(topUp))
.catch(error => res.status(400).send(error));
}
};
Personally, I would stick with what you have and write
import userController from '../server/controllers';

auto-generate models for sequelize

I'm wanting to start using Sequelize, a module that allows ORM for mysql in node.js. I was wondering if it's possible to auto-generate the models like CakePHP does. In CakePHP, it will read the table's info, and automatically create the associations and fields with their types in the model. i'd really hate to have to completely map out all my tables by hand, as some are relatively large. Is there something out there that will do this for me? Or am I on my own to hand-type all the models out?
You can auto generate models through sequelize-auto. Just Follow the following link
https://github.com/sequelize/sequelize-auto
It will generate models of your table.
Now you can use sequelize-automate to generate models automatically. sequelize-auto seems to be unmaintained for a long time, and the package is out-of-date.
$ npm install sequelize-automate
$ ./node_modules/.bin/sequelize-automate --help
For example:
$ ./node_modules/.bin/sequelize-automate -t js -h localhost -d test -u root -p root -o models
Sequelizer - A desktop application to export sequelize models automatically and visually.
Pretty impressive GUI client made with ElectronJS, grab here:
Source: https://github.com/andyforever/sequelizer
You can use a sync method for each of model
example:
Object.keys(db).forEach((modelName) => {
db[modelName].sync().then(result => {
// some logic
}).catch(err => {
// some logic
})
});
the logic will create a new table if the table not exist
full script index.js
'use strict';
const fs = require("fs");
const path = require("path");
const Sequelize = require("sequelize");
const sequelize = new Sequelize(process.env.DB_DATEBASE, process.env.DB_USER, process.env.DB_PASS, {
host: process.env.DB_HOST,
port: process.env.DB_PORT,
dialect: 'mysql',
operatorsAliases: false
});
const db = {};
fs
.readdirSync(__dirname)
.filter((file) => {
return (file.indexOf(".") !== 0) && (file !== "index.js") && (file !== "migrations") && (file !== "redshift-migrations");
})
.forEach((file) => {
const model = sequelize.import(path.join(__dirname, file));
db[model.name] = model;
});
Object.keys(db).forEach((modelName) => {
if ("associate" in db[modelName]) {
db[modelName].associate(db);
}
db[modelName].sync().then(result => {
// some logic
}).catch(err => {
// some logic
})
});
db.sequelize = sequelize;
db.Sequelize = Sequelize;
module.exports = db;
the script takes all files(models) in given directory where you will put the index.js file
the structure looks like here:
Use npm install --save-dev sequelize-cli
Use npm install --save-dev sequelize-auto
npx sequelize-auto -o "./database/models" -d -h localhost -u root -p 3306 -x '' -e mysql
see https://github.com/sequelize/sequelize/issues/339
Sequelize provides methods to read the existing table names of a database. Furthermore there is a method to read the structure of a table. Combined, it should be possible to automate the creation of models.

Resources