Run Sequelize migration after deployment to App Engine - node.js

I have create a sample project for myself to try Node.js with Sequelize on Google App Engine.
I can run the project locally and it works fine, however when I deploy it to App Engine gcloud app deploy
I get the following error:
{
name: "SequelizeDatabaseError",
message: "ER_NO_SUCH_TABLE: Table 'sql8175762.Likes' doesn't exist",
parent: {
code: "ER_NO_SUCH_TABLE",
errno: 1146,
sqlState: "42S02",
index: 0,
sql: "SELECT `id`, `code`, `likes`, `createdAt`, `updatedAt` FROM `Likes` AS `Likes`;"
},
original: {
code: "ER_NO_SUCH_TABLE",
errno: 1146,
sqlState: "42S02",
index: 0,
sql: "SELECT `id`, `code`, `likes`, `createdAt`, `updatedAt` FROM `Likes` AS `Likes`;"
},
sql: "SELECT `id`, `code`, `likes`, `createdAt`, `updatedAt` FROM `Likes` AS `Likes`;"
}
Thats because the Sequelize migration doesnt run however I do have it specified in npm start command:
"scripts": {
"deploy": "gcloud app deploy",
"start": "npm install & sequelize db:migrate & NODE_ENV=production node ./bin/www"
},
I have never used app engine for node deployment and I am not even sure if the steps I am talking are correct to deploy, migrate and run the app.
Does anyone have any tips regarding this?

I had the same experience, the problem is that for some reasons you have to add tableName and freezeTableName: true property in the defining model. for instance:
'use strict';
module.exports = (sequelize, DataTypes) => {
const File = sequelize.define('File', {
name: DataTypes.STRING,
courseId: DataTypes.INTEGER
}, {
freezeTableName: true,
tableName: 'files'
});
File.associate = function(models) {
// associations can be defined here
};
return File;
};

Im a little late to this question, but in case others are looking, it's worth noting that the example script shown uses & between the commands.
This will background each command and immediately run the next, so there's a good chance that both npm install and sequelize db:migrate are still running when node launches.
If you're on a unix system (Mac/Linux) You can test this locally by creating a file called ./test.sh containing
#!/bin/sh
echo "Starting long process ..."
sleep 5
echo "Long process done"
and then from the command line running
chmod +x test.sh
./test.sh & echo "Running second process"
Changing the commands to && should fix this, ie:
npm install && sequelize db:migrate && NODE_ENV=production node ./bin/www

I don't know your setup, but do you have sequelize-cli package dependency, also have you tried adding a .sequelizerc file in your root directory and add the paths for your config, migration and models? Example:
var path = require('path');
module.exports = {
'config': path.resolve('./', 'config/config.js'),
'migrations-path': path.resolve('./', 'src/server/migrations'),
'seeders-path': path.resolve('./', 'src/server/seeders'),
'models-path': path.resolve('./', 'src/server/models')
};

you should use && instead of & in your start command.
& means to run the command at background.
&& means "logic and" so that the shell will run the command one by one and wait for each returned code.

Related

"No migrations pending" when attempting to run migrations , used to work with no problem

I have a web app, and I've written a migrator to create all my tables and relations, recently no matter what I try, typeorm does not appear to find this migrator and hence, does not run it.
My file structure (just the migrations)
src> Databas> Migrations>1663525805095-add_users.ts,1663529676790-make_institute_nullable.ts
ormconfig.ts
import { DataSource } from 'typeorm';
import { ConfigService } from '#nestjs/config';
import { config } from 'dotenv';
config();
const configService = new ConfigService();
const source = new DataSource({
type: 'postgres',
host: configService.get('POSTGRES_HOST'),
port: configService.get('POSTGRES_PORT'),
username: configService.get('POSTGRES_USER'),
password: configService.get('POSTGRES_PASSWORD'),
database: configService.get('POSTGRES_DB'),
synchronize: false,
logging: false,
migrations: ['src/database/migrations/*.ts'],
migrationsTableName: 'migrations',
entities: ['src/**/*.entity.ts'],
});
export default source;
In order to run this, I type yarn start:dev in order to get my Server started.
Then I run yarn migrations:run which I get:
query: SELECT * FROM current_schema()
query: SELECT version();
query: SELECT * FROM "information_schema"."tables" WHERE "table_schema" = 'public' AND "table_name" = 'migrations'
query: CREATE TABLE "migrations" ("id" SERIAL NOT NULL, "timestamp" bigint NOT NULL, "name" character varying NOT NULL, CONSTRAINT "PK_8c82d7f526340ab734260ea46be" PRIMARY KEY ("id"))
query: SELECT * FROM "migrations" "migrations" ORDER BY "id" DESC
No migrations are pending
When I look at my db, I see a migrations table with no entries.
I have tried to delete my migrator file and create it again with a more recent timestamp and that does not work either.
scripts from my package.json
"migrations:run": "yarn typeorm migration:run"
"typeorm": "typeorm-ts-node-commonjs -d ./ormconfig.ts"
"start:dev": "nest start --watch"
Other info
I'm using docker for the postgres DB and pgAdmin, it connects with no problem.
Any help would be greatly appreciated.

"Cannot use import statement outside a module" in typeorm migration when run nestjs app

I have created the nestjs app. In the root app folder I have these subfolders:
dist
migration
src
test
The migration folder contains typeorm migrations.
When run application with npm run start:dev I have this error:
import {MigrationInterface, QueryRunner} from "typeorm";
^^^^^^
SyntaxError: Cannot use import statement outside a module
at Module._compile (internal/modules/cjs/loader.js:891:18)
at Object.Module._extensions..js (internal/modules/cjs/loader.js:991:10)
at Module.load (internal/modules/cjs/loader.js:811:32)
at Function.Module._load (internal/modules/cjs/loader.js:723:14)
at Module.require (internal/modules/cjs/loader.js:848:19)
at require (internal/modules/cjs/helpers.js:74:18)
at Function.PlatformTools.load (C:\Users\dakru1\Documents\employo\employo-api\node_modules\typeorm\platform\PlatformTools.js:114:28)
at C:\Users\dakru1\Documents\employo\employo-api\node_modules\typeorm\util\DirectoryExportedClassesLoader.js:39:69
at Array.map (<anonymous>)
at Object.importClassesFromDirectories (C:\Users\dakru1\Documents\employo\employo-api\node_modules\typeorm\util\DirectoryExportedClassesLoader.js:39:10)
I understand the error message and I know how to fix it when it relates to application's code.
However, my problem is that this error come from typeorm migration file: [app-root-folder]\migration\1587067680466-Init.ts which shouldn't be used when application runs.
Why nestjs uses migration files. How can I ignore migration folder when running nestjs app?
to solve it just put the following code on your "scripts" at package.json:
"typeorm": "ts-node-dev ./node_modules/typeorm/cli.js",
After that you'll be able to run your typeorm migration:run :)
I found the solution:
This happends to me because I was using Nest and when you run the command nest start, under the hood, Nest executes node and tsc and node commands and because of that you need your files to be in JavaScript format. So, as the migrations are generated in Typescript... the statement import gives the error because is only for TypeScript.
The only thing i did to solve it is this:
migrations: ["dist/migrations/*.js"],
This is telling TypeOrm that search for the migration in the dist directory where all the JavaScript code was compiled previously.
Of course in my tsconfig.json file the output dir is pointing to dist.
"outDir": "./dist",
I had the same issue. I did the following as a workaround:
Set your ormconfig.json to look for migrations files with only js suffix:
"migrations": ["migrations/*.js"],
Install typescript globally: npm install -g typescript
After generating your migration, transpile typescript files to javascript files with typescript command:
tsc migrations/migration-file.ts
Run your migration: npm run typeorm migration:run
You can now run your application: npm run start:dev
I know this question is old, but I just came across it because I had the same problem. I was able to solve it by adding "migration/**/*.ts" to the "exclude" array in the /tsconfig.build.json file like so:
{
"extends": "./tsconfig.json",
"exclude": ["node_modules", "test", "dist", "**/*spec.ts", "migration/**/*.ts"]
}
The same issue here, and in my case the root cause was a wrong migrations path in the connection configuration, like this;
const databaseConfig: ConnectionOptions = {
name: 'default',
type: 'postgres',
host: process.env.TYPEORM_HOST || '',
port: Number(process.env.TYPEORM_PORT),
username: process.env.TYPEORM_USERNAME || '',
password: process.env.TYPEORM_PASSWORD || '',
database: process.env.TYPEORM_DATABASE || '',
logging: false,
synchronize: false,
entities: [],
migrations: ['src/migrations/*.ts'],
};
Instead, I have changed it to
const migrationPaths: string[] = process.env.TYPEORM_USE_CLI === 'true' ? ['src/migrations/*{.ts,.js}'] : [];
const databaseConfig: ConnectionOptions = {
name: 'default',
type: 'postgres',
host: process.env.TYPEORM_HOST || '',
port: Number(process.env.TYPEORM_PORT),
username: process.env.TYPEORM_USERNAME || '',
password: process.env.TYPEORM_PASSWORD || '',
database: process.env.TYPEORM_DATABASE || '',
logging: false,
synchronize: false,
entities: [],
migrations: migrationPaths,
};
another test you can try to see if the problem is really with migration is to move the migration filter out of src.
For TypeORM v0.3.0 or higher, glob patterns in migrations are DEPRECATED. It is recommended to directly include the migration classes.
entities, migrations, subscribers options inside DataSourceOptions accepting string directories support is deprecated. You'll be only able to pass entity references in the future versions.
import { DataSource, DataSourceOptions } from 'typeorm';
import { CreateUser1669370712331 } from '../../migrations/1669370712331-CreateUser';
import { UserModify1669457039074 } from '../../migrations/1669457039074-UserModify';
import { User } from '../users/entities/user.entity';
export const config: DataSourceOptions = {
type: 'sqlite',
database: 'db/database.sqlite',
logging: true,
synchronize: false,
entities: [User],
migrations: [CreateUser1669370712331, UserModify1669457039074],
};

"ERROR: Validation error" message when executing two Sequelize commands in "pretest" script

I'm writing tests for my project. It uses Sequelize and I thought about doing the following:
"pretest": "NODE_ENV=testing yarn sequelize db:migrate && yarn sequelize db:seed:all",
"test": "mocha --require #babel/register 'src/tests/**/*.spec.js'",
"posttest": "NODE_ENV=testing yarn sequelize db:migrate:undo:all"
But the following shows:
❯ yarn test
yarn run v1.19.2
$ NODE_ENV=testing yarn sequelize db:migrate && yarn sequelize db:seed:all
$ /home/gabriel/Workspace/graphql-apollo/node_modules/.bin/sequelize db:migrate
Sequelize CLI [Node: 12.13.1, CLI: 5.5.1, ORM: 5.21.2]
Loaded configuration file "src/config/database.js".
== 20191123132531-create-users: migrating =======
== 20191123132531-create-users: migrated (0.047s)
== 20191123132658-create-messages: migrating =======
== 20191123132658-create-messages: migrated (0.028s)
$ /home/gabriel/Workspace/graphql-apollo/node_modules/.bin/sequelize db:seed:all
Sequelize CLI [Node: 12.13.1, CLI: 5.5.1, ORM: 5.21.2]
Loaded configuration file "src/config/database.js".
== 20191123132945-users: migrating =======
ERROR: Validation error
error Command failed with exit code 1.
If I execute the migration and seeding command separately, it works fine. I also tried to use concurrently, but the same happens.
"pretest": "concurrently 'NODE_ENV=testing yarn sequelize db:migrate' 'yarn sequelize db:seed:all'",
#gamofe Add the --debug flag to the command to see more info about the error. e.g.
$ sequelize db:seed:all --debug
It's very likely you are getting this error because you are running sequelize db:seed:all without undoing previous seed. If your table already contains the data you are trying to seed in which some of them have unique constraint, you'll hit the unique constraint error.
To solve this you would need to run the sequelize db:seed:undo:all before seeding.
You can find more info here https://sequelize.org/master/manual/migrations.html
I had the same problem and managed to solve it as follows:
Wrong: "pretest": "NODE_ENV=test sequelize db:migrate && sequelize db:seed:all"
Right: "pretest": "NODE_ENV=test sequelize db:migrate && NODE_ENV=test sequelize db:seed:all"
Check your queryInterface.bulkInsert() or other insert statements are passed created_at and updated_at values. The seed part of sequelize does not automatically add this in - and the database has a NOT NULL restriction on these fields.
In my case I had to change the content of my seed file to include the createdAt and updatedAt fields.
I changed:
module.exports = {
up: async (queryInterface, Sequelize) => {
await queryInterface.bulkInsert('Users', [
{ id:1, name: `God`, type: UserTypes.Person },
]);
}
}
to:
module.exports = {
up: async (queryInterface, Sequelize) => {
const currentTime = new Date(new Date().toUTCString()).toISOString();
await queryInterface.bulkInsert('Users', [
{ id:++i, name: `God`, type: UserTypes.Person, createdAt: currentTime, updatedAt: currentTime },
]);
}
}
Note: I use the underscored: true option. If you don't you will need to use created_at and updated_at in place of createdAt and updatedAt.

How to execute npm script using grunt-run?

I have a npm task in my package.json file as follows to execute jest testing:
"scripts": {
"test-jest": "jest",
"jest-coverage": "jest --coverage"
},
"jest": {
"testEnvironment": "jsdom"
},
I want to execute this task npm run test-jest using grunt. I installed grunt-run for the same and added the run task, but how do I invoke this npm task there?
run: {
options: {
// Task-specific options go here.
},
your_target: {
cmd: 'node'
}
}
Configure your Gruntfile.js similar to the example shown in the docs.
Set the value for the cmd to npm.
Set run and test-jest in the args Array.
Gruntfile.js
module.exports = function (grunt) {
grunt.loadNpmTasks('grunt-run');
grunt.initConfig({
run: {
options: {
// ...
},
npm_test_jest: {
cmd: 'npm',
args: [
'run',
'test-jest',
'--silent'
]
}
}
});
grunt.registerTask('default', [ 'run:npm_test_jest' ]);
};
Running
Running $ grunt via your CLI using the configuration shown above will invoke the npm run test-jest command.
Note: Adding --silent (or it's shorthand equivalent -s) to the args Array simply helps avoids the additional npm log to the console.
EDIT:
Cross Platform
Using the grunt-run solution shown above failed on Windows OS when running via cmd.exe. The following error was thrown:
Error: spawn npm ENOENT Warning: non-zero exit code -4058 Use --force to continue.
For a cross-platform solution consider installing and utlizing grunt-shell to invoke the npm run test-jest instead.
npm i -D grunt-shell
Gruntfile.js
module.exports = function (grunt) {
require('load-grunt-tasks')(grunt); // <-- uses `load-grunt-tasks`
grunt.initConfig({
shell: {
npm_test_jest: {
command: 'npm run test-jest --silent',
}
}
});
grunt.registerTask('default', [ 'shell:npm_test_jest' ]);
};
Notes
grunt-shell requires load-grunt-tasks for loading the Task instead of the typical grunt.loadNpmTasks(...), so you'll need to install that too:
npm i -D load-grunt-tasks
For older version of Windows I had to install an older version of grunt-shell, namely version 1.3.0, so I recommend installing an earlier version.
npm i -D grunt-shell#1.3.0
EDIT 2
grunt-run does seem to work on Windows if you use the exec key instead of the cmd and args keys...
For cross platform purposes... I found it necessary to specify the command as a single string using the exec key as per the documentation that reads:
If you would like to specify your command as a single string, useful
for specifying multiple commands in one task, use the exec: key
Gruntfile.js
module.exports = function (grunt) {
grunt.loadNpmTasks('grunt-run');
grunt.initConfig({
run: {
options: {
// ...
},
npm_test_jest: {
exec: 'npm run test-jest --silent' // <-- use the exec key.
}
}
});
grunt.registerTask('default', [ 'run:npm_test_jest' ]);
};

nodemon : Passing arguments to the executable when using as a required module

I'm trying to start a script with nodemon, using it as a required module, and I cannot pass arguments correctly.
For example, for
var args = [
process.argv[0], '--harmony',
'/path/to/script.js', '-i', 'logs'
];`
I'm expecting the script to be launched as :
node --harmony /path/to/script.js -i logs
But it doesn't work and all I can manage to get is
node --harmony /path/to/script.js -i logs /path/to/script.js
This is what I tried :
var app = require('nodemon')({
script: args[2],
exec: args.join(' ')
});
I know about execMap, but it's no good as I cannot pass arguments at the end anyway.
How can it be done?
Skimming through the source code, I found the args config options (undocumented...). It turns out to be what I needed.
var app = require('nodemon')({
exec: args.slice(0, 2),
script: args[2],
args: args.slice(3)
});
I recommend use gulp with nodemon
var argv = require('optimist').argv
gulp = require("gulp"),
nodemon = require("gulp-nodemon");
gulp.task("default", [], function(){
nodemon({
script: 'app.js',
ignore: ["public/*"],
env: {'NODE_ENV': 'development'},
args: ["--port="+argv.port],
exec: "node --harmony"
}).on("start");
});

Resources