unable to create a relationship with sequelize in postgres - node.js

So i wish to create the following 2 tables : user and user_address and have a 1:many relationship among them, i.e a user can have multiple addresses. I followed the sequelize docs and ran the .belongsTo and hasMany on the models of these 2 tables, but i can't seem to query them when i request for a all users with include=[{model:'user_address'}] filter. I wanted to create a scalable system, so i used a somewhat modular approach.
The tables are defined in different files as js objects:
//user.js
const { DataTypes} = require('sequelize');
const name = "user"
const model ={
email : {type:DataTypes.STRING, allowNull:false,unique:true},
password : {type:DataTypes.STRING(16), allowNull:false},
phone : {type:DataTypes.STRING(20), allowNull:true},
name : {type:DataTypes.STRING(50), allowNull:true},
birthday : {type:DataTypes.STRING(50), allowNull:true},
}
const initConfig = {freezeTableName: true}
const getInstance = (email, pwd, phone = null, name = null, dob = null)=>{
return {email: email, password: pwd, phone: phone, name: name, birthday: dob}
}
module.exports = {
name:name,
model:model,
config:initConfig,
getInstance:getInstance
}
//user_address.js
const {DataTypes} = require("sequelize");
const name = "user_address"
const model = {
address : {type:DataTypes.STRING, allowNull:false},
zipcode : {type:DataTypes.INTEGER, allowNull:false},
is_main_address : {type:DataTypes.BOOLEAN, allowNull:false, default:false},
address_alias : {type:DataTypes.STRING, allowNull:true},
address_phone_number : {type:DataTypes.STRING, allowNull:true},
city : {type:DataTypes.STRING, allowNull:true},
}
// user_id : {type:DataTypes.INTEGER, allowNull:true, references: {model: 'user', key: 'id',},}
const config = {
freezeTableName: true
}
module.exports = {
name:name,
model:model,
config:config,
getInstance:()=>{},
}
then we have this ModelKeys.js class that provides an easy access to these model schemas:
const user = require("./user");
const note = require("./note");
const user_address = require('./user_addresses')
module.exports = {
ModelNames:{
USER : user,
USER_ADDRESS: user_address,
NOTE: note,
}
}
then we have this init db class that is supposed to initialise db, define models and run associations on them
const {Sequelize} = require('sequelize');
const {db_creds} = require("../.secrets/db_secrets");
const {ModelNames} = require("./models/model_keys");
const {logDB} = require("../a_commons/logutils");
let db = undefined
async function initTables() {
// create tables
Object.keys(ModelNames).forEach(key=>{
let model = ModelNames[key]
try {
logDB("initialising MODEL=",model==null? "null": `{js_obj : ${model.name}}`)
db.define(model.name,model.model,model.config)
logDB(`model created successfully : ${model.name}` )
}
catch (e) {logError(e)}
})
//sync with dbms
logDB("initialising sync with dbms server...")
await db.sync()
logDB("models synced successfully")
}
function validateTables() {
logDB("all tables are created.","tables=",db.models)
}
async function runAssociations(){
//create associations
logDB("init associations")
let userTable = db.models[ModelNames.USER.name]
let userAddressTable = db.models[ModelNames.USER_ADDRESS.name]
await userTable.hasMany(userAddressTable)
await userAddressTable.belongsTo(userTable)
logDB("associations created successfully")
//sync with dbms
logDB("initialising sync with dbms server...")
await db.sync()
logDB("models synced successfully")
}
module.exports = {
MyDatabase:{
initDB : async () => {
db = new Sequelize(db_creds)
await db
.authenticate()
.then(_ => logDB('Connection established successfully.'))
.then(_ => initTables())
.then(_ => validateTables())
.then(async _ => (await runAssociations()))
.catch(error => logError('Something went wrong', error))
},
getDB: ()=> {
// only useful once initDB is called
return db;
},
getTable : (modelName) => {
// only useful once initDB is called
return db.models[modelName]
}
}
}
if you notice, i did not created any identifier primary keys for my tables but since sequelize automatically generates id key, i was expecting the tables to get created successfully, which they do. during runAssociations() , function, i was also not sure, if it would work since the tables have already been created, and neither table has any reference to them, but i expected this to work.
this whole file runs when someone calls MyDatabase.initDB() and it runs without any errors.
therefore i further created a user_repo.js which looks like this :
const {MyDatabase} = require("./init_db");
const {ModelNames} = require("./models/model_keys");
const {logRepo} = require("../a_commons/logutils");
let userTable = null
module.exports = {
UserRepo: {
initDatabase: async () => {await MyDatabase.initDB()},
init: () => {
userTable = MyDatabase.getTable(ModelNames.USER.name)
logRepo("table initialised. table=",userTable.name)
},
createUser: async (user) => {
await userTable
.create(user)
.then(it => logRepo("user build successfully!", it.toJSON()))
},
getAllUsers: async () => {
let users = await userTable.findAll({include:[{model:MyDatabase.getTable(ModelNames.USER_ADDRESS.name)}]})
logRepo("available users = ", users.map(it=>it.email))
return users
},
getSingleUser: async (email) =>{
return await userTable.findOne(
{where: {email: email}, include:[{model:MyDatabase.getTable(ModelNames.USER_ADDRESS)}]},
)
}
}
}
however, when i run these lines:
const {UserRepo} = require("./user_repo");
const {ModelNames} = require("./models/model_keys");
async function x(){
let repo = UserRepo
await repo.initDatabase()
await repo.init()
await repo.createUser(ModelNames.USER.getInstance("ansh#12345.com","12345678"))
await repo.getAllUsers()
}
x()
1st few lines run perfectly, but for getAllUsers() line ,i get this error :
Executing (default): SELECT "user"."id", "user"."email", "user"."password", "user"."phone", "user"."name", "user"."birthday", "user"."createdAt", "user"."updatedAt", "user_addresses"."id" AS "user_addresses.id", "user_addresses"."address" AS "user_addresses.address", "user_addresses"."zipcode" AS "user_addresses.zipcode", "user_addresses"."is_main_address" AS "user_addresses.is_main_address", "user_addresses"."address_alias" AS "user_addresses.address_alias", "user_addresses"."address_phone_number" AS "user_addresses.address_phone_number", "user_addresses"."city" AS "user_addresses.city", "user_addresses"."createdAt" AS "user_addresses.createdAt", "user_addresses"."updatedAt" AS "user_addresses.updatedAt", "user_addresses"."userId" AS "user_addresses.userId" FROM "user" AS "user" LEFT OUTER JOIN "user_address" AS "user_addresses" ON "user"."id" = "user_addresses"."userId";
node:internal/process/promises:265
triggerUncaughtException(err, true /* fromPromise */);
^
Error
at Query.run (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/sequelize/lib/dialects/postgres/query.js:50:25)
at /Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/sequelize/lib/sequelize.js:314:28
at processTicksAndRejections (node:internal/process/task_queues:96:5)
at async PostgresQueryInterface.select (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/sequelize/lib/dialects/abstract/query-interface.js:407:12)
at async Function.findAll (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/sequelize/lib/model.js:1134:21)
at async Object.getAllUsers (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/db/user_repo.js:22:25)
at async x (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/db/test_io.js:101:5) {
name: 'SequelizeDatabaseError',
parent: error: column user_addresses.userId does not exist
at Parser.parseErrorMessage (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:287:98)
at Parser.handlePacket (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:126:29)
at Parser.parse (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:39:38)
at Socket.<anonymous> (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/index.js:11:42)
at Socket.emit (node:events:520:28)
at addChunk (node:internal/streams/readable:315:12)
at readableAddChunk (node:internal/streams/readable:289:9)
at Socket.Readable.push (node:internal/streams/readable:228:10)
at TCP.onStreamRead (node:internal/stream_base_commons:190:23) {
length: 120,
severity: 'ERROR',
code: '42703',
detail: undefined,
hint: undefined,
position: '839',
internalPosition: undefined,
internalQuery: undefined,
where: undefined,
schema: undefined,
table: undefined,
column: undefined,
dataType: undefined,
constraint: undefined,
file: 'parse_relation.c',
line: '3643',
routine: 'errorMissingColumn',
sql: 'SELECT "user"."id", "user"."email", "user"."password", "user"."phone", "user"."name", "user"."birthday", "user"."createdAt", "user"."updatedAt", "user_addresses"."id" AS "user_addresses.id", "user_addresses"."address" AS "user_addresses.address", "user_addresses"."zipcode" AS "user_addresses.zipcode", "user_addresses"."is_main_address" AS "user_addresses.is_main_address", "user_addresses"."address_alias" AS "user_addresses.address_alias", "user_addresses"."address_phone_number" AS "user_addresses.address_phone_number", "user_addresses"."city" AS "user_addresses.city", "user_addresses"."createdAt" AS "user_addresses.createdAt", "user_addresses"."updatedAt" AS "user_addresses.updatedAt", "user_addresses"."userId" AS "user_addresses.userId" FROM "user" AS "user" LEFT OUTER JOIN "user_address" AS "user_addresses" ON "user"."id" = "user_addresses"."userId";',
parameters: undefined
},
original: error: column user_addresses.userId does not exist
at Parser.parseErrorMessage (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:287:98)
at Parser.handlePacket (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:126:29)
at Parser.parse (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:39:38)
at Socket.<anonymous> (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/index.js:11:42)
at Socket.emit (node:events:520:28)
at addChunk (node:internal/streams/readable:315:12)
at readableAddChunk (node:internal/streams/readable:289:9)
at Socket.Readable.push (node:internal/streams/readable:228:10)
at TCP.onStreamRead (node:internal/stream_base_commons:190:23) {
length: 120,
severity: 'ERROR',
code: '42703',
detail: undefined,
hint: undefined,
position: '839',
internalPosition: undefined,
internalQuery: undefined,
where: undefined,
schema: undefined,
table: undefined,
column: undefined,
dataType: undefined,
constraint: undefined,
file: 'parse_relation.c',
line: '3643',
routine: 'errorMissingColumn',
sql: 'SELECT "user"."id", "user"."email", "user"."password", "user"."phone", "user"."name", "user"."birthday", "user"."createdAt", "user"."updatedAt", "user_addresses"."id" AS "user_addresses.id", "user_addresses"."address" AS "user_addresses.address", "user_addresses"."zipcode" AS "user_addresses.zipcode", "user_addresses"."is_main_address" AS "user_addresses.is_main_address", "user_addresses"."address_alias" AS "user_addresses.address_alias", "user_addresses"."address_phone_number" AS "user_addresses.address_phone_number", "user_addresses"."city" AS "user_addresses.city", "user_addresses"."createdAt" AS "user_addresses.createdAt", "user_addresses"."updatedAt" AS "user_addresses.updatedAt", "user_addresses"."userId" AS "user_addresses.userId" FROM "user" AS "user" LEFT OUTER JOIN "user_address" AS "user_addresses" ON "user"."id" = "user_addresses"."userId";',
parameters: undefined
},
sql: 'SELECT "user"."id", "user"."email", "user"."password", "user"."phone", "user"."name", "user"."birthday", "user"."createdAt", "user"."updatedAt", "user_addresses"."id" AS "user_addresses.id", "user_addresses"."address" AS "user_addresses.address", "user_addresses"."zipcode" AS "user_addresses.zipcode", "user_addresses"."is_main_address" AS "user_addresses.is_main_address", "user_addresses"."address_alias" AS "user_addresses.address_alias", "user_addresses"."address_phone_number" AS "user_addresses.address_phone_number", "user_addresses"."city" AS "user_addresses.city", "user_addresses"."createdAt" AS "user_addresses.createdAt", "user_addresses"."updatedAt" AS "user_addresses.updatedAt", "user_addresses"."userId" AS "user_addresses.userId" FROM "user" AS "user" LEFT OUTER JOIN "user_address" AS "user_addresses" ON "user"."id" = "user_addresses"."userId";',
parameters: {}
}
Process finished with exit code 1
Any idea on how to fix this?

You need to register all models and all their associations before calling sync because hasMany/belongsTo only registers associations inside Sequelize models do nothing in DB. Its the responsibility of sync to create the whole structure taking into account all registered models and associations.
await db
.authenticate()
.then(_ => logDB('Connection established successfully.'))
.then(_ => runAssociations())
.then(_ => initTables())
.then(_ => validateTables())
.catch(error => logError('Something went wrong', error))

Related

Sequelize: unable to delete a table entry due to a foreign key constraint issue

I am building a simple API to relearn how to use the sequelize ORM and am running into an issue with foreign key constraints. I have a many to many relationship between ThingsToDoLists and ThingsToDoListTags through ThingsToDoListTagJoins. This is what the models look like for each file:
thingstodolist.js
'use strict';
module.exports = (sequelize, DataTypes) => {
const ThingsToDoList = sequelize.define('ThingsToDoList', {
listName: DataTypes.STRING(128),
listDescription: DataTypes.STRING(128),
userId: DataTypes.INTEGER,
}, {});
ThingsToDoList.associate = function(models) {
ThingsToDoList.belongsTo(models.User, {
foreignKey: "userId",
})
ThingsToDoList.belongsToMany(
models.ThingsToDo,
{
through:"ThingsToDoTOThingsToDoListJoins",
otherKey: "thingToDoId",
foreignKey: "thingToDoListId"
}
)
ThingsToDoList.belongsToMany(
models.ThingsToDoListTag,
{
through: "ThingsToDoListTagJoins",
otherKey: "thingsToDoListTagId",
foreignKey: "thingsToDoListId",
}
)
// associations can be defined here
};
return ThingsToDoList;
};
thingstodotagjoins.js
'use strict';
module.exports = (sequelize, DataTypes) => {
const ThingsToDoListTagJoins = sequelize.define('ThingsToDoListTagJoins', {
thingsToDoListId: DataTypes.INTEGER,
thingsToDoListTagId: DataTypes.INTEGER
}, {});
ThingsToDoListTagJoins.associate = function(models) {
// associations can be defined here
};
return ThingsToDoListTagJoins;
};
thingstodolisttag.js
'use strict';
module.exports = (sequelize, DataTypes) => {
const ThingsToDoListTag = sequelize.define('ThingsToDoListTag', {
name: DataTypes.STRING(128)
}, {});
ThingsToDoListTag.associate = function(models) {
ThingsToDoListTag.belongsToMany(models.ThingsToDoList,
{
through: "ThingsToDoListTagJoins",
otherKey: "thingsToDoListId",
foreignKey: "thingsToDoListTagId"
})
// associations can be defined here
};
return ThingsToDoListTag;
};
I am trying to delete a ThingToDoList entry and the connection to the ThingToDoListTag, but not delete the ThingToDoListTag itself. When I try to use
await retrievedThingToDoList.destroy()
I get an error saying I voilated a foreign key constraint. I did try to use {"onDelete": "CASCADE", "hooks": true} and got the same issue. This is the specific error:
Executing (default): DELETE FROM "ThingsToDoLists" WHERE "id" = 1
ForeignKeyConstraintError [SequelizeForeignKeyConstraintError]: update or delete on table "ThingsToDoLists" violates foreign key constraint "ThingsToDoListTagJoins_thingsToDoListId_fkey" on table "ThingsToDoListTagJoins"
at Query.formatError (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/sequelize/lib/dialects/postgres/query.js:295:16)
at /Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/sequelize/lib/dialects/postgres/query.js:72:18
at tryCatcher (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/util.js:16:23)
at Promise._settlePromiseFromHandler (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/promise.js:547:31)
at Promise._settlePromise (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/promise.js:604:18)
at Promise._settlePromise0 (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/promise.js:649:10)
at Promise._settlePromises (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/promise.js:725:18)
at _drainQueueStep (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/async.js:93:12)
at _drainQueue (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/async.js:86:9)
at Async._drainQueues (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/async.js:102:5)
at Async.drainQueues [as _onImmediate] (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/async.js:15:14)
at process.processImmediate (node:internal/timers:471:21) {
parent: error: update or delete on table "ThingsToDoLists" violates foreign key constraint "ThingsToDoListTagJoins_thingsToDoListId_fkey" on table "ThingsToDoListTagJoins"
at Parser.parseErrorMessage (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/pg-protocol/dist/parser.js:287:98)
at Parser.handlePacket (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/pg-protocol/dist/parser.js:126:29)
at Parser.parse (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/pg-protocol/dist/parser.js:39:38)
at Socket.<anonymous> (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/pg-protocol/dist/index.js:11:42)
at Socket.emit (node:events:513:28)
at addChunk (node:internal/streams/readable:324:12)
at readableAddChunk (node:internal/streams/readable:297:9)
at Readable.push (node:internal/streams/readable:234:10)
at TCP.onStreamRead (node:internal/stream_base_commons:190:23) {
length: 374,
severity: 'ERROR',
code: '23503',
detail: 'Key (id)=(1) is still referenced from table "ThingsToDoListTagJoins".',
hint: undefined,
position: undefined,
internalPosition: undefined,
internalQuery: undefined,
where: undefined,
schema: 'public',
table: 'ThingsToDoListTagJoins',
column: undefined,
dataType: undefined,
constraint: 'ThingsToDoListTagJoins_thingsToDoListId_fkey',
file: 'ri_triggers.c',
line: '2553',
routine: 'ri_ReportViolation',
sql: 'DELETE FROM "ThingsToDoLists" WHERE "id" = 1',
parameters: undefined
}
Am I missing something in the relationships between the models or am I missing something else?

Sequelize trying to SELECT a column which doesn't exist of a join table

I'm having trouble querying a join table. I use, for example, the findAll() Model method. It gives me this error
code:
const { UserItems, sequelize } = require('./dbObjects');
const { Op } = require('sequelize');
async function run() {
const userItem = await UserItems.findAll();
console.log(user);
}
run();
error:
at async Function.findAll (E:\1dsktp\bot\node_modules\sequelize\dist\lib\model.js:1119:21)
at async run (E:\1dsktp\bot\dbTesting.js:14:15) {
name: 'SequelizeDatabaseError',
parent: [Error: SQLITE_ERROR: no such column: id] {
errno: 1,
code: 'SQLITE_ERROR',
sql: 'SELECT `id`, `amount` FROM `userItems` AS `userItems`;'
},
original: [Error: SQLITE_ERROR: no such column: id] {
errno: 1,
code: 'SQLITE_ERROR',
sql: 'SELECT `id`, `amount` FROM `userItems` AS `userItems`;'
},
sql: 'SELECT `id`, `amount` FROM `userItems` AS `userItems`;',
parameters: {}
}
For some reason, It's trying to look for a column named 'id', which of course doesn't exist.
My db is pretty simple, it consists of 3 tables, one being a join table:
User
Item
UserItems (join table and the one I'm having trouble with)
UserItems has two foreign keys, which are userId from User and itemId from Item. These were added associating both tables with the belongsToMany() method.
User.belongsToMany(Item, { through: UserItems, foreignKey: 'userId' });
Item.belongsToMany(User, { through: UserItems, foreignKey: 'itemId' });
Here is a picture of what the UserItems table looks like (I'm using SQLiteStudio for this)
Here is the UserItems model
module.exports = (sequelize, DataTypes) => {
const model = sequelize.define('userItems', {
amount: {
type: DataTypes.INTEGER,
defaultValue: 0,
allowNull: false,
},
}, {
timestamps: false,
});
return model;
};
Obviously, you need to add a primary key column for UserItems (both in a table and in a model) because Sequelize does not support composite primary and foreign keys and by default, it suggests you have id as a primary key if you don't define any primary key explicitly.

postgresql error when inserting a documet using express

I am making an app with Postgres and Nodejs but I get an error when inserting any doc.
Here are the table commands:
CREATE TABLE threads (
_id SERIAL NOT NULL PRIMARY KEY,
board_id INT NOT NULL REFERENCES board(_id),
text TEXT NOT NULL,
delete_password TEXT NOT NULL,
reported BOOLEAN NOT NULL,
created_on TIMESTAMP NOT NULL,
bumped_on TIMESTAMP
);
CREATE TABLE board (
_id SERIAL NOT NULL PRIMARY KEY,
name TEXT NOT NULL
);
CREATE TABLE replies (
_id SERIAL PRIMARY KEY ,
thread_id INT NOT NULL REFERENCES threads(_id),
text TEXT NOT NULL,
delete_password TEXT NOT NULL,
reported BOOLEAN NOT NULL
);
routes
const pool = require('../pg')
module.exports = function (app) {
// _id, text, created_on(date&time), bumped_on(date&time, starts same as created_on), reported(boolean), delete_password,
// table for every board
function exist(exist) {
const {_id } = exist.rows[0]
return _id
}
async function donotexist(name) {
const board = await pool.query('INSERT INTO board (name) values ($1) returning (_id)', [name])
return board.rows[0]._id
}
app.route('/api/threads/:board')
.get(async (req, res)=> {
const name = req.params.board
const thread_id = req.query
// let board = await pool.query
// ('SELECT * FROM board WHERE name = $1', [name]);
let thread = await pool.query('SELECT * FROM threads where _id = $1', [thread_id]);
let {board_id} = thread.rows[0]
let joined =
await pool.query('select * from board left join threads on board._id = threads.board_id where id = $1 ',[board_id])
})
.post(async (req, res)=> {
try{
const name = req.params.board;
console.log(name)
const { text, delete_password } = req.body;
console.log(req.body)
const exist = await pool.query('select * from board where name = $1', [name]);
let board_id;
if(exist.rows.length > 1) {
board_id = exist(exist)
}
board_id = donotexist(name)
let created_on = Date.now
let bumped_on = Date.now
let reported = false
const thread =
await pool.query(
'INSERT INTO threads (board_id, text, delete_password, reported ,created_on, bumped_on) values ($1, $2, $3, $4, $5, $6) RETURNING (_id)',
[board_id, text, delete_password, reported ,created_on, bumped_on])
console.log(thread)
const thread_id = thread.rows[0]._id
res.redirect(`b/${name}/${thread_id}/`)
}catch(e) {
console.error(e)
}
})
.put((req, res)=> {
const name = req.params.board
})
.delete((req, res)=> {
const name = req.params.board
})
app.route('/api/replies/:name')
.get((req, res)=> {
const name = req.params.board
})
.post((req, res)=> {
const name = req.params.board
})
.put((req, res)=> {
const name = req.params.board
})
.delete((req, res)=> {
const name = req.params.board
})
}
The error I receive when making a post request to /api/threads/:board
{ error: invalid input syntax for type integer: "{}"
at Parser.parseErrorMessage (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\parser.js:278:15)
at Parser.handlePacket (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\parser.js:126:29)
at Parser.parse (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\parser.js:39:38)
at Socket.stream.on (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\index.js:8:42)
at Socket.emit (events.js:198:13)
at addChunk (_stream_readable.js:288:12)
at readableAddChunk (_stream_readable.js:269:11)
at Socket.Readable.push (_stream_readable.js:224:10)
at TCP.onStreamRead [as onread] (internal/stream_base_commons.js:94:17)
length: 169,
name: 'error',
severity: 'ERROR',
code: '22P02',
detail: undefined,
hint: undefined,
position: undefined,
internalPosition: undefined,
internalQuery: undefined,
where: undefined,
schema: undefined,
table: undefined,
column: undefined,
dataType: undefined,
constraint: undefined,
file:
'd:\\pginstaller_12.auto\\postgres.windows-x64\\src\\backend\\utils\\adt\\numutils.c',
line: '259',
routine: 'pg_strtoint32' }
popps
{ board: 'popps', text: 'we', delete_password: 'wewe' }
(node:8472) UnhandledPromiseRejectionWarning: error: duplicate key value violates unique constraint "board_name_key"
at Parser.parseErrorMessage (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\parser.js:278:15)
at Parser.handlePacket (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\parser.js:126:29)
at Parser.parse (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\parser.js:39:38)
at Socket.stream.on (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\index.js:8:42)
at Socket.emit (events.js:198:13)
at addChunk (_stream_readable.js:288:12)
at readableAddChunk (_stream_readable.js:269:11)
at Socket.Readable.push (_stream_readable.js:224:10)
at TCP.onStreamRead [as onread] (internal/stream_base_commons.js:94:17)
(node:8472) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 2)
{ error: invalid input syntax for type integer: "{}"
at Parser.parseErrorMessage (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\parser.js:278:15)
at Parser.handlePacket (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\parser.js:126:29)
at Parser.parse (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\parser.js:39:38)
at Socket.stream.on (D:\boilerplate-project-messageboard\node_modules\pg-protocol\dist\index.js:8:42)
at Socket.emit (events.js:198:13)
at addChunk (_stream_readable.js:288:12)
at readableAddChunk (_stream_readable.js:269:11)
at Socket.Readable.push (_stream_readable.js:224:10)
at TCP.onStreamRead [as onread] (internal/stream_base_commons.js:94:17)
length: 169,
name: 'error',
severity: 'ERROR',
code: '22P02',
detail: undefined,
hint: undefined,
position: undefined,
internalPosition: undefined,
internalQuery: undefined,
where: undefined,
schema: undefined,
table: undefined,
column: undefined,
dataType: undefined,
constraint: undefined,
file:
'd:\\pginstaller_12.auto\\postgres.windows-x64\\src\\backend\\utils\\adt\\numutils.c',
line: '259',
routine: 'pg_strtoint32' }

nodeJS inserting Data into PostgreSQL error

I have a weird error using NodeJS with a PostgreSQL and I hope you can maybe help me out.
I have a huge amount of data sets, about 2 Million entries that I want to insert into my DB.
One data consists of 4 columns:
id: string,
points: float[][]
mid: float[]
occurences: json[]
I am inserting data like so:
let pgp = require('pg-promise')(options);
let connectionString = 'postgres://archiv:archiv#localhost:5432/fotoarchivDB';
let db = pgp(connectionString);
cityNet.forEach((arr) => {
db
.none(
"INSERT INTO currentcitynet(id,points,mid,occurences) VALUES $1",
Inserts("${id},${points}::double precision[],${mid}::double precision[],${occurences}::json[]",arr))
.then(data => {
//success
})
.catch(error => {
console.log(error);
//error
});
})
function Inserts(template, data) {
if (!(this instanceof Inserts)) {
return new Inserts(template, data);
}
this._rawDBType = true;
this.formatDBType = function() {
return data.map(d => "(" + pgp.as.format(template, d) + ")").join(",");
};
This works out for exactly for the first 309248 data pieces, then suddenly it just errors out with the following for (what it seems like) every next data it tries to insert:
{ error: syntax error at end of input
at Connection.parseE (/home/christian/Masterarbeit_reworked/projekt/server/node_modules/pg-promise/node_modules/pg/lib/connection.js:539:11)
at Connection.parseMessage (/home/christian/Masterarbeit_reworked/projekt/server/node_modules/pg-promise/node_modules/pg/lib/connection.js:366:17)
at Socket.<anonymous> (/home/christian/Masterarbeit_reworked/projekt/server/node_modules/pg-promise/node_modules/pg/lib/connection.js:105:22)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)
at readableAddChunk (_stream_readable.js:176:18)
at Socket.Readable.push (_stream_readable.js:134:10)
at TCP.onread (net.js:548:20)
name: 'error',
length: 88,
severity: 'ERROR',
code: '42601',
detail: undefined,
hint: undefined,
position: '326824',
internalPosition: undefined,
internalQuery: undefined,
where: undefined,
schema: undefined,
table: undefined,
column: undefined,
dataType: undefined,
constraint: undefined,
file: 'scan.l',
line: '1074',
routine: 'scanner_yyerror' }
The 'position' entry changes for every iterating error-message.
I can redo that and it will always error after 309248 entries.
When I try to insert less, like 1000 entries, the error does not occur.
That really confuses me. I thought PostgreSQL does not have any max amount of rows. Also the error message does not help me at all.
SOLVED
The error was found. In my data there were "null" entries that have slipped into it. Filtering out null-data worked out.
I will try out the other recommendations for inserting data, since the current way works, but the performance is very crappy.
I'm the author of pg-promise. Your whole approach should be changed to the one below.
Proper way to do massive inserts via pg-promise:
const pgp = require('pg-promise')({
capSQL: true
});
const db = pgp(/*connection details*/);
var cs = new pgp.helpers.ColumnSet([
'id',
{name: 'points', cast: 'double precision[]'},
{name: 'mid', cast: 'double precision[]'},
{name: 'occurences', cast: 'json[]'}
], {table: 'currentcitynet'});
function getNextInsertBatch(index) {
// retrieves the next data batch, according to the index, and returns it
// as an array of objects. A normal batch size: 1000 - 10,000 objects,
// depending on the size of the objects.
//
// returns null when there is no more data left.
}
db.tx('massive-insert', t => {
return t.sequence(index => {
const data = getNextInsertBatch(index);
if (data) {
const inserts = pgp.helpers.insert(data, cs);
return t.none(inserts);
}
});
})
.then(data => {
console.log('Total batches:', data.total, ', Duration:', data.duration);
})
.catch(error => {
console.log(error);
});
UPDATE
And if getNextInsertBatch can only get the data asynchronously, then return a promise from it, and update the sequence->source callback accordingly:
return t.sequence(index => {
return getNextInsertBatch(index)
.then(data => {
if (data) {
const inserts = pgp.helpers.insert(data, cs);
return t.none(inserts);
}
});
});
Related Links:
tx
sequence / spex.sequence
ColumnSet
Multi-row insert with pg-promise
I'm not sure, but it looks like you got wrong data structure at the last element(309249) and PostgreSQL cannot parse some property

Sequelize hasOne Relationship association method not a function?

I'm working on a backend that is utilizing Node and Sequelize ORM. The database being used is PostgreSQL. I've used hasOne() in the code before and it worked great all the createAssociation(), getAssociation() methods from the hasOne() relationship worked when I used it before. However for some reason it is giving me a problem now.
index.js
const Connection = new Sequelize(
'postgres',
'postgres',
null,
{
dialect: 'postgres',
host: '192.168.99.100',
port: '5432',
}
);
const ModelSeven = Connection.define('modelSeven', ModelSevenSchema),
ModelFive = Connection.define('modelFive', ModelFiveSchema),
ModelOne = Connection.define('modelOne', ModelOneSchema),
ModelSix = Connection.define('modelSix', ModelSixSchema),
ModelThree = Connection.define('modelThree', ModelThreeSchema),
ModelFour = Connection.define('modelFour', ModelFourSchema),
ModelOne.hasOne(ModelThree); // Defines hasOne() relationship
ModelOne.hasOne(ModelFour); // Defines hasOne() relationship
ModelSix.belongsToMany(ModelOne, {through: 'Pivot'});
ModelOne.belongsToMany(ModelSix, {through: 'Pivot' });
ModelOne.belongsTo(ModelSeven);
ModelThree.hasMany(ModelFive);
ModelFour.hasMany(ModelFive);
export default Connection;
utilize.js
import DB from './somePathToIndex.js';
//GraphQL resolve function
async resolve (root, args, context) {
//creates ModelOne
var ModelOne = await DB.models.ModelOne.create({
value: args.value,
valueTwo: args.valueTwo
});
// attempts to createModelThree/createModelFour from hasOne() relationship
var ModelThree = await ModelOne.createModelThree({});
var ModelFour = await ModelOne.createModelFour({});
// Maps through value array finding the modelFive with and id and adding
// it to modelThree via hasMany() relationship
args.value.map(async (id, key) => {
var ModelFive = await DB.models.ModelFive.findAll({
where: {
id: id
}
});
await ModelThree.addModelFive(ModelFive);
});
// Maps through value array finding the modelFive with and id and adding
// it to modelFour via hasMany() relationship
args.valueTwo.map(async (id, key) => {
var ModelFive = await DB.models.ModelFive.findAll({
where: {
id: id
}
});
await ModelFour.addModelFive(ModelFive);
});
return ModelOne;
}
The problem is that when node hits the line:
var ModelThree = await ModelOne.createModelThree({});
var ModelFour = await ModelOne.createModelFour({});
Gives me the following error:
"errors": [
{
"message": "ModelOne.createModelThree is not a function",
"locations": [
{
"line": 38,
"column": 3
}
]
}
]
This error is saying that the hasOne() relationship method create[Association]() is not a function. I cannot really tell why this is. If you know why please let me know.

Resources