I am building a simple API to relearn how to use the sequelize ORM and am running into an issue with foreign key constraints. I have a many to many relationship between ThingsToDoLists and ThingsToDoListTags through ThingsToDoListTagJoins. This is what the models look like for each file:
thingstodolist.js
'use strict';
module.exports = (sequelize, DataTypes) => {
const ThingsToDoList = sequelize.define('ThingsToDoList', {
listName: DataTypes.STRING(128),
listDescription: DataTypes.STRING(128),
userId: DataTypes.INTEGER,
}, {});
ThingsToDoList.associate = function(models) {
ThingsToDoList.belongsTo(models.User, {
foreignKey: "userId",
})
ThingsToDoList.belongsToMany(
models.ThingsToDo,
{
through:"ThingsToDoTOThingsToDoListJoins",
otherKey: "thingToDoId",
foreignKey: "thingToDoListId"
}
)
ThingsToDoList.belongsToMany(
models.ThingsToDoListTag,
{
through: "ThingsToDoListTagJoins",
otherKey: "thingsToDoListTagId",
foreignKey: "thingsToDoListId",
}
)
// associations can be defined here
};
return ThingsToDoList;
};
thingstodotagjoins.js
'use strict';
module.exports = (sequelize, DataTypes) => {
const ThingsToDoListTagJoins = sequelize.define('ThingsToDoListTagJoins', {
thingsToDoListId: DataTypes.INTEGER,
thingsToDoListTagId: DataTypes.INTEGER
}, {});
ThingsToDoListTagJoins.associate = function(models) {
// associations can be defined here
};
return ThingsToDoListTagJoins;
};
thingstodolisttag.js
'use strict';
module.exports = (sequelize, DataTypes) => {
const ThingsToDoListTag = sequelize.define('ThingsToDoListTag', {
name: DataTypes.STRING(128)
}, {});
ThingsToDoListTag.associate = function(models) {
ThingsToDoListTag.belongsToMany(models.ThingsToDoList,
{
through: "ThingsToDoListTagJoins",
otherKey: "thingsToDoListId",
foreignKey: "thingsToDoListTagId"
})
// associations can be defined here
};
return ThingsToDoListTag;
};
I am trying to delete a ThingToDoList entry and the connection to the ThingToDoListTag, but not delete the ThingToDoListTag itself. When I try to use
await retrievedThingToDoList.destroy()
I get an error saying I voilated a foreign key constraint. I did try to use {"onDelete": "CASCADE", "hooks": true} and got the same issue. This is the specific error:
Executing (default): DELETE FROM "ThingsToDoLists" WHERE "id" = 1
ForeignKeyConstraintError [SequelizeForeignKeyConstraintError]: update or delete on table "ThingsToDoLists" violates foreign key constraint "ThingsToDoListTagJoins_thingsToDoListId_fkey" on table "ThingsToDoListTagJoins"
at Query.formatError (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/sequelize/lib/dialects/postgres/query.js:295:16)
at /Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/sequelize/lib/dialects/postgres/query.js:72:18
at tryCatcher (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/util.js:16:23)
at Promise._settlePromiseFromHandler (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/promise.js:547:31)
at Promise._settlePromise (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/promise.js:604:18)
at Promise._settlePromise0 (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/promise.js:649:10)
at Promise._settlePromises (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/promise.js:725:18)
at _drainQueueStep (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/async.js:93:12)
at _drainQueue (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/async.js:86:9)
at Async._drainQueues (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/async.js:102:5)
at Async.drainQueues [as _onImmediate] (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/bluebird/js/release/async.js:15:14)
at process.processImmediate (node:internal/timers:471:21) {
parent: error: update or delete on table "ThingsToDoLists" violates foreign key constraint "ThingsToDoListTagJoins_thingsToDoListId_fkey" on table "ThingsToDoListTagJoins"
at Parser.parseErrorMessage (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/pg-protocol/dist/parser.js:287:98)
at Parser.handlePacket (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/pg-protocol/dist/parser.js:126:29)
at Parser.parse (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/pg-protocol/dist/parser.js:39:38)
at Socket.<anonymous> (/Users/aseefried/Projects/ThingsToDoApp/backend/node_modules/pg-protocol/dist/index.js:11:42)
at Socket.emit (node:events:513:28)
at addChunk (node:internal/streams/readable:324:12)
at readableAddChunk (node:internal/streams/readable:297:9)
at Readable.push (node:internal/streams/readable:234:10)
at TCP.onStreamRead (node:internal/stream_base_commons:190:23) {
length: 374,
severity: 'ERROR',
code: '23503',
detail: 'Key (id)=(1) is still referenced from table "ThingsToDoListTagJoins".',
hint: undefined,
position: undefined,
internalPosition: undefined,
internalQuery: undefined,
where: undefined,
schema: 'public',
table: 'ThingsToDoListTagJoins',
column: undefined,
dataType: undefined,
constraint: 'ThingsToDoListTagJoins_thingsToDoListId_fkey',
file: 'ri_triggers.c',
line: '2553',
routine: 'ri_ReportViolation',
sql: 'DELETE FROM "ThingsToDoLists" WHERE "id" = 1',
parameters: undefined
}
Am I missing something in the relationships between the models or am I missing something else?
So i wish to create the following 2 tables : user and user_address and have a 1:many relationship among them, i.e a user can have multiple addresses. I followed the sequelize docs and ran the .belongsTo and hasMany on the models of these 2 tables, but i can't seem to query them when i request for a all users with include=[{model:'user_address'}] filter. I wanted to create a scalable system, so i used a somewhat modular approach.
The tables are defined in different files as js objects:
//user.js
const { DataTypes} = require('sequelize');
const name = "user"
const model ={
email : {type:DataTypes.STRING, allowNull:false,unique:true},
password : {type:DataTypes.STRING(16), allowNull:false},
phone : {type:DataTypes.STRING(20), allowNull:true},
name : {type:DataTypes.STRING(50), allowNull:true},
birthday : {type:DataTypes.STRING(50), allowNull:true},
}
const initConfig = {freezeTableName: true}
const getInstance = (email, pwd, phone = null, name = null, dob = null)=>{
return {email: email, password: pwd, phone: phone, name: name, birthday: dob}
}
module.exports = {
name:name,
model:model,
config:initConfig,
getInstance:getInstance
}
//user_address.js
const {DataTypes} = require("sequelize");
const name = "user_address"
const model = {
address : {type:DataTypes.STRING, allowNull:false},
zipcode : {type:DataTypes.INTEGER, allowNull:false},
is_main_address : {type:DataTypes.BOOLEAN, allowNull:false, default:false},
address_alias : {type:DataTypes.STRING, allowNull:true},
address_phone_number : {type:DataTypes.STRING, allowNull:true},
city : {type:DataTypes.STRING, allowNull:true},
}
// user_id : {type:DataTypes.INTEGER, allowNull:true, references: {model: 'user', key: 'id',},}
const config = {
freezeTableName: true
}
module.exports = {
name:name,
model:model,
config:config,
getInstance:()=>{},
}
then we have this ModelKeys.js class that provides an easy access to these model schemas:
const user = require("./user");
const note = require("./note");
const user_address = require('./user_addresses')
module.exports = {
ModelNames:{
USER : user,
USER_ADDRESS: user_address,
NOTE: note,
}
}
then we have this init db class that is supposed to initialise db, define models and run associations on them
const {Sequelize} = require('sequelize');
const {db_creds} = require("../.secrets/db_secrets");
const {ModelNames} = require("./models/model_keys");
const {logDB} = require("../a_commons/logutils");
let db = undefined
async function initTables() {
// create tables
Object.keys(ModelNames).forEach(key=>{
let model = ModelNames[key]
try {
logDB("initialising MODEL=",model==null? "null": `{js_obj : ${model.name}}`)
db.define(model.name,model.model,model.config)
logDB(`model created successfully : ${model.name}` )
}
catch (e) {logError(e)}
})
//sync with dbms
logDB("initialising sync with dbms server...")
await db.sync()
logDB("models synced successfully")
}
function validateTables() {
logDB("all tables are created.","tables=",db.models)
}
async function runAssociations(){
//create associations
logDB("init associations")
let userTable = db.models[ModelNames.USER.name]
let userAddressTable = db.models[ModelNames.USER_ADDRESS.name]
await userTable.hasMany(userAddressTable)
await userAddressTable.belongsTo(userTable)
logDB("associations created successfully")
//sync with dbms
logDB("initialising sync with dbms server...")
await db.sync()
logDB("models synced successfully")
}
module.exports = {
MyDatabase:{
initDB : async () => {
db = new Sequelize(db_creds)
await db
.authenticate()
.then(_ => logDB('Connection established successfully.'))
.then(_ => initTables())
.then(_ => validateTables())
.then(async _ => (await runAssociations()))
.catch(error => logError('Something went wrong', error))
},
getDB: ()=> {
// only useful once initDB is called
return db;
},
getTable : (modelName) => {
// only useful once initDB is called
return db.models[modelName]
}
}
}
if you notice, i did not created any identifier primary keys for my tables but since sequelize automatically generates id key, i was expecting the tables to get created successfully, which they do. during runAssociations() , function, i was also not sure, if it would work since the tables have already been created, and neither table has any reference to them, but i expected this to work.
this whole file runs when someone calls MyDatabase.initDB() and it runs without any errors.
therefore i further created a user_repo.js which looks like this :
const {MyDatabase} = require("./init_db");
const {ModelNames} = require("./models/model_keys");
const {logRepo} = require("../a_commons/logutils");
let userTable = null
module.exports = {
UserRepo: {
initDatabase: async () => {await MyDatabase.initDB()},
init: () => {
userTable = MyDatabase.getTable(ModelNames.USER.name)
logRepo("table initialised. table=",userTable.name)
},
createUser: async (user) => {
await userTable
.create(user)
.then(it => logRepo("user build successfully!", it.toJSON()))
},
getAllUsers: async () => {
let users = await userTable.findAll({include:[{model:MyDatabase.getTable(ModelNames.USER_ADDRESS.name)}]})
logRepo("available users = ", users.map(it=>it.email))
return users
},
getSingleUser: async (email) =>{
return await userTable.findOne(
{where: {email: email}, include:[{model:MyDatabase.getTable(ModelNames.USER_ADDRESS)}]},
)
}
}
}
however, when i run these lines:
const {UserRepo} = require("./user_repo");
const {ModelNames} = require("./models/model_keys");
async function x(){
let repo = UserRepo
await repo.initDatabase()
await repo.init()
await repo.createUser(ModelNames.USER.getInstance("ansh#12345.com","12345678"))
await repo.getAllUsers()
}
x()
1st few lines run perfectly, but for getAllUsers() line ,i get this error :
Executing (default): SELECT "user"."id", "user"."email", "user"."password", "user"."phone", "user"."name", "user"."birthday", "user"."createdAt", "user"."updatedAt", "user_addresses"."id" AS "user_addresses.id", "user_addresses"."address" AS "user_addresses.address", "user_addresses"."zipcode" AS "user_addresses.zipcode", "user_addresses"."is_main_address" AS "user_addresses.is_main_address", "user_addresses"."address_alias" AS "user_addresses.address_alias", "user_addresses"."address_phone_number" AS "user_addresses.address_phone_number", "user_addresses"."city" AS "user_addresses.city", "user_addresses"."createdAt" AS "user_addresses.createdAt", "user_addresses"."updatedAt" AS "user_addresses.updatedAt", "user_addresses"."userId" AS "user_addresses.userId" FROM "user" AS "user" LEFT OUTER JOIN "user_address" AS "user_addresses" ON "user"."id" = "user_addresses"."userId";
node:internal/process/promises:265
triggerUncaughtException(err, true /* fromPromise */);
^
Error
at Query.run (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/sequelize/lib/dialects/postgres/query.js:50:25)
at /Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/sequelize/lib/sequelize.js:314:28
at processTicksAndRejections (node:internal/process/task_queues:96:5)
at async PostgresQueryInterface.select (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/sequelize/lib/dialects/abstract/query-interface.js:407:12)
at async Function.findAll (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/sequelize/lib/model.js:1134:21)
at async Object.getAllUsers (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/db/user_repo.js:22:25)
at async x (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/db/test_io.js:101:5) {
name: 'SequelizeDatabaseError',
parent: error: column user_addresses.userId does not exist
at Parser.parseErrorMessage (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:287:98)
at Parser.handlePacket (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:126:29)
at Parser.parse (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:39:38)
at Socket.<anonymous> (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/index.js:11:42)
at Socket.emit (node:events:520:28)
at addChunk (node:internal/streams/readable:315:12)
at readableAddChunk (node:internal/streams/readable:289:9)
at Socket.Readable.push (node:internal/streams/readable:228:10)
at TCP.onStreamRead (node:internal/stream_base_commons:190:23) {
length: 120,
severity: 'ERROR',
code: '42703',
detail: undefined,
hint: undefined,
position: '839',
internalPosition: undefined,
internalQuery: undefined,
where: undefined,
schema: undefined,
table: undefined,
column: undefined,
dataType: undefined,
constraint: undefined,
file: 'parse_relation.c',
line: '3643',
routine: 'errorMissingColumn',
sql: 'SELECT "user"."id", "user"."email", "user"."password", "user"."phone", "user"."name", "user"."birthday", "user"."createdAt", "user"."updatedAt", "user_addresses"."id" AS "user_addresses.id", "user_addresses"."address" AS "user_addresses.address", "user_addresses"."zipcode" AS "user_addresses.zipcode", "user_addresses"."is_main_address" AS "user_addresses.is_main_address", "user_addresses"."address_alias" AS "user_addresses.address_alias", "user_addresses"."address_phone_number" AS "user_addresses.address_phone_number", "user_addresses"."city" AS "user_addresses.city", "user_addresses"."createdAt" AS "user_addresses.createdAt", "user_addresses"."updatedAt" AS "user_addresses.updatedAt", "user_addresses"."userId" AS "user_addresses.userId" FROM "user" AS "user" LEFT OUTER JOIN "user_address" AS "user_addresses" ON "user"."id" = "user_addresses"."userId";',
parameters: undefined
},
original: error: column user_addresses.userId does not exist
at Parser.parseErrorMessage (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:287:98)
at Parser.handlePacket (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:126:29)
at Parser.parse (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/parser.js:39:38)
at Socket.<anonymous> (/Users/anshsachdeva/Downloads/f1_self/f4_web/f4_web_individual_gits/ecommerce/sequalize_tests/notes_db/node_modules/pg-protocol/dist/index.js:11:42)
at Socket.emit (node:events:520:28)
at addChunk (node:internal/streams/readable:315:12)
at readableAddChunk (node:internal/streams/readable:289:9)
at Socket.Readable.push (node:internal/streams/readable:228:10)
at TCP.onStreamRead (node:internal/stream_base_commons:190:23) {
length: 120,
severity: 'ERROR',
code: '42703',
detail: undefined,
hint: undefined,
position: '839',
internalPosition: undefined,
internalQuery: undefined,
where: undefined,
schema: undefined,
table: undefined,
column: undefined,
dataType: undefined,
constraint: undefined,
file: 'parse_relation.c',
line: '3643',
routine: 'errorMissingColumn',
sql: 'SELECT "user"."id", "user"."email", "user"."password", "user"."phone", "user"."name", "user"."birthday", "user"."createdAt", "user"."updatedAt", "user_addresses"."id" AS "user_addresses.id", "user_addresses"."address" AS "user_addresses.address", "user_addresses"."zipcode" AS "user_addresses.zipcode", "user_addresses"."is_main_address" AS "user_addresses.is_main_address", "user_addresses"."address_alias" AS "user_addresses.address_alias", "user_addresses"."address_phone_number" AS "user_addresses.address_phone_number", "user_addresses"."city" AS "user_addresses.city", "user_addresses"."createdAt" AS "user_addresses.createdAt", "user_addresses"."updatedAt" AS "user_addresses.updatedAt", "user_addresses"."userId" AS "user_addresses.userId" FROM "user" AS "user" LEFT OUTER JOIN "user_address" AS "user_addresses" ON "user"."id" = "user_addresses"."userId";',
parameters: undefined
},
sql: 'SELECT "user"."id", "user"."email", "user"."password", "user"."phone", "user"."name", "user"."birthday", "user"."createdAt", "user"."updatedAt", "user_addresses"."id" AS "user_addresses.id", "user_addresses"."address" AS "user_addresses.address", "user_addresses"."zipcode" AS "user_addresses.zipcode", "user_addresses"."is_main_address" AS "user_addresses.is_main_address", "user_addresses"."address_alias" AS "user_addresses.address_alias", "user_addresses"."address_phone_number" AS "user_addresses.address_phone_number", "user_addresses"."city" AS "user_addresses.city", "user_addresses"."createdAt" AS "user_addresses.createdAt", "user_addresses"."updatedAt" AS "user_addresses.updatedAt", "user_addresses"."userId" AS "user_addresses.userId" FROM "user" AS "user" LEFT OUTER JOIN "user_address" AS "user_addresses" ON "user"."id" = "user_addresses"."userId";',
parameters: {}
}
Process finished with exit code 1
Any idea on how to fix this?
You need to register all models and all their associations before calling sync because hasMany/belongsTo only registers associations inside Sequelize models do nothing in DB. Its the responsibility of sync to create the whole structure taking into account all registered models and associations.
await db
.authenticate()
.then(_ => logDB('Connection established successfully.'))
.then(_ => runAssociations())
.then(_ => initTables())
.then(_ => validateTables())
.catch(error => logError('Something went wrong', error))
Schema:
const locationSchema = new mongoose.Schema({
id:String,
name:{type: String , required:true},
address:String,
rating:{type:Number , "default":0, min:0 , max:5 },
facilities:[String],
// geoJSON schema 1
coords:
{
type: {type: String},
coordinates:[]
},
openingTimes: [openingTimeSchema],
reviews: [reviewSchema]
});
locationSchema.index({'coords':'2dsphere'})
endpoint : http://localhost:3000/api/locations?lng=-0.7992599&lat=51.378091
const theEarth = (function() {
console.log('theEarth');
const earthRadius = 6371; // km, miles is 3959
const getDistanceFromRads = function(rads) {
return parseFloat(rads * earthRadius);
};
const getRadsFromDistance = function(distance) {
return parseFloat(distance / earthRadius);
};
return {
getDistanceFromRads: getDistanceFromRads,
getRadsFromDistance: getRadsFromDistance
};
})();
module.exports.locationsListByDistance = (req,res) => {
const longitude = parseFloat(req.query.lng);
const latitude = parseFloat(req.query.lat);
//const maxDistance = parseFloat(req.query.maxDistance);
Loc.aggregate(
[{
$geoNear: {
near: { type: "Point", coordinates: [ longitude , latitude ] },
distanceField: "coords.calculated", // required
maxDistance: theEarth.getRadsFromDistance(20),
includeLocs: "coords.location",
spherical:true
}
}
],function(err,results){
console.log(results);
});
sendJsonResponse(res,200,results);
};
Error:
UnhandledPromiseRejectionWarning: MongoError: There is more than one 2dsphere index on Loc8r.locations; unsure which to use for $geoNear
at MessageStream.messageHandler (E:\Udemy\NodeTutorial\getting-mean\loc8r\node_modules\mongodb\lib\cmap\connection.js:261:20)
at MessageStream.emit (events.js:198:13)
at processIncomingData (E:\Udemy\NodeTutorial\getting-mean\loc8r\node_modules\mongodb\lib\cmap\message_stream.js:144:12)
at MessageStream._write (E:\Udemy\NodeTutorial\getting-mean\loc8r\node_modules\mongodb\lib\cmap\message_stream.js:42:5)
at doWrite (_stream_writable.js:415:12)
at writeOrBuffer (_stream_writable.js:399:5)
at MessageStream.Writable.write (_stream_writable.js:299:11)
at Socket.ondata (_stream_readable.js:709:20)
at Socket.emit (events.js:198:13)
at addChunk (_stream_readable.js:288:12)
at readableAddChunk (_stream_readable.js:269:11)
at Socket.Readable.push (_stream_readable.js:224:10)
at TCP.onStreamRead [as onread] (internal/stream_base_commons.js:94:17)
(node:5040) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async
function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 2)
GET /bootstrap/bootstrap.min.css 304 0.954 ms - -
GET /stylesheets/style.css 304 5.782 ms - -
GET /api/bootstrap/jquery.min.js 404 65.965 ms - 3123
GET /api/bootstrap/bootstrap.min.js 404 116.055 ms - 3123
I am using the above approach in order to find the documents whose distance is nearby the longitude and latitude put in the query params in the endpoint but results is returning undefined, please tell how to correct this error.
module.exports.locationsListByDistance = (req,res) => {
const longitude = parseFloat(req.query.lng);
const latitude = parseFloat(req.query.lat);
//const maxDistance = parseFloat(req.query.maxDistance);
// const options = {near :[longitude,latitude],maxDistance:10000}
Loc.find({
coords: {
$near : {
$maxDistance: 10,
$geometry : {
type : 'Point',
coordinates:[longitude,latitude]
}
}
}
}).find((error,results)=>{
if (error) console.log(error);
//console.log(JSON.stringify(results, 0, 2));
sendJsonResponse(res,200,results);
});
};
This is working fine.
You basically have multiple 2dsphere indexes. You must specify the one to use for this query. Try modifying your geoNear query to include the correct index:
$geoNear: {
near: { type: "Point", coordinates: [longitude, latitude] },
distanceField: "coords.calculated", // required
maxDistance: theEarth.getRadsFromDistance(20),
includeLocs: "coords.location",
spherical: true,
key: 'indexName'}
I have a weird error using NodeJS with a PostgreSQL and I hope you can maybe help me out.
I have a huge amount of data sets, about 2 Million entries that I want to insert into my DB.
One data consists of 4 columns:
id: string,
points: float[][]
mid: float[]
occurences: json[]
I am inserting data like so:
let pgp = require('pg-promise')(options);
let connectionString = 'postgres://archiv:archiv#localhost:5432/fotoarchivDB';
let db = pgp(connectionString);
cityNet.forEach((arr) => {
db
.none(
"INSERT INTO currentcitynet(id,points,mid,occurences) VALUES $1",
Inserts("${id},${points}::double precision[],${mid}::double precision[],${occurences}::json[]",arr))
.then(data => {
//success
})
.catch(error => {
console.log(error);
//error
});
})
function Inserts(template, data) {
if (!(this instanceof Inserts)) {
return new Inserts(template, data);
}
this._rawDBType = true;
this.formatDBType = function() {
return data.map(d => "(" + pgp.as.format(template, d) + ")").join(",");
};
This works out for exactly for the first 309248 data pieces, then suddenly it just errors out with the following for (what it seems like) every next data it tries to insert:
{ error: syntax error at end of input
at Connection.parseE (/home/christian/Masterarbeit_reworked/projekt/server/node_modules/pg-promise/node_modules/pg/lib/connection.js:539:11)
at Connection.parseMessage (/home/christian/Masterarbeit_reworked/projekt/server/node_modules/pg-promise/node_modules/pg/lib/connection.js:366:17)
at Socket.<anonymous> (/home/christian/Masterarbeit_reworked/projekt/server/node_modules/pg-promise/node_modules/pg/lib/connection.js:105:22)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)
at readableAddChunk (_stream_readable.js:176:18)
at Socket.Readable.push (_stream_readable.js:134:10)
at TCP.onread (net.js:548:20)
name: 'error',
length: 88,
severity: 'ERROR',
code: '42601',
detail: undefined,
hint: undefined,
position: '326824',
internalPosition: undefined,
internalQuery: undefined,
where: undefined,
schema: undefined,
table: undefined,
column: undefined,
dataType: undefined,
constraint: undefined,
file: 'scan.l',
line: '1074',
routine: 'scanner_yyerror' }
The 'position' entry changes for every iterating error-message.
I can redo that and it will always error after 309248 entries.
When I try to insert less, like 1000 entries, the error does not occur.
That really confuses me. I thought PostgreSQL does not have any max amount of rows. Also the error message does not help me at all.
SOLVED
The error was found. In my data there were "null" entries that have slipped into it. Filtering out null-data worked out.
I will try out the other recommendations for inserting data, since the current way works, but the performance is very crappy.
I'm the author of pg-promise. Your whole approach should be changed to the one below.
Proper way to do massive inserts via pg-promise:
const pgp = require('pg-promise')({
capSQL: true
});
const db = pgp(/*connection details*/);
var cs = new pgp.helpers.ColumnSet([
'id',
{name: 'points', cast: 'double precision[]'},
{name: 'mid', cast: 'double precision[]'},
{name: 'occurences', cast: 'json[]'}
], {table: 'currentcitynet'});
function getNextInsertBatch(index) {
// retrieves the next data batch, according to the index, and returns it
// as an array of objects. A normal batch size: 1000 - 10,000 objects,
// depending on the size of the objects.
//
// returns null when there is no more data left.
}
db.tx('massive-insert', t => {
return t.sequence(index => {
const data = getNextInsertBatch(index);
if (data) {
const inserts = pgp.helpers.insert(data, cs);
return t.none(inserts);
}
});
})
.then(data => {
console.log('Total batches:', data.total, ', Duration:', data.duration);
})
.catch(error => {
console.log(error);
});
UPDATE
And if getNextInsertBatch can only get the data asynchronously, then return a promise from it, and update the sequence->source callback accordingly:
return t.sequence(index => {
return getNextInsertBatch(index)
.then(data => {
if (data) {
const inserts = pgp.helpers.insert(data, cs);
return t.none(inserts);
}
});
});
Related Links:
tx
sequence / spex.sequence
ColumnSet
Multi-row insert with pg-promise
I'm not sure, but it looks like you got wrong data structure at the last element(309249) and PostgreSQL cannot parse some property
I've done some research and tried few things, like dropping the collection etc. Nothing has helped.
Code:
MongoClient.saveData = function(schemaDefinition, data, collectionName){
console.log("Schema definition: "+schemaDefinition+" collection name: "+collectionName);
var RecordSchema = new mongoose.Schema(schemaDefinition);//{ Email: String, FirstName: String});//({any: Schema.Types.Mixed });
console.log("Schema created.");
var RecordModel = mongoose.model(collectionName, RecordSchema);
console.log("Model created. Inserting in batches.")
RecordModel.insertMany(data)
.then(function(mongooseDocuments) {
console.log("Insertion was successful.");
})
.catch(function(err) {
console.log("Error while inserting to DB.")
});
The error:
/home/ubuntu/ds_queuesystem/node_modules/mongoose/lib/schema.js:381
var keys = Object.keys(obj);
^
TypeError: Object.keys called on non-object
at Function.keys (native)
at Schema.add (/home/ubuntu/ds_queuesystem/node_modules/mongoose/lib/schema.js:381:21)
at new Schema (/home/ubuntu/ds_queuesystem/node_modules/mongoose/lib/schema.js:98:10)
at Function.MongoClient.saveData (/home/ubuntu/ds_queuesystem/MongoClient.js:34:21)
at /home/ubuntu/ds_queuesystem/DS_QueueSystem.js:84:18
at nextTask (/home/ubuntu/ds_queuesystem/node_modules/async/dist/async.js:6627:18)
at /home/ubuntu/ds_queuesystem/node_modules/async/dist/async.js:6621:17
at /home/ubuntu/ds_queuesystem/node_modules/async/dist/async.js:339:31
at /home/ubuntu/ds_queuesystem/node_modules/async/dist/async.js:840:20
at /home/ubuntu/ds_queuesystem/DS_QueueSystem.js:143:3
at null.<anonymous> (/home/ubuntu/ds_queuesystem/node_modules/csv-parse/lib/index.js:71:16)
at EventEmitter.emit (events.js:117:20)
at _stream_readable.js:920:16
at process._tickCallback (node.js:415:13)
Schema definition:
var schemaDefinition = "{SCHID: String, Priority: Number, Status: String, Json: Schema.Types.Mixed})";
schemaDefinition should be an Object, not a literal.
Try with:
var schemaDefinition = {
SCHID: String,
Priority: Number,
Status: String,
Json: Schema.Types.Mixed
};
Documentation: http://mongoosejs.com/docs/guide.html