Sequelize Top level where with eagerly loaded models creates sub query - subquery

I'm running an into issue where Sequelize creates a subquery of the primary model and then joins the includes with that subquery instead of directly with the primary model table. The query conditions for the include(s) ends up inside the subquery's WHERE clause which makes it invalid. I have shortened names down trying to keep this compact hopefully without losing any relevant info.
Environment:
Nodejs: 6.11.3
Sequelize: 3.23.6 => Updated to 4.38.1 and problem persists
MySql: 5.7.23
Code snip models:
I.model:
models.I.hasMany(models.II);
models.I.belongsTo(models.CJ);
models.I.belongsTo(models.CJS);
II.model:
models.II.belongsTo(models.I);
CJ.model:
models.CJ.hasMany(models.I);
models.CJ.hasMany(models.CJS);
CJS.model:
models.CJS.hasMany(models.I);
Code snip query definition:
let where = { cId: '2',
iAmt: { '$gt': 0 },
'$or':
[ { '$CJ.a1$': {$like: '%246%'}} },
{ '$CJ.a2$': {$like: '%246%'} },
{ '$I.cPN$': {$like: '%246%'} }
] };
let query = {
where: where,
order: orderBy,
distinct: true,
offset: offset,
limit: limit,
include: [
{
model: CJ,
as: 'CJ',
required: false
}, {
model: CJS,
as: 'CJS',
required: false
}, {
model: II,
as: 'IIs',
required: false
}
]
};
I.findAll(query)
Produces SQL like the following:
SELECT `I`.*, `CJ`.`_id` AS `CJ._id`, `CJS`.`_id` AS `CJS._id`, `IIs`.`_id` AS `IIs._id`
FROM (SELECT `I`.`_id`, `I`.`CJId`, `I`.`CJSId`, `I`.`CId`
FROM `Is` AS `I`
WHERE `I`.`CId` = '2' AND
`I`.`iA` > 0 AND
(`CJ`.`a1` LIKE '%246%' OR
`CJ`.`a2` LIKE '%246%' OR
`I`.`cPN` LIKE '%246%'
)
ORDER BY `I`.`iNum` DESC LIMIT 0, 10) AS `I`
LEFT OUTER JOIN `CJs` AS `CJ` ON `I`.`CJId` = `CJ`.`_id`
LEFT OUTER JOIN `CJSs` AS `CJS` ON `I`.`CJSId` = `CJS`.`_id`
LEFT OUTER JOIN `IIs` AS `IIs` ON `I`.`_id` = `IIs`.`IId`
ORDER BY `I`.`iNum` DESC;
I was expecting something like this:
SELECT `I`.*, `CJ`.`_id` AS `CJ._id`, `CJS`.`_id` AS `CJS._id`, `IIs`.`_id` AS `IIs._id`
FROM `Is` AS `I`
LEFT OUTER JOIN `CJs` AS `CJ` ON `I`.`CJId` = `CJ`.`_id`
LEFT OUTER JOIN `CJSs` AS `CJS` ON `I`.`CJSId` = `CJS`.`_id`
LEFT OUTER JOIN `IIs` AS `IIs` ON `I`.`_id` = `IIs`.`IId`
WHERE `I`.`CId` = '2' AND
`I`.`iA` > 0 AND
(`CJ`.`a1` LIKE '%246%' OR
`CJ`.`a2` LIKE '%246%' OR
`I`.`cPN` LIKE '%246%'
)
ORDER BY `I`.`iNum` DESC LIMIT 0, 10
If I remove the II model from the include it does work and moves the the WHERE to the top level. I admit the structure of the query is not straight forward here, with I being a child of CJ and CJS, which in turn is a child of CJ. And then II a child of I. What am I missing here?
Bueller's or anyone's 2 cent welcome!

what happened here is because you also using order and limit together with eager loading association see the issue
to make it work, there is a little hacky solution, you need to add subQuery: false together to your root model query
let query = {
where: where,
order: orderBy,
distinct: true,
offset: offset,
limit: limit,
subQuery: false,
include: [...]
};

Related

Sequelize column reference is ambiguous

I'm trying to accomplish the task of joining all associated models, and then retrieving the count of all the models
I would want the data to be returned as
product {
main_dish: {},
side_dish: {},
drink: {},
purchase_product_count: x,
purchased_products: []
}
I have constructed this in sequelize to try to do this.
await Product.findAll({
include: [{
model: MainCourse,
required: false,
}, {
model: SideDish,
required: false,
}, {
model: Drink,
required: false,
}, {
model: ProductPurchase,
required: false,
where: {
createdAt: {[Op.between]: [two_days_before, current_date]}
},
}],
attributes: {
include: ['*',
[Sequelize.literal(
"(SELECT COUNT(*) from product_purchases as p where p.purchase_product_id = products.product_id)"), "product_purchase_count"],
]
},
offset: 1 * 5,
limit: 5
})
.then((results) => {
console.log(results)
res.status(200).json(results)
})
.catch((err) => {
console.log(err)
res.status(400).json(err)
})
Attempting to run this eager load gives this error
column reference "product_id" is ambiguous
This is the sql generated by sequelize
SELECT "products".*, "main_course"."main_course_id" AS "main_course.main_course_id", "main_course"."type" AS "main_course.type", "main_course"."createdAt" AS "main_course.createdAt", "main_course"."updatedAt" AS "main_course.updatedAt", "main_course"."deletedAt" AS "main_course.deletedAt", "main_course"."main_product_id" AS "main_course.main_product_id", "side_dish"."side_dish_id" AS "side_dish.side_dish_id", "side_dish"."type" AS "side_dish.type", "side_dish"."createdAt" AS "side_dish.createdAt", "side_dish"."updatedAt" AS "side_dish.updatedAt", "side_dish"."deletedAt" AS "side_dish.deletedAt", "side_dish"."side_product_id" AS "side_dish.side_product_id", "drink"."drink_id" AS "drink.drink_id", "drink"."type" AS "drink.type", "drink"."createdAt" AS "drink.createdAt", "drink"."updatedAt" AS "drink.updatedAt", "drink"."deletedAt" AS "drink.deletedAt", "drink"."drink_product_id" AS "drink.drink_product_id", "product_purchases"."product_purchase_id" AS "product_purchases.product_purchase_id", "product_purchases"."price" AS "product_purchases.price", "product_purchases"."quantity" AS "product_purchases.quantity", "product_purchases"."createdAt" AS "product_purchases.createdAt", "product_purchases"."updatedAt" AS "product_purchases.updatedAt", "product_purchases"."deletedAt" AS "product_purchases.deletedAt", "product_purchases"."purchase_product_id" AS "product_purchases.purchase_product_id", "product_purchases"."restaurant_order_id" AS "product_purchases.restaurant_order_id" FROM (SELECT "products"."product_id", "products"."title", "products"."price", "products"."createdAt", "products"."updatedAt", "products"."deletedAt", "products"."restaurant_id", "products".*, (SELECT COUNT(*) from product_purchases as p where p.purchase_product_id = products.product_id) AS "product_purchase_count" FROM "products" AS "products" WHERE ("products"."deletedAt" IS NULL) LIMIT 5 OFFSET 5) AS "products" LEFT OUTER JOIN "main_courses" AS "main_course" ON "products"."product_id" = "main_course"."main_product_id" AND ("main_course"."deletedAt" IS NULL) LEFT OUTER JOIN "side_dishes" AS "side_dish" ON "products"."product_id" = "side_dish"."side_product_id" AND ("side_dish"."deletedAt" IS NULL) LEFT OUTER JOIN "drinks" AS "drink" ON "products"."product_id" = "drink"."drink_product_id" AND ("drink"."deletedAt" IS NULL) LEFT OUTER JOIN "product_purchases" AS "product_purchases" ON "products"."product_id" = "product_purchases"."purchase_product_id" AND ("product_purchases"."deletedAt" IS NULL AND "product_purchases"."createdAt" BETWEEN '2022-05-06 08:35:12.577 +00:00' AND '2022-05-08 08:35:12.577 +00:00');
According to pgadmin the problem starts when doing the join on main_courses
ERROR: column reference "product_id" is ambiguous
LINE 1: ...EFT OUTER JOIN "main_courses" AS "main_course" ON "products"...
I seem to not understand how the sequelize join works as I am using the ORM method of not having any columns be the same. The foreign keys are all different from 'product_id'. Is product_id getting created and joined again on the aggregation possibly? Why am I getting this error, and how can I achieve the result of getting the count of the product_purchases along with the joined array of purchased products.
Obviously I could just get the purchased_products.length and then add it to the object itself, but I'm trying to figure out if you can do it through sequelize without having to hack it in.

How sequelize generate T-SQL query with sum, order and group?

I have this sequelize query, it gives a total taxable by article categories group.
const result = await models.Sale.findAll({
raw: true,
attributes: [
[Sequelize.literal('SUM(Sales.totalTaxable)'), 'total'],
"Article.category"
],
limit: 25,
offset: 0,
where: Sequelize.where(Sequelize.fn('YEAR', Sequelize.col('docDate')), year)
include: [{ model: models.Article, attributes: [] }],
group: ["Article.category"],
order: [[models.Article, "category", "DESC"], [Sequelize.literal("total"), "DESC"],],
})
and the output is that query
SELECT
SUM(Sales.totalTaxable) AS [total], [Article].[category]
FROM
[Sales] AS [Sales]
LEFT OUTER JOIN
[Articles] AS [Article] ON [Sales].[ArticleId] = [Article].[id]
AND ([Article].[deletedAt] IS NULL)
WHERE
(YEAR([docDate]) = N'2021'
GROUP BY
[Article].[category]
ORDER BY
[Sales].[id]
OFFSET 0 ROWS FETCH NEXT 25 ROWS ONLY;
It does not work because of ORDER BY [Sales].[id] but it's automatically added by sequelize and it's clearly unwanted.
(manually testing without it works)
I'm using sequelize v6.6.5, dialect: mssql (tedious)
Is this a sequelize bug or am I doing something wrong?

Sequelize query with two JOINs on custom fields

Currently I've got a solution with the following raw query, which works well.
Here's a SqlFiddle playground.
And here's the raw query that works well:
const sql = `SELECT u.*, g.name AS "gender"
FROM users u
INNER JOIN connections con ON con.user_id = u.id
INNER JOIN completed_quizzes q ON q.user_id = u.id
LEFT JOIN genders g ON g.id = u.gender_id
WHERE
con.other_user_id='foo'
AND q.quiz_id=1
AND con.status='approved'
AND u.deleted_at IS NULL
AND con.deleted_at IS NULL
AND q.deleted_at IS NULL
LIMIT 10
OFFSET 0
`;
But I'm trying to replace it with Sequelize's object modelling. So far, the most meaningful solution I've come up with this:
const sequelize = {
include: [
{
model: ConnectionsModel,
as: 'connections',
where: { otherUserId: userId, status: ConnectionStatus.approved },
required: true,
duplicating: false
},
{
model: CompletedQuizzesModel,
as: 'completedQuizzes',
where: { userId, quizId },
required: true,
duplicating: false
},
{
model: GendersModel
}
],
nest: true,
// have to specify `raw: false`, otherwise Sequelize returns only the first associated record
raw: false
};
It generates the following query (I've prettified it, for ease of reading):
SELECT u.*, g.name AS gender FROM users AS u
INNER JOIN connections AS con ON u.id = con.user_id AND (con.status = 'approved' AND con.other_user_id = 'foo')
INNER JOIN completed_quizzes AS q ON u.id = q.user_id AND (q.user_id = 'foo' AND q.quiz_id = '1')
LEFT OUTER JOIN genders AS g ON u.gender_id = g.id
WHERE u.deleted_at IS NULL
LIMIT 20
OFFSET 0;
But it returns nothing... no rows. Whereas, the original solution returns the rows as needed.
I can see the difference between the general where (which is applied to the whole query in the raw query) and the same attributes appended to the on clause in the joins. Not sure if that's the problem, though.
Here's the SqlFiddle playground again.
You added an extra condition in the include option CompletedQuizzesModel. In the original SQL query you don't have a condition on userId.
So this include should be like this:
{
model: CompletedQuizzesModel,
as: 'completedQuizzes',
where: { quizId }, // removed userId
required: true,
duplicating: false
}
The result query works just like the original one:
SELECT u.*, g.name AS gender FROM users AS u
INNER JOIN connections AS con ON u.id = con.user_id AND (con.status = 'approved' AND con.other_user_id = 'foo')
INNER JOIN completed_quizzes AS q ON u.id = q.user_id AND (q.quiz_id = '1')
LEFT OUTER JOIN genders AS g ON u.gender_id = g.id
WHERE u.deleted_at IS NULL
LIMIT 20
OFFSET 0;

Sequelize - How to pull records with 1 associated record and order by associated record attribute

I have a Convos table, which has many Messages.
What I want: pull all convos and last message. Order the convos by last_message.created_at
models.Convos.findAll({
include: [
{
model: models.Messages,
as: "last_message",
order: [ [ 'created_at', 'DESC' ]],
limit: 1,
}
],
where:{
[Op.or]: [
{
sender_id: req.decoded.id
},
{
recipient_id: req.decoded.id
}
],
},
)}
The closest I've gotten to ordering is with :
order: [
[{model: models.Messages, as: 'last_message'}, 'created_at', 'DESC'],
],
But this gives the error:
`Unhandled rejection SequelizeDatabaseError: missing FROM-clause entry for table "last_message"`
From here, I am guessing this error may allude that there are convos without any messages, making last_message.created_at undefined (I could be completely misunderstanding this error though).
So, from there, I have been trying to add a clause to the where statement that only pulls convos that have at least 1 message. Here are a bunch of things I've tried, and they all throw an error:
adding to where:
Sequelize.literal("`last_message`.`id` IS NOT NULL")
'$models.Messages.id$': { [Op.ne]: null },
Sequelize.fn("COUNT", Sequelize.col("last_message")): { [Op.gt]: 0 }
'$last_message.id$': { [Op.ne]: null }
'$last_message.id$': {
[Op.ne]: null
}
I've also tried having instead of a where statement:
having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col('$last_message.id$')), '>=', 0)
How can I properly sort the convos by it's associated record, last_message.created_at?
UPDATE - RELEVANT PARTS OF CONVOS MODEL
"use strict";
module.exports = (sequelize, DataTypes) => {
let Convos = sequelize.define(
"Convos",
{
sender_id: {
type: DataTypes.INTEGER,
references: {
model: "Users",
key: "id"
}
},
recipient_id: {
type: DataTypes.INTEGER,
references: {
model: "Users",
key: "id"
}
},
created_at: DataTypes.DATE,
updated_at: DataTypes.DATE
},
{
timestamps: false,
freezeTableName: true,
schema: "public",
tableName: "convos"
}
);
Convos.associate = models => {
Convos.hasMany(models.Messages, {
as: "last_message",
foreignKey: "convo_id",
sourceKey: "id"
});
};
return Convos;
};
UPDATE
I've figured out the issue is using Sequelize.literal when the associated model has limit. For example, this works:
models.Convos.findAll({
include: [
{
model: models.Messages,
as: "last_message",
order: [ [ 'created_at', 'DESC' ]],
//limit: 1,
required: true,
duplicating: false,
},
],
where: {
[Op.or]: [
{
sender_id: req.decoded.id
},
{
recipient_id: req.decoded.id
}
],
},
order: [[Sequelize.literal(`last_message.created_at`), 'DESC']],
offset: offset,
limit: 10,
}).then(convos => { ....
But when I uncomment the limit: 1 in the include part, I get the error:
Unhandled rejection SequelizeDatabaseError: missing FROM-clause entry for table "last_message"
Here is the query logs without limit 1:
Executing (default): SELECT "Convos"."id", "Convos"."sender_id", "Convos"."recipient_id", "Convos"."created_at", "Convos"."updated_at", "last_message"."id" AS "last_message.id", "last_message"."body" AS "last_message.body", "last_message"."read" AS "last_message.read", "last_message"."group_meeting_id" AS "last_message.group_meeting_id", "last_message"."user_id" AS "last_message.user_id", "last_message"."created_at" AS "last_message.created_at", "last_message"."updated_at" AS "last_message.updated_at", "last_message"."convo_id" AS "last_message.convo_id", "last_message->user"."id" AS "last_message.user.id", "last_message->user"."first_name" AS "last_message.user.first_name", "last_message->user"."avatar_file_name" AS "last_message.user.avatar_file_name", "senderUser"."id" AS "senderUser.id", "senderUser"."first_name" AS "senderUser.first_name", "senderUser"."avatar_file_name" AS "senderUser.avatar_file_name", "recipientUser"."id" AS "recipientUser.id", "recipientUser"."first_name" AS "recipientUser.first_name", "recipientUser"."avatar_file_name" AS "recipientUser.avatar_file_name" FROM "public"."convos" AS "Convos" INNER JOIN "public"."msgs" AS "last_message" ON "Convos"."id" = "last_message"."convo_id" LEFT OUTER JOIN "public"."users" AS "last_message->user" ON "last_message"."user_id" = "last_message->user"."id" LEFT OUTER JOIN "public"."users" AS "senderUser" ON "Convos"."sender_id" = "senderUser"."id" LEFT OUTER JOIN "public"."users" AS "recipientUser" ON "Convos"."recipient_id" = "recipientUser"."id" WHERE ("Convos"."sender_id" = 32 OR "Convos"."recipient_id" = 32) ORDER BY last_message.created_at DESC LIMIT 10 OFFSET 70;
Query with limit: 1:
Executing (default): SELECT "Convos"."id", "Convos"."sender_id", "Convos"."recipient_id", "Convos"."created_at", "Convos"."updated_at", "senderUser"."id" AS "senderUser.id", "senderUser"."first_name" AS "senderUser.first_name", "senderUser"."avatar_file_name" AS "senderUser.avatar_file_name", "recipientUser"."id" AS "recipientUser.id", "recipientUser"."first_name" AS "recipientUser.first_name", "recipientUser"."avatar_file_name" AS "recipientUser.avatar_file_name" FROM "public"."convos" AS "Convos" LEFT OUTER JOIN "public"."users" AS "senderUser" ON "Convos"."sender_id" = "senderUser"."id" LEFT OUTER JOIN "public"."users" AS "recipientUser" ON "Convos"."recipient_id" = "recipientUser"."id" WHERE ("Convos"."sender_id" = 32 OR "Convos"."recipient_id" = 32) ORDER BY last_message.created_at DESC LIMIT 10 OFFSET 0;
Here are some links that were useful in understanding limit is causing the issue, but I have still not found a solution that solves this problem.
Link 1
Link 2
Link 3
Link 4
Thanks!
Disclaimer: I do not know sequelize. And - there are three documented versions and you have not stated which one you are using.
What I want: pull all convos and last message. Order the convos by
last_message.created_at
I can offer a SQL (Postgres) solution.
I assume a convos table (something like):
CREATE TABLE convos (
id INT PRIMARY KEY,
sender_id INT,
recipient_id INT
);
and a messages table
CREATE TABLE messages (
id INT PRIMARY KEY,
convo_id INT REFERENCES convos (id),
created_at timestamp without time zone
);
And some test data:
INSERT INTO
convos (id, sender_id, recipient_id)
VALUES (1,1,1), (2,2,2), (3,3,4);
INSERT INTO
messages (id, convo_id, created_at)
VALUES
(1,1, '1990-07-24'),
(2,1, '2019-07-24'),
(3,2, '1990-07-24'),
(4,2, '2019-07-24'),
(5,3, null);
When you want to query the convos table and fetch the latest message from messages table you will have to use the result from messages in a subquery (or a CTE, or lateral join, ...).
For example:
SELECT
convos.sender_id,
convos.recipient_id,
messages.last_message_date
FROM convos
LEFT JOIN
(
SELECT convo_id, max(created_at) as last_message_date
FROM messages
GROUP BY convo_id
) messages ON convos.id=messages.convo_id
ORDER BY messages.last_message_date DESC
So for sequelize.js - you have to find out how it does subqueries with associated models and use the result.

Use JOIN in Sequelize

prompt how to use JOIN in Sequelize ?
I have the following code:
db.User.findAll({
attributes: ['id'],
where: {vipEnd: {lt: expiresVip}, vip: true},
limit: 100,
order: 'id'
}).then(function(users) {
In terms of 4, it looks like this:
SELECT "vkId" AS "id"
FROM "user" AS "user"
WHERE "user"."vipEnd" < 1469699683 --expiresVip
AND "user"."vip" = true
ORDER BY id
LIMIT '100';
How to achieve such a result?
SELECT "vkId" AS "id"
FROM "user" AS "user"
LEFT JOIN "notification" ON "notification"."userId" = "user"."vkId"
WHERE "user"."vipEnd" < 1469699683
AND "user"."vip" = true
AND "notification"."type" = 4
AND "notification"."expiresVipDate" < 1469699683
ORDER BY id
LIMIT '100';
Thank you!
You need to define association for models.
ex:
db.User.hasMany(db.Notification, {foreignKey: 'UserId'});
use properties include in sequelize:
db.User.findAll({
include: [{
model: db.Notification,
required: true//true INNER JOIN, false LEFT OUTER JOIN - default LEFT OUTER JOIN
}]}).then(function(user) {}, function(err) {
res.json(err, 400);});
You should consult more here
http://docs.sequelizejs.com/en/latest/docs/associations/

Resources