query multiple SQLalchemy ORM - python-3.x

I'm new in SQLalchemy I need to calculate multiple of some price in my one of the table. This is my tables:
class Order(DeclarativeBase):
__tablename__ = 'order'
id = Field(Integer, primary_key=True)
products = relationship("OrderProduct", back_populates="order", lazy='dynamic')
and
class OrderProduct(DeclarativeBase):
__tablename__ = 'order_products'
id = Field(Integer, primary_key=True)
order_id = Column(Integer, ForeignKey('order.id'), nullable=False)
order = relationship("Order", back_populates="products", protected=True)
product_id = Column(Integer, ForeignKey('product.id'), nullable=False)
product = relationship("Product", back_populates="order_product")
quantity = Field(Integer, nullable=False)
and
class Product(DeclarativeBase):
__tablename__ = 'product'
id = Field(Integer, primary_key=True)
price = Field(Integer)
order_product = relationship("OrderProduct", back_populates="product", protected=True)
I want to multiple price with this situation OrderProduct.quantity * Product.price and products in Order table is an array of Products
I write SQL query like this and it works:
SELECT SUM(price*quantity) FROM product
JOIN order_products ON product.id = order_products.product_id
JOIN order ON order_products.order_id = order.id;
I tried to make it in ORM like this but it takes me Product and I can calculate only price without multiple in quantity:
result = 0
for product in Product.query\
.join(OrderProduct).filter(Product.id == OrderProduct.product_id)\
.join(Order).filter(OrderProduct.order_id == self.id):
result = product.price + result
return result
I make this as #hybrid_property and its work well.
I use a framework that name is restfulpy. it has ORM for sqlalchemy, session in this framework is scoped_session, but it gives me SQL query in debuging mode instead of executing the query like this :
sum_price = {Query}SELECT sum(product.price * order_products.quantity) AS sum_1
FROM product JOIN order_products ON product.id = order_products.product_id JOIN "order" ON "order".id = order_products.order_id
WHERE product.id = order_products.product_id AND order_products.order_id = :order_id_1
Well, Can anyone help me out this problem?
with regards

I just realized that SQLalchemy is a great ORM! When you have a relation between two, three or ... tables, SQLalchemy make a join between them and you just query it!
I made it hard for me and my friends :) SQLalchemy is powerful than what I'm thinking!
This is the right answer:
#hybrid_property
def total_price(self):
return DBSession.query(
func.sum(Product.price * OrderProduct.quantity))\
.filter(OrderProduct.product_id == Product.id) \
.filter(OrderProduct.order_id == self.id).scalar()
DBSession is the same with session

from sqlalchemy import func
query = session.query(func.sum(Product.price*OrderProduct.quantity))
.join(OrderProduct).filter(Product.id == OrderProduct.product_id)
.join(Order).filter(OrderProduct.order_id == self.id)

Related

sqlalchemy: express relationship which depends on len of collection child in joined entity?

How do I express relationship which depends on len of collection child in joined entity?
In below example, parent entity is AlgoOrder. Child entity is Order. And PrivateTrade is child entity of Order.
AlgoOrder --> Order --> PrivateTrade
The problem I am having is with "orders_pending_private_trade_update".
class AlgoOrder(DbModel):
__tablename__ = "algo_order"
id = sa.Column(sa.Integer, primary_key=True)
... stuff ...
# https://docs.sqlalchemy.org/en/14/orm/loading_relationships.html
open_orders = orm.relation(Order, primaryjoin=and_(Order.algo_order_id == id, Order.status == 'OPEN'), lazy='select')
orders_pending_private_trade_update = orm.relation(Order, primaryjoin=and_(Order.algo_order_id == id, , Order.status == 'CLOSED', len(Order.private_trades)==0), lazy='select')
#property
def pending_orders(self):
return self.open_orders + self.orders_pending_private_trade_update
class Order(DbModel):
__tablename__ = "order_hist"
algo_order_id = sa.Column(sa.Integer, sa.ForeignKey("algo_order.id"))
... stiff ...
private_trades = orm.relation(PrivateTrade, primaryjoin=and_(PrivateTrade.order_id == order_id))
class PrivateTrade(DbModel):
__tablename__ = "private_trade"
id = sa.Column(sa.Integer, primary_key=True)
order_id = sa.Column(sa.String, sa.ForeignKey("order_hist.order_id"))
In particular, the error at "orders_pending_private_trade_update" was with "len" on Order.private_trades:
Exception has occurred: TypeError (note: full exception trace is shown but execution is paused at: _run_module_as_main) object of type 'InstrumentedAttribute' has no len()
So, I tried:
from sqlalchemy.sql.expression import func
orders_pending_private_trade_update = orm.relation(Order, primaryjoin=and_(Order.algo_order_id == id, Order.status == 'CLOSED', func.count(Order.private_trades)==0), lazy='select', viewonly=True)
But then error was "foreign key columns are present in neither the parent nor the child's mapped tables":
Can't determine relationship direction for relationship 'AlgoOrder.orders_pending_private_trade_update' - foreign key columns are present in neither the parent nor the child's mapped tables <class 'sqlalchemy.exc.ArgumentError'> Can't determine relationship direction for relationship 'AlgoOrder.orders_pending_private_trade_update' - foreign key columns are present in neither the parent nor the child's mapped tables
I checked my tables, I do have them:
op.create_table(
'algo_order',
sa.Column('id', sa.Integer(), primary_key=True),
...
op.create_table(
'order_hist',
sa.Column('id', sa.Integer(), primary_key=True),
sa.Column('algo_order_id', sa.Integer, sa.ForeignKey("algo_order.id")),
...
op.create_table(
'private_trade',
sa.Column('id', sa.Integer(), primary_key=True),
sa.Column('order_id', sa.String(), sa.ForeignKey("order_hist.order_id"))
...
Thanks in advance.
I think I found it, but syntax pretty ugly: I used closed_order.session to do a new Query
import sqlalchemy as sa
import sqlalchemy.orm as orm
from sqlalchemy.sql.expression import func
import sqlalchemy.dialects.postgresql as psql
from sqlalchemy.ext.mutable import MutableDict
from sqlalchemy.sql.expression import and_
class AlgoOrder(DbModel):
__tablename__ = "algo_order"
id = sa.Column(sa.Integer, primary_key=True)
... other stuff ...
open_orders = orm.relation(Order, primaryjoin=and_(Order.algo_order_id == id, Order.status == 'OPEN'), lazy='select')
closed_orders = orm.relation(Order, primaryjoin=and_(Order.algo_order_id == id, Order.status == 'CLOSED'), lazy='dynamic', viewonly=True)
#property
def orders_pending_private_trade_update(self):
order_ids_with_no_private_trades = [ order.id for order in list(self.closed_orders.session.query(Order.id, func.count(PrivateTrade.id).label('count_private_trades')).join(PrivateTrade, isouter=True).group_by(Order.id).having(func.count(PrivateTrade.id) == 0).all())]
orders_with_no_private_trades = self.closed_orders.session.query(Order).filter(Order.id.in_(order_ids_with_no_private_trades)).order_by(Order.id.desc()).limit(1000).all()
return orders_with_no_private_trades
#property
def pending_orders(self):
return list(self.open_orders) + list(self.orders_pending_private_trade_update)
I also don't like "first(100)" as an attempt to limit number of rows fetched. And how/when you dispose of the list to prevent memory leak? I think above approach is bad. Should use generator instead of returning list.
Essentially raw sql what I am looking for is a generator which returns below:
select
order_id,
cnt
from (
select
order_hist.id,
order_hist.order_id,
count(private_trade.id) cnt
from order_hist
left join private_trade on private_trade.order_id = order_hist.order_id
where order_hist.status in ('CLOSED', 'CANCELLED')
group by order_hist.id, order_hist.order_id
) src
where
cnt=0
Any better way to do this? I think my solution shows the sqlalchemy syntax but it's computationally inefficient.
Here's solution using generator instead to avoid MemoryError:
def order_hist_missing_private_trade_get(engine):
order_hist_missing_private_trade_sql = '''
select
order_id,
cnt
from (
select
order_hist.id,
order_hist.order_id,
count(private_trade.id) cnt
from order_hist
left join private_trade on private_trade.order_id = order_hist.order_id
where order_hist.status in ('CLOSED', 'CANCELLED')
group by order_hist.id, order_hist.order_id
) src
where
cnt=0
'''
with engine.connect() as conn:
# https://stackoverflow.com/questions/7389759/memory-efficient-built-in-sqlalchemy-iterator-generator
conn.execution_options(stream_results=True)
rs = conn.execute(order_hist_missing_private_trade_sql)
while True:
batch = rs.fetchmany(10000)
for row in batch:
order_id = row['order_id']
yield order_id
Usage:
from sqlalchemy import create_engine
connstr : str = "postgresql://postgres:your_secret#localhost/postgres"
engine = create_engine(connstr)
generator = order_hist_missing_private_trade_get(engine)
while True:
order_id = next(generator)
print(f"order_id: {order_id}")

Add filtering to a Flask-SQLAlchemy ORM many-to-many relation

Given two ORM models, class A and B, and a association table relation_a_b (defining a many-to-many relation between A and B), I wish to put a filter condition on the relationship. To explain the details in-depth, I will start off providing a minimal example for demonstration:
import db #some FlaskSQLAlchemy object
relation_a_b = \
db.Table(
'dolor',
db.Column(
'id',
db.Integer,
db.ForeignKey('ipsum.id'),
primary_key=True),
db.Column(
'id',
db.Integer,
db.ForeignKey('lorem.id'),
primary_key=True)
)
class A(db.Model):
__tablename__ = 'lorem'
id = db.Column(
db.Integer,
primary_key
)
b_rels = db.Relationship(
'B',
secondary=relation_a_b,
lazy = 'subquery',
backref=db.backref('a_rels')
)
class B(db.Model):
__tablename__ = 'ipsum'
id = db.Column(
db.Integer,
primary_key
)
#This object should only be allowed in
#relations with A, when ::activated=1
activated = db.Column(
db.Integer,
nullable=False
)
What I wish to achieve is, that the relationship b_rels, defined in class A, should filter for (B.activated == 1). That is, only existing relations in the table relation_a_b where B.activated == 1 should be present, when querying for b_rels. I am aware this can be done by filtering on b_rels everytime it is used, but I want the ORM to handle the filtering automatically.
Is this use case even possible, and if so, how?

integer out of range error in BigInteger sqlalchemy column on heroku

I'm trying to understand why I'm getting sql errors for the following objects.
I'm using a PostGresSql database on heroku
class Member(BASE):
__tablename__ = "members"
name = Column(String)
discord_id = Column(BigInteger, primary_key=True,nullable=False)
role = Column(String, ForeignKey('roles.name'))
nick = Column(String)
rs_runs = relationship("RSRun",secondary='member_on_rs_run')
def __init__(self,name,discord_id,nick="" ):
self.name = name
self.discord_id = discord_id
self.nick = nick
class RSRun(BASE):
__tablename__ = "rs_runs"
id = Column(Integer, primary_key=True, nullable=False)
level = Column(String)
dtg = Column(DateTime)
members = relationship("Member",secondary='member_on_rs_run')
class MemberOnRSRun(BASE):
__tablename__ = "member_on_rs_run"
member_id = Column(BigInteger, ForeignKey('members.discord_id'),primary_key = True)
run_id = Column(Integer,ForeignKey('rs_runs.id'),primary_key=True)
The errors I'm getting are here
sqlalchemy.exc.DataError: (psycopg2.errors.NumericValueOutOfRange) integer out of range
[SQL: INSERT INTO members (name, discord_id, role, nick) VALUES (%(name)s, %(discord_id)s, %(role)s, %(nick)s)]
[parameters: {'name': '', 'discord_id': 126793648221192192, 'role': None, 'nick': ''}]
I've checked and the discord_id is within legal range of a PostGres BigInt which is what I'm assuming I'm getting with an sqlalchemy Biginteger Column. But it keeps telling me it's out of range for integer. Obviously I want to use the same ID that Discord is using since that is the identifier for a member on Discord.
Making changes to the python objects that you are using in the sqlalchemy declarative form will not FORCE heroku to update the database. You must manually blow away the database for these changes to take affect.

I am not able to load Self Reference in SQLAlchemy

I currently having an issue with a self reference relationship.
I have the table Customer which can have a parent (also a Customer) like this:
class CustomerModel(db.Model):
__tablename__ = 'customer'
id = Column(Integer, primary_key=True)
parent_id = Column(Integer, ForeignKey('customer.id'))
parent = relationship("CustomerModel", foreign_keys=[parent_id])
So my problem is that when I'm trying to load the parent, the following query is built by SQLAlchemy:
Lets take this customer for example: Customer(id=1, parent_id=10)
SELECT *
FROM customer
WHERE 1 = customer.parent_id
So the WHERE condition is wrong because it compares the parent_id to the id of the customer I'm trying to load the parent from.
The correct query should be:
SELECT *
FROM customer
WHERE 10 = customer.parent_id
What am I doing wrong?
So i finally found the answer.
I need to add the param remote_side like this:
class CustomerModel(db.Model):
__tablename__ = 'customer'
id = Column(Integer, primary_key=True)
parent_id = Column(Integer, ForeignKey('customer.id'))
parent = relationship("CustomerModel", foreign_keys=[parent_id], remote_side=[id])

Trying to join two tables in sqlalchemy orm query, getting an error

Trying to join the tables below using this command:
Subscription.query.filter( return Subscription.query.filter(Subscription.watch_id == id).join(User).filter_by(watch_id=id)
I get this error:
sqlalchemy.exc.InvalidRequestError: Could not find a FROM clause to join from. Tried joining to <class 'app.user.model.User'>, but got: Can't find any foreign key relationships between 'wm_subscription' and 'user'.
Essentially my end goal is to get a query that gets a List of Users that share a watch_id. Not sure if the models or the query is correct. Anybody know what's wrong?
Database = declarative_base(cls=DbBase)
class Subscription(Database):
__tablename__ = 'wm_subscription'
subscription_id = UniqueIdPk()
watch_id = UniqueIdRefNotNull(index=True)
user_id = UniqueIdRefNotNull(ForeignKey('User.user_id'), index=True)
subscription_watch = relationship('Watch',
primaryjoin='Subscription.watch_id == Watch.watch_id',
foreign_keys='Watch.watch_id',
uselist=True)
subscription_user = relationship('User',
primaryjoin='Subscription.watch_id == User.user_id',
foreign_keys='User.user_id',
uselist=True,
backref='user')
class User(Database, UserMixin):
__tablename__ = 'user'
user_id = UniqueIdPk()
# Google sub ID - unique to user https://developers.google.com/identity/protocols/OpenIDConnect
google_id = Column(String(length=50))
# override email mixin for unique index
email = Email(unique=True)
first_name = Name()
last_name = Name()
def get_id(self):
return self.user_id
This is the correct query:
Subscription.query.filter(Subscription.watch_id == id).join(User)

Resources