How do I express relationship which depends on len of collection child in joined entity?
In below example, parent entity is AlgoOrder. Child entity is Order. And PrivateTrade is child entity of Order.
AlgoOrder --> Order --> PrivateTrade
The problem I am having is with "orders_pending_private_trade_update".
class AlgoOrder(DbModel):
__tablename__ = "algo_order"
id = sa.Column(sa.Integer, primary_key=True)
... stuff ...
# https://docs.sqlalchemy.org/en/14/orm/loading_relationships.html
open_orders = orm.relation(Order, primaryjoin=and_(Order.algo_order_id == id, Order.status == 'OPEN'), lazy='select')
orders_pending_private_trade_update = orm.relation(Order, primaryjoin=and_(Order.algo_order_id == id, , Order.status == 'CLOSED', len(Order.private_trades)==0), lazy='select')
#property
def pending_orders(self):
return self.open_orders + self.orders_pending_private_trade_update
class Order(DbModel):
__tablename__ = "order_hist"
algo_order_id = sa.Column(sa.Integer, sa.ForeignKey("algo_order.id"))
... stiff ...
private_trades = orm.relation(PrivateTrade, primaryjoin=and_(PrivateTrade.order_id == order_id))
class PrivateTrade(DbModel):
__tablename__ = "private_trade"
id = sa.Column(sa.Integer, primary_key=True)
order_id = sa.Column(sa.String, sa.ForeignKey("order_hist.order_id"))
In particular, the error at "orders_pending_private_trade_update" was with "len" on Order.private_trades:
Exception has occurred: TypeError (note: full exception trace is shown but execution is paused at: _run_module_as_main) object of type 'InstrumentedAttribute' has no len()
So, I tried:
from sqlalchemy.sql.expression import func
orders_pending_private_trade_update = orm.relation(Order, primaryjoin=and_(Order.algo_order_id == id, Order.status == 'CLOSED', func.count(Order.private_trades)==0), lazy='select', viewonly=True)
But then error was "foreign key columns are present in neither the parent nor the child's mapped tables":
Can't determine relationship direction for relationship 'AlgoOrder.orders_pending_private_trade_update' - foreign key columns are present in neither the parent nor the child's mapped tables <class 'sqlalchemy.exc.ArgumentError'> Can't determine relationship direction for relationship 'AlgoOrder.orders_pending_private_trade_update' - foreign key columns are present in neither the parent nor the child's mapped tables
I checked my tables, I do have them:
op.create_table(
'algo_order',
sa.Column('id', sa.Integer(), primary_key=True),
...
op.create_table(
'order_hist',
sa.Column('id', sa.Integer(), primary_key=True),
sa.Column('algo_order_id', sa.Integer, sa.ForeignKey("algo_order.id")),
...
op.create_table(
'private_trade',
sa.Column('id', sa.Integer(), primary_key=True),
sa.Column('order_id', sa.String(), sa.ForeignKey("order_hist.order_id"))
...
Thanks in advance.
I think I found it, but syntax pretty ugly: I used closed_order.session to do a new Query
import sqlalchemy as sa
import sqlalchemy.orm as orm
from sqlalchemy.sql.expression import func
import sqlalchemy.dialects.postgresql as psql
from sqlalchemy.ext.mutable import MutableDict
from sqlalchemy.sql.expression import and_
class AlgoOrder(DbModel):
__tablename__ = "algo_order"
id = sa.Column(sa.Integer, primary_key=True)
... other stuff ...
open_orders = orm.relation(Order, primaryjoin=and_(Order.algo_order_id == id, Order.status == 'OPEN'), lazy='select')
closed_orders = orm.relation(Order, primaryjoin=and_(Order.algo_order_id == id, Order.status == 'CLOSED'), lazy='dynamic', viewonly=True)
#property
def orders_pending_private_trade_update(self):
order_ids_with_no_private_trades = [ order.id for order in list(self.closed_orders.session.query(Order.id, func.count(PrivateTrade.id).label('count_private_trades')).join(PrivateTrade, isouter=True).group_by(Order.id).having(func.count(PrivateTrade.id) == 0).all())]
orders_with_no_private_trades = self.closed_orders.session.query(Order).filter(Order.id.in_(order_ids_with_no_private_trades)).order_by(Order.id.desc()).limit(1000).all()
return orders_with_no_private_trades
#property
def pending_orders(self):
return list(self.open_orders) + list(self.orders_pending_private_trade_update)
I also don't like "first(100)" as an attempt to limit number of rows fetched. And how/when you dispose of the list to prevent memory leak? I think above approach is bad. Should use generator instead of returning list.
Essentially raw sql what I am looking for is a generator which returns below:
select
order_id,
cnt
from (
select
order_hist.id,
order_hist.order_id,
count(private_trade.id) cnt
from order_hist
left join private_trade on private_trade.order_id = order_hist.order_id
where order_hist.status in ('CLOSED', 'CANCELLED')
group by order_hist.id, order_hist.order_id
) src
where
cnt=0
Any better way to do this? I think my solution shows the sqlalchemy syntax but it's computationally inefficient.
Here's solution using generator instead to avoid MemoryError:
def order_hist_missing_private_trade_get(engine):
order_hist_missing_private_trade_sql = '''
select
order_id,
cnt
from (
select
order_hist.id,
order_hist.order_id,
count(private_trade.id) cnt
from order_hist
left join private_trade on private_trade.order_id = order_hist.order_id
where order_hist.status in ('CLOSED', 'CANCELLED')
group by order_hist.id, order_hist.order_id
) src
where
cnt=0
'''
with engine.connect() as conn:
# https://stackoverflow.com/questions/7389759/memory-efficient-built-in-sqlalchemy-iterator-generator
conn.execution_options(stream_results=True)
rs = conn.execute(order_hist_missing_private_trade_sql)
while True:
batch = rs.fetchmany(10000)
for row in batch:
order_id = row['order_id']
yield order_id
Usage:
from sqlalchemy import create_engine
connstr : str = "postgresql://postgres:your_secret#localhost/postgres"
engine = create_engine(connstr)
generator = order_hist_missing_private_trade_get(engine)
while True:
order_id = next(generator)
print(f"order_id: {order_id}")
I'm trying to get all data from a table by a model, i can get all data from the table this works ok, but i need get data from other table mapped by a foreign key to another table, this means i have a field called ID_USUARIO in Parcela Model and i want to get all data that have my actual query with the value from the other table called FULLNAME in Usuario Model.
How can i do that?
Usuario Model
class Usuario(Model):
ID_USER = models.AutoField(primary_key=True)
FULLNAME = models.CharField(max_length=500)
EMAIL = models.EmailField()
PASSWORD = models.CharField(max_length=450)
Parcela Model
class Parcela(Model):
"""docstring for Parcela."""
ID = models.AutoField(primary_key=True)
ID_USUARIO = models.ForeignKey(Usuario, on_delete=models.CASCADE)
ID_PARCELA = models.BigIntegerField()
A_DEAD_LEAVES = models.FloatField()
A_SOIL_DEPTH = models.FloatField()
A_LIGHT_FUEL = models.FloatField()
A_DUFF = models.FloatField()
A_MID_HEA_FUEL = models.FloatField()
A_SOIL_ROCK_COLOR = models.FloatField()
B_DOMINANT_VEG_TYPE = models.CharField(max_length=200)
B_FCOV = models.FloatField()
B_FOLIAGE_ALTERED = models.FloatField()
B_FREQ_LIVING = models.FloatField()
B_NEW_SPROUTS = models.FloatField()
C_DOMINANT_VEG_TYPE = models.CharField(max_length=200)
C_FCOV = models.FloatField()
C_FOLIAGE_ALTERED = models.FloatField()
C_FREQ_LIVING = models.FloatField()
C_LAI_CHANGE = models.FloatField()
D_DOMINANT_VEG_TYPE = models.CharField(max_length=200)
...
This is the query i'm doing
export = Parcela.objects.select_related('ID_USUARIO')
Result of query
ID,ID_USUARIO,ID_PARCELA,A_DEAD_LEAVES,A_SOIL_DEPTH,A_LIGHT_FUEL,A_DUFF,A_MID_HEA_FUEL,A_SOIL_ROCK_COLOR,B_DOMINANT_VEG_TYPE,B_FCOV,B_FOLIAGE_ALTERED,B_FREQ_LIVING,B_NEW_SPROUTS,C_DOMINANT_VEG_TYPE,C_FCOV,C_FOLIAGE_ALTERED,C_FREQ_LIVING,C_LAI_CHANGE,D_DOMINANT_VEG_TYPE,D_FCOV,D_GREEN_UNALTERAD,D_BLACK_BROWN,D_FREQ_LIVING,D_LAI_CHANGE,D_CHAR_HEIGHT,E_DOMINANT_VEG_TYPE,E_FCOV,E_GREEN_UNALTERAD,E_BLACK_BROWN,E_FREQ_LIVING,E_LAI_CHANGE,E_CHAR_HEIGHT,COORDINATE_X,COORDINATE_Y,DATE_CAPTURED,DATE_SAVED,PHOTO_1,PHOTO_2,PHOTO_3,PHOTO_4,PHOTO_5,SEVERITY
2,1,1,20.0,30.0,10.0,20.0,15.0,6.0,Cactus,12.0,11.0,13.0,3.0,Pasto,2.0,6.0,2.0,3.1,Arbol,1.2,14.0,8.0,4.0,0.5,2.0,Hongos,3.0,1.9,11.0,10.2,20.1,18.4,624811.32,665561.23,2019-08-27 16:56:20,2019-08-27 16:56:20,,,,,,Moderada
3,1,1,20.0,30.0,10.0,20.0,15.0,6.0,Cactus,12.0,11.0,13.0,3.0,Pasto,2.0,6.0,2.0,3.1,Arbol,1.2,14.0,8.0,4.0,0.5,2.0,Hongos,3.0,1.9,11.0,10.2,20.1,18.4,624811.32,665561.23,2019-08-27 16:56:20,2019-08-27 16:56:20,,,,,,Moderada
4,1,1,20.0,30.0,10.0,20.0,15.0,6.0,Cactus,12.0,11.0,13.0,3.0,Pasto,2.0,6.0,2.0,3.1,Arbol,1.2,14.0,8.0,4.0,0.5,2.0,Hongos,3.0,1.9,11.0,10.2,20.1,18.4,624811.32,665561.23,2019-08-27 22:46:52,2019-08-25 15:46:12,,,,,,Moderada
5,1,2,100.0,2.0,2.1,1.2,1.0,1.1,Margaritas,40.0,0.3,1.25,2.65,Pinos,2.0,6.0,2.0,3.1,Guayacan,1.2,14.0,8.0,4.0,0.5,2.0,Roble,3.0,1.9,11.0,10.2,20.1,18.4,624811.32,665561.23,2019-08-24 22:46:52,2019-08-28 15:46:12,,,,,,Alta
6,2,3,100.0,2.0,2.1,1.2,1.0,1.1,Cedro,40.0,0.3,1.25,2.65,Rosas,2.0,6.0,2.0,3.1,Savila,1.2,14.0,8.0,4.0,0.5,2.0,Eucalipto,3.0,1.9,11.0,10.2,20.1,18.4,624811.32,665561.23,2019-08-24 22:46:52,2019-08-28 15:46:12,,,,,,No Quemada
Yes, you are doing it correctly. One must use select_related to do join in django. you can verify the generated query
result = Parcela.objects.select_related('ID_USUARIO')
print(str(result.query))
# You can access ID_USARIO fields like this
print(result.first().ID_USARIO.FULLNAME)
To get rows in parcela table with a name, you can do reverse lookup. Then the query will be
result = Usuario.objects.get(FULLNAME='<name_to_filter>')
parcela_rows = result.parcela_set.all()
I'm new in SQLalchemy I need to calculate multiple of some price in my one of the table. This is my tables:
class Order(DeclarativeBase):
__tablename__ = 'order'
id = Field(Integer, primary_key=True)
products = relationship("OrderProduct", back_populates="order", lazy='dynamic')
and
class OrderProduct(DeclarativeBase):
__tablename__ = 'order_products'
id = Field(Integer, primary_key=True)
order_id = Column(Integer, ForeignKey('order.id'), nullable=False)
order = relationship("Order", back_populates="products", protected=True)
product_id = Column(Integer, ForeignKey('product.id'), nullable=False)
product = relationship("Product", back_populates="order_product")
quantity = Field(Integer, nullable=False)
and
class Product(DeclarativeBase):
__tablename__ = 'product'
id = Field(Integer, primary_key=True)
price = Field(Integer)
order_product = relationship("OrderProduct", back_populates="product", protected=True)
I want to multiple price with this situation OrderProduct.quantity * Product.price and products in Order table is an array of Products
I write SQL query like this and it works:
SELECT SUM(price*quantity) FROM product
JOIN order_products ON product.id = order_products.product_id
JOIN order ON order_products.order_id = order.id;
I tried to make it in ORM like this but it takes me Product and I can calculate only price without multiple in quantity:
result = 0
for product in Product.query\
.join(OrderProduct).filter(Product.id == OrderProduct.product_id)\
.join(Order).filter(OrderProduct.order_id == self.id):
result = product.price + result
return result
I make this as #hybrid_property and its work well.
I use a framework that name is restfulpy. it has ORM for sqlalchemy, session in this framework is scoped_session, but it gives me SQL query in debuging mode instead of executing the query like this :
sum_price = {Query}SELECT sum(product.price * order_products.quantity) AS sum_1
FROM product JOIN order_products ON product.id = order_products.product_id JOIN "order" ON "order".id = order_products.order_id
WHERE product.id = order_products.product_id AND order_products.order_id = :order_id_1
Well, Can anyone help me out this problem?
with regards
I just realized that SQLalchemy is a great ORM! When you have a relation between two, three or ... tables, SQLalchemy make a join between them and you just query it!
I made it hard for me and my friends :) SQLalchemy is powerful than what I'm thinking!
This is the right answer:
#hybrid_property
def total_price(self):
return DBSession.query(
func.sum(Product.price * OrderProduct.quantity))\
.filter(OrderProduct.product_id == Product.id) \
.filter(OrderProduct.order_id == self.id).scalar()
DBSession is the same with session
from sqlalchemy import func
query = session.query(func.sum(Product.price*OrderProduct.quantity))
.join(OrderProduct).filter(Product.id == OrderProduct.product_id)
.join(Order).filter(OrderProduct.order_id == self.id)