How to get a field from from last row of an change log table in SQLAlchemy - python-3.x

I have the follow example. When calling Order.next I would expect to return a single order in the with the status NEW. However I am getting Orders in other OrderStatus.
If it matters this is using SQLite backend.
class OrderStatus(enum.Enum):
NEW = 0
MAKING = 1
MADE = 2
COLLECTED = 3
class Order(Base):
__tablename__ = 'orders'
id = Column(Integer, primary_key=True)
updates = relationship("OrderUpdate", order_by = "OrderUpdate.position", collection_class=ordering_list('position'))
#hybrid_method
def next(self):
order = Order.query \
.filter(Order.status in [OrderStatus.NEW]) \
.order_by(Order.id) \
.first()
return order
#hybrid_property
def status(self):
update = OrderUpdate \
.query.filter_by(order_id = self.id) \
.order_by(OrderUpdate.id.desc()) \
.first()
if update is None:
return None
return update.status
class OrderUpdate(Base):
__tablename__ = 'order_updates'
id = Column(Integer, primary_key=True)
position = Column(Integer)
status = Column('status', Enum(OrderStatus))
comment = Column(String)
order_id = Column(Integer, ForeignKey("orders.id"))
order = relationship("Order", back_populates="updates")

I didn't check, but I think the problem with line:
.filter(Order.status in [OrderStatus.NEW])
If you need to use in condition you need to use sqlalchemy in_(). So in your case it should be like this: Order.status.in_([1, 2]). Also you can do it without in_():
Order.query.filter_by(Order.status=0).order_by(Order.id).first()
Note! About Enum. This is not about Flask or Flask-SqlAlchemy. Just for you information.
# Python 3.6.1
class OrderStatus(enum.Enum):
NEW = 0
0 in [OrderStatus.NEW] # False
0 in [OrderStatus.NEW.value] # True
val = OrderStatus(0)
val in [OrderStatus.NEW] # True
Hope this helps.

Related

Index a relationship in SQLAlchemy and Almbic

I have two tables like bellow:
Table 1:
class Table1(Base):
__tablename__ = "table1"
id = Column(UUID, primary_key=True)
created_at = Column(DateTime(True), nullable=False, server_default=func.now())
status_id = Column(ForeignKey("lookups.id"), index=True)
status = relationship(
"Lookup",
primaryjoin="Tables.status_id == Lookup.id",
backref="table1_statuses",
)
Table 2:
class Lookup(Base):
__tablename__ = "lookups"
id = Column(UUID, primary_key=True)
type = Column(String(64))
value = Column(String(256))
maximum = Column(Float(53), server_default="0")
sys_period = Column(TSTZRANGE, nullable=False, index=True)
HOW CAN I do a full text search on Table1 and find the status value. My query will look like this:
model = Table1
# search_term is `value` in the Lookup table
result = (session.query(model)
.filter(model.__ts_vector__.match(search_term)).all)

SQLalchemy query not updating relationship columns in put api

I have a model.py with a class Histogram and a table histogram_axes -
histograms_axes = Table(
"histograms_axes",
_ORMBase.metadata,
Column("histogram_id", ForeignKey("histograms.id"), primary_key=True),
Column("axis_id", ForeignKey("axes.id"), primary_key=True),
)
class Histogram(Base):
block = Column(Integer)
num = Column(Integer)
id = Column(Integer, ForeignKey("s.id"))
axes = relationship(
"Axis", secondary=histograms_axes, backref="histograms", order_by="asc(Axis.name)"
)
I have written a put API in FastAPI to update the rows based on id, but it only updates the columns not the relationship in Histogram class.
#router.put("/histograms")
def put_histograms(response: HistogramSchema, id: int, axes: List[int], db: Session = Depends(get_db)):
histogram = db.query(Histogram).filter(Histogram.id == id)
if not histogram.first():
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Histogram id {id} is not found!")
histogram.axes = db.query(Axis).filter(Axis.id.in_(axes)).all()
response = response.dict(exclude_unset=True)
histogram.update(response)
db.commit()
return histogram.one_or_none()
How can I update the relationship table as well.

How to bulk create or update in Django

I have to process an item report CSV file every 1 hour. The CSV contains 150k+ records for 1 account and there are multiple accounts in my system. I was working previously on rails and there was active record gem to handle this use case very efficiently. I am looking for an alternate to this gem in Django or any built in method that will be helpful to import such large data in bulk.
So far I have tried this code.
class ItemReportService:
def call(self, file_url):
with open(file_url, 'r') as file:
reader = csv.DictReader(file)
products = []
for row in reader:
product = self.process_product(row)
products.append(product)
self.update_products(products)
def process_product(self, row):
print(f'Processing sku: {row["SKU"]}')
product = Product.objects.filter(
sku=row['SKU']).first() or Product(sku=row['SKU'])
product.listing_title = row['Product Name']
product.listed_price = row['Price']
product.buy_box_price = row['Buy Box Item Price'] + \
row['Buy Box Shipping Price']
product.status = row['Lifecycle Status']
return product
def update_products(self, products):
Product.objects.bulk_update(
products,
[
'listing_title',
'listed_price',
'buy_box_price',
'Lifecycle Status'
]
)
It is raising this exception because when there is a new product it doesn't have primary key assigned to it
ValueError: All bulk_update() objects must have a primary key set.
Django 4.1 has new parameters for bulk_create(update_conflicts=bool and update_fields=[])
If your model has a field UNIQUE usually Django would ignore it when creating new data. But if you set the update_conflicts parameter to True, the fields inside update_fields will be updated.
You are not saving the product in the database before applying bulk_update.
I have checked your code for this purpose, you can use bulk_insert with an additional parameter
Model.objects.bulk_create(self.data, ignore_conflicts=True)
or
columns = ['column1', 'column2']
obj = Model.objects.filter(column1="sku").first()
if not obj:
obj = Model.objects.create(column1="sku")
obj.column1 = row["column1"] or obj.column1
obj.column2 = row["column2"] or obj.column2
items_to_be_inserted.append(obj)
In the end, you can do bulk update like
Model.objects.bulk_update(items_to_be_inserted, columns)
This will solve your problem.
I made this class function which can be used on any Django model in a project.
from django.db import models
class BaseModel(models.Model):
#classmethod
def bulk_create_or_update(
cls, uniques: list[str],
defaults: list[str],
data: list[dict]
):
# Get existing object list
data_dict, select = {}, None
for entry in data:
sub_entry, key = {}, ''
for uniq in uniques:
sub_entry[uniq] = entry[uniq]
key += str(entry[uniq])
data_dict[key] = entry
if not select:
select = models.Q(**sub_entry)
continue
select |= models.Q(**sub_entry)
records = cls.objects.filter(select).values('pk', *uniques)
existing = {}
for rec in records:
key = ''
for uniq in uniques:
key += str(rec[uniq])
existing[key] = rec
# Split new objects from existing ones
to_create, to_update = [], []
for key, entry in data_dict.items():
obj = cls(**entry)
if key not in existing:
to_create.append(obj)
continue
obj.pk = existing[key]['pk']
to_update.append(obj)
cls.objects.bulk_create(to_create, batch_size=1000)
cls.objects.bulk_update(to_create, defaults, batch_size=1000)
Let take an usage example
class Product(BaseModel)
price = models.IntegerField()
name = models.CharField(max_length=128, unique=True)
status = models.CharField(max_length=128)
if __name__ == '__main__':
data = [
{'price': 50, 'name': 'p1', 'status': 'New'},
{'price': 33, 'name': 'p2', 'status': 'Old'}
]
Product.bulk_create_or_update(uniques=['name'], defaults=['price', 'status'], data=data)
Any improvement suggestion of the code is welcome.

Save method of Django model does not update fields of existing record even if force update

I am trying to update the record that already exists in the database and therefore I use this code
if 'supplierId' not in req.keys():
return JsonResponse({'code': 0, 'msg': "supplier was not selected", 'result': ''}, safe=False)
assigneeId = User.objects.get(pk=req.get('assigneeId', 1))
responsibleId = User.objects.get(pk=req.get('responsibleId', 1))
redistributionMethod = req.get('redistributionMethod', 0)
amount = req.get('allCost', 0)
procurement_doc = ProcurementDocJournal.objects.get(id=pk)
print(procurement_doc)
procurement_doc.docType = req['docType']
procurement_doc.status = req['status']
procurement_doc.companyId = Company.objects.get(pk=req['companyId'])
procurement_doc.datetime = req['datetime']
procurement_doc.supplierId = Partner.objects.get(pk=req['supplierId'])
procurement_doc.assigneeId = assigneeId
procurement_doc.warehouseId = Warehouse.objects.get(pk=req['warehouseId'])
procurement_doc.responsibleId = responsibleId
procurement_doc.redistributionMethod = redistributionMethod
procurement_doc.amount = amount
procurement_doc.comment = req['comment']
procurement_doc.save(force_update=True, update_fields=['comment', 'amount', 'redistributionMethod',
'responsibleId', 'warehouseId',
'supplierId', 'datetime',
'companyId', 'assigneeId', 'status', 'docType'])
where req contains a request
something like this
{
'docType': 3,
'status': 1,
'companyId': '2',
'warehouseId': '3',
'assigneeId': '5',
'supplierId': '12671',
'responsibleId': '5',
'datetime': '2020-04-01 08:01:00',
'comment': ''
}
As you can see there is a print which assures me that I selected the correct row
when I noticed that these records are not updated I searched for causes and found
this question where the guy who asked says The message field was missing from the model definition
in my case, none of these are missing from the model's description
class ProcurementDocJournal(models.Model):
id = models.IntegerField(primary_key=True, null=False)
docNumber = models.IntegerField()
status = models.IntegerField()
docType = models.IntegerField()
depended = models.IntegerField()
companyId = models.ForeignKey(Company, on_delete=models.CASCADE,
db_column='companyId')
created_at = models.DateTimeField(auto_now_add=True)
datetime = models.DateTimeField()
currencyId = models.ForeignKey(Currency, db_column='currencyId', on_delete=models.CASCADE)
currencyRate = models.FloatField()
redistributionMethod = models.IntegerField()
assigneeId = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ProcurementDocJournal',
db_column='assigneeId')
warehouseId = models.ForeignKey(Warehouse, on_delete=models.CASCADE,
db_column='warehouseId')
responsibleId = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ProcurementDoc',
db_column='responsibleId')
supplierId = models.ForeignKey(Partner, on_delete=models.CASCADE,
db_column='supplierId')
amount = models.FloatField()
comment = models.TextField()
class Meta:
db_table = 'procurementDocJournal'
get_latest_by = 'id'
Edit
I have an action that contains
procurement_doc_journal_item = ProcurementDocJournal.objects.get(id=pk)
currencyId = req['currency']
currency = Currency.objects.get(id=currencyId)
currencyRate = CurrencyRate(date, currency.name)
procurement_doc_journal_item.currencyId = currency
procurement_doc_journal_item.currencyRate = currencyRate['rate']
procurement_doc_journal_item.save()
and works like a charm
there is no any error that logs say
You don't specify if there's any error in your logs. I'd kind of expect to see something since if it's not saving, it must be bombing out before that as no fields are mandatory.
I am, however, not sure you're setting your FKs correctly here:
procurement_doc.companyId = Company(req['companyId'])
Should be
procurement_doc.companyId = Company.objects.get(pk=req['companyId'])
and that assumes companyId is set to a value that exists of course ... Whether this is the source of your issue or not, I'm not sure. I don't see any of your FKs are mandatory.
Edit: I've just tested this on one of my models with a simple id key:
>>> Contact(1)
Contact()
>>> c = Contact(1)
>>> c.name
''
>>> c = Contact.objects.get(pk=1)
>>> c.name
'Mike'
It really doesn't seem to me as though it works ...
Sidenote: You can simplify code like this:
if 'allCost' not in req.keys():
amount = 0
else:
amount = req['allCost']
By doing:
amount = req.get('allCost', 0)

SQLAlchemy Order joined table by field in another joined table

My project requires that Orders are split into their individual lines which can be displayed in their own views I want these views to order the lines by eta which is a value in the Order table.
I have 3 tables with a 1>1 join on tables 1&2 and a many>many join on tables 2 and 3 defined by table 4 as follows:
class Order(db.Model):
id = db.Column(db.Integer, primary_key=True)
eta = db.Column(db.DateTime())
order_lines = db.relationship('Line', backref='order', order_by=lambda: Line.id)
def __repr__(self):
return '<Order No. {}>'.format(self.increment_id)
class Line(db.Model):
id = db.Column(db.Integer, primary_key=True)
line_name = db.Column(db.String())
order_id = db.Column(db.Integer, db.ForeignKey('order.id'))
product_id = db.Column(db.String, db.ForeignKey('product.product_id'))
def __repr__(self):
return '<Line SKU: {}>'.format(self.line_sku)
class Line_view(db.Model):
id = db.Column(db.Integer, primary_key=True)
view_name = db.Column(db.String())
view_lines = relationship('Line',
secondary='line_view_join',
backref='views',
lazy='dynamic',
order_by= ***???*** ) #ordery by eta on Order table
def __repr__(self):
return '<View: {}>'.format(self.view_name)
class Line_view_join(db.Model):
__tablename__ = 'line_view_join'
id = db.Column(db.Integer(), primary_key=True)
line_id = db.Column(db.Integer(), db.ForeignKey('line.id', ondelete='CASCADE'))
view_id = db.Column(db.Integer(), db.ForeignKey('line_view.id', ondelete='CASCADE'))
I am trying to work out how to query table 3, Line_View and have the joined Lines ordered by the eta of Order table.
Such that when querying:
chosen_view = Line_view.query.filter_by(id = 1).one()
chosen_view.view_lines are ordered by Order.eta
I have Tried
class Line_view(db.Model):
id = db.Column(db.Integer, primary_key=True)
view_name = db.Column(db.String())
view_lines = relationship('Line',
secondary='line_view_join',
backref='views',
lazy='dynamic',
**order_by=lambda: asc(Line.order.eta))**
def __repr__(self):
return '<View: {}>'.format(self.view_name)
But this results in the error:
AttributeError: Neither 'InstrumentedAttribute' object nor 'Comparator' object associated with Line.order has an attribute 'eta'
Do you need to store the Line_views in the database? If not, you can query the Lines sorted by the eta attribute of the related order. Below, I create two orders with one line each, and then query the lines sorted by the eta attribute of their order:
eta = datetime(2019,10,10)
o = Order(eta = eta)
l = Line(order=o, line_name="sample")
db.session.add(o)
db.session.add(l)
eta = datetime(2019,11,11)
o1 = Order(eta = eta)
l1 = Line(order=o1, line_name="sample1")
db.session.add(o1)
db.session.add(l1)
db.session.commit()
lines = Line.query.join(Order).order_by(Order.eta)

Resources