Here is my data structure:
{
house_id : 1,
houseType : "condo" ,
family: [
{'name': "pete", "gender": "male", etc},
{'name': "jon", "gender": "male", etc}
]
}
How do i delete 'pete' from the EmbeddedDocumentListField?
db = MongoEngine()
db.init_app(app)
class Family(db.EmbeddedDocument):
name = db.StringField()
gender = db.StringField()
class House(db.Document):
house_id = db.IntField(required=True, unique=True)
housingType = db.StringField(required=True)
family = db.EmbeddedDocumentListField(Family)
#app.route('/api/del_member/<h_id>/<fam_name>', methods=['DELETE'])
def del_member(h_id, fam_name):
try:
h = House.objects.get(house_id=h_id)
t = h.family.get(name=fam_name)
return make_response(jsonify(t), 200)
except Exception:
return make_response(f'Something went wrong trying to delete '
f'Member: {fam_name} in House ID: {h_id}', 500)
im able to select the item im interested in but i cant delete it. I have tried .delete() but an error saying
t = h.family.get(name=fam_name).delete()
AttributeError: 'Family' object has no attribute 'delete'
pops up.
I found a solution. Might not be the best but it works. Hopefully someone can improve on it.
#app.route('/api/del_member/<h_id>/<fam_name>', methods=['DELETE'])
def del_member(h_id, fam_name):
h = House.objects.get(house_id=h_id).to_json()
h = json.loads(h)
family = h['family']
new_family = family.copy()
for i in range(len(family)):
name = family[i].get('name')
if name == fam_name:
del new_family[i]
break
House.objects.get(house_id=h_id).update(family=new_family)
return make_response('', 200)
Related
I have to process an item report CSV file every 1 hour. The CSV contains 150k+ records for 1 account and there are multiple accounts in my system. I was working previously on rails and there was active record gem to handle this use case very efficiently. I am looking for an alternate to this gem in Django or any built in method that will be helpful to import such large data in bulk.
So far I have tried this code.
class ItemReportService:
def call(self, file_url):
with open(file_url, 'r') as file:
reader = csv.DictReader(file)
products = []
for row in reader:
product = self.process_product(row)
products.append(product)
self.update_products(products)
def process_product(self, row):
print(f'Processing sku: {row["SKU"]}')
product = Product.objects.filter(
sku=row['SKU']).first() or Product(sku=row['SKU'])
product.listing_title = row['Product Name']
product.listed_price = row['Price']
product.buy_box_price = row['Buy Box Item Price'] + \
row['Buy Box Shipping Price']
product.status = row['Lifecycle Status']
return product
def update_products(self, products):
Product.objects.bulk_update(
products,
[
'listing_title',
'listed_price',
'buy_box_price',
'Lifecycle Status'
]
)
It is raising this exception because when there is a new product it doesn't have primary key assigned to it
ValueError: All bulk_update() objects must have a primary key set.
Django 4.1 has new parameters for bulk_create(update_conflicts=bool and update_fields=[])
If your model has a field UNIQUE usually Django would ignore it when creating new data. But if you set the update_conflicts parameter to True, the fields inside update_fields will be updated.
You are not saving the product in the database before applying bulk_update.
I have checked your code for this purpose, you can use bulk_insert with an additional parameter
Model.objects.bulk_create(self.data, ignore_conflicts=True)
or
columns = ['column1', 'column2']
obj = Model.objects.filter(column1="sku").first()
if not obj:
obj = Model.objects.create(column1="sku")
obj.column1 = row["column1"] or obj.column1
obj.column2 = row["column2"] or obj.column2
items_to_be_inserted.append(obj)
In the end, you can do bulk update like
Model.objects.bulk_update(items_to_be_inserted, columns)
This will solve your problem.
I made this class function which can be used on any Django model in a project.
from django.db import models
class BaseModel(models.Model):
#classmethod
def bulk_create_or_update(
cls, uniques: list[str],
defaults: list[str],
data: list[dict]
):
# Get existing object list
data_dict, select = {}, None
for entry in data:
sub_entry, key = {}, ''
for uniq in uniques:
sub_entry[uniq] = entry[uniq]
key += str(entry[uniq])
data_dict[key] = entry
if not select:
select = models.Q(**sub_entry)
continue
select |= models.Q(**sub_entry)
records = cls.objects.filter(select).values('pk', *uniques)
existing = {}
for rec in records:
key = ''
for uniq in uniques:
key += str(rec[uniq])
existing[key] = rec
# Split new objects from existing ones
to_create, to_update = [], []
for key, entry in data_dict.items():
obj = cls(**entry)
if key not in existing:
to_create.append(obj)
continue
obj.pk = existing[key]['pk']
to_update.append(obj)
cls.objects.bulk_create(to_create, batch_size=1000)
cls.objects.bulk_update(to_create, defaults, batch_size=1000)
Let take an usage example
class Product(BaseModel)
price = models.IntegerField()
name = models.CharField(max_length=128, unique=True)
status = models.CharField(max_length=128)
if __name__ == '__main__':
data = [
{'price': 50, 'name': 'p1', 'status': 'New'},
{'price': 33, 'name': 'p2', 'status': 'Old'}
]
Product.bulk_create_or_update(uniques=['name'], defaults=['price', 'status'], data=data)
Any improvement suggestion of the code is welcome.
I am trying to update the record that already exists in the database and therefore I use this code
if 'supplierId' not in req.keys():
return JsonResponse({'code': 0, 'msg': "supplier was not selected", 'result': ''}, safe=False)
assigneeId = User.objects.get(pk=req.get('assigneeId', 1))
responsibleId = User.objects.get(pk=req.get('responsibleId', 1))
redistributionMethod = req.get('redistributionMethod', 0)
amount = req.get('allCost', 0)
procurement_doc = ProcurementDocJournal.objects.get(id=pk)
print(procurement_doc)
procurement_doc.docType = req['docType']
procurement_doc.status = req['status']
procurement_doc.companyId = Company.objects.get(pk=req['companyId'])
procurement_doc.datetime = req['datetime']
procurement_doc.supplierId = Partner.objects.get(pk=req['supplierId'])
procurement_doc.assigneeId = assigneeId
procurement_doc.warehouseId = Warehouse.objects.get(pk=req['warehouseId'])
procurement_doc.responsibleId = responsibleId
procurement_doc.redistributionMethod = redistributionMethod
procurement_doc.amount = amount
procurement_doc.comment = req['comment']
procurement_doc.save(force_update=True, update_fields=['comment', 'amount', 'redistributionMethod',
'responsibleId', 'warehouseId',
'supplierId', 'datetime',
'companyId', 'assigneeId', 'status', 'docType'])
where req contains a request
something like this
{
'docType': 3,
'status': 1,
'companyId': '2',
'warehouseId': '3',
'assigneeId': '5',
'supplierId': '12671',
'responsibleId': '5',
'datetime': '2020-04-01 08:01:00',
'comment': ''
}
As you can see there is a print which assures me that I selected the correct row
when I noticed that these records are not updated I searched for causes and found
this question where the guy who asked says The message field was missing from the model definition
in my case, none of these are missing from the model's description
class ProcurementDocJournal(models.Model):
id = models.IntegerField(primary_key=True, null=False)
docNumber = models.IntegerField()
status = models.IntegerField()
docType = models.IntegerField()
depended = models.IntegerField()
companyId = models.ForeignKey(Company, on_delete=models.CASCADE,
db_column='companyId')
created_at = models.DateTimeField(auto_now_add=True)
datetime = models.DateTimeField()
currencyId = models.ForeignKey(Currency, db_column='currencyId', on_delete=models.CASCADE)
currencyRate = models.FloatField()
redistributionMethod = models.IntegerField()
assigneeId = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ProcurementDocJournal',
db_column='assigneeId')
warehouseId = models.ForeignKey(Warehouse, on_delete=models.CASCADE,
db_column='warehouseId')
responsibleId = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ProcurementDoc',
db_column='responsibleId')
supplierId = models.ForeignKey(Partner, on_delete=models.CASCADE,
db_column='supplierId')
amount = models.FloatField()
comment = models.TextField()
class Meta:
db_table = 'procurementDocJournal'
get_latest_by = 'id'
Edit
I have an action that contains
procurement_doc_journal_item = ProcurementDocJournal.objects.get(id=pk)
currencyId = req['currency']
currency = Currency.objects.get(id=currencyId)
currencyRate = CurrencyRate(date, currency.name)
procurement_doc_journal_item.currencyId = currency
procurement_doc_journal_item.currencyRate = currencyRate['rate']
procurement_doc_journal_item.save()
and works like a charm
there is no any error that logs say
You don't specify if there's any error in your logs. I'd kind of expect to see something since if it's not saving, it must be bombing out before that as no fields are mandatory.
I am, however, not sure you're setting your FKs correctly here:
procurement_doc.companyId = Company(req['companyId'])
Should be
procurement_doc.companyId = Company.objects.get(pk=req['companyId'])
and that assumes companyId is set to a value that exists of course ... Whether this is the source of your issue or not, I'm not sure. I don't see any of your FKs are mandatory.
Edit: I've just tested this on one of my models with a simple id key:
>>> Contact(1)
Contact()
>>> c = Contact(1)
>>> c.name
''
>>> c = Contact.objects.get(pk=1)
>>> c.name
'Mike'
It really doesn't seem to me as though it works ...
Sidenote: You can simplify code like this:
if 'allCost' not in req.keys():
amount = 0
else:
amount = req['allCost']
By doing:
amount = req.get('allCost', 0)
I´m new to programming and I chose python (3.7) as the first working language. I have been working for 5 days on code that consumes an API and returns a nested JSON. My intention is to insert this data into a table in a PostgreSQL database. I can already insert data from other data that this API provides, but this particular JSON structure is giving me problems because my code stops working when it doesn't find a certain key inside the python object (when it doesn't exist. I need it to return as a null value on a table).
I'll set you up to make it clearer. In short, and with a piece of dummy data:
#Note that some keys are sometimes present and sometimes not.
myapidata = [
{
"MaxRpm": 2300,
"StartPosition": {
"Longitude": -12.3456,
"Latitude": -78.9456
},
"Engine": 10623,
"Fuel": 20.133
},
{
"MaxRpm": 0.0,
"StartPosition": {
"Longitude": -74.1258,
"Latitude": -96.3258
},
"EndPosition": {
"Longitude": -78.9456,
"Latitude": -85.2369
},
"Engine": 0,
"Fuel": 150.35
},
{
"MaxRpm": 800,
"StartPosition": {
"Longitude": 85.4125,
"Latitude": -45.62145
},
"EndPosition": {
"Longitude": 85.2145,
"Latitude": 74.6789
},
"Engine": 104,
"Fuel": 0.021,
"Pulse": 7
}
]
#Python Code:
import json
import psycopg2
api_json_list = json.loads(myapidata.content)
#Tried to add Null to keys not present (works with non Nested JSON):
allkeys = frozenset().union(*api_json_list)
for a in api_json_list:
for b in allkeys:
if b not in a:
a[b] = None
#Insert data on PostgreSQL:
conn = psycopg2.connect ("host = my_host dbname = my_db user = my_user password = my_pass")
cur = conn.cursor()
cur.execute("TRUNCATE TABLE mytable")
data_extract = []
def get_data():
for data in api_json_list:
dictionary = data
maxrpm = dictionary['MaxRpm']
start_lng = dictionary['StartPosition']['Longitude']
start_lat = dictionary['StartPosition']['Latitude']
end_lng = dictionary['EndPosition']['Longitude']
end_lat = dictionary['EndPosition']['Latitude']
engine = dictionary['Engine']
fuel = dictionary['Fuel']
pulse = dictionary['Pulse']
data_extract.append([maxrpm,start_lng,start_lat,end_lng,end_lat,engine,fuel,pulse])
get_data() #Get a TypeError
def post_gre():
for item in data_extract:
my_data = tuple(item)
cur.execute('INSERT INTO mytable VALUES (%s,%s,%s,%s,%s,%s,%s,%s)', my_data)
post_gre()
conn.commit()
conn.close()
The result I hope to achieve in my database is something like the table below:
Table with null items
Thank you for any help!
EDIT : Answer with the correct code
import json
import psycopg2
api_json_list = json.loads(myapidata.content)
#Insert data on PostgreSQL:
conn = psycopg2.connect ("host = my_host dbname = my_db user = my_user password = my_pass")
cur = conn.cursor()
cur.execute("TRUNCATE TABLE mytable")
data_extract = []
def get_data():
for data in api_json_list:
dictionary = data
maxrpm = dictionary.get('MaxRpm')
if 'StartPosition' in dictionary:
start_lng = dictionary['StartPosition'].get('Longitude')
start_lat = dictionary['StartPosition'].get('Latitude')
else:
start_lng = None
start_lat = None
if 'EndPosition' in dictionary:
end_lng = dictionary['EndPosition'].get('Longitude')
end_lat = dictionary['EndPosition'].get('Latitude')
else:
end_lng = None
end_lat = None
engine = dictionary.get('Engine')
fuel = dictionary.get('Fuel')
pulse = dictionary.get('Pulse')
data_extract.append([maxrpm,start_lng,start_lat,end_lng,end_lat,engine,fuel,pulse])
get_data()
def post_gre():
for item in data_extract:
my_data = tuple(item)
cur.execute('INSERT INTO mytable VALUES (%s,%s,%s,%s,%s,%s,%s,%s)', my_data)
post_gre()
conn.commit()
conn.close()
You could do something like this:
maxrpm = dictionary.get('MaxRpm')
if 'StartPosition' in dictionary:
start_lng = dictionary['StartPosition'].get('Longitude')
start_lat = dictionary['StartPosition'].get('Latitude')
else:
start_lng = None
start_lat = None
if 'EndPosition' in dictionary:
end_lng = dictionary['EndPosition'].get('Longitude')
end_lat = dictionary['EndPosition'].get('Latitude')
else:
end_lng = None
end_lat = None
engine = dictionary.get('Engine')
fuel = dictionary.get('Fuel')
pulse = dictionary.get('Pulse')
Using the get method on a dictionary will return the value if it exists or None if it doesn't.
I am trying to run a program that implements Object Oriented Programming to help a student register for a course. I keep running into an Attribute Error and can't seem to figure out where my code went wrong.
I initially thought it had something to do with the fact that I had not implemented the self parameter in all the method def statements. But after I fixed this, I still encountered the same error message.
ONE = 1
TWO = 2
THREE = 3
FOUR = 4
FIVE = 5
TRUE = True
FALSE = False
COURSES_INPUT = 'courses-sample.txt'
STUDENTS_INPUT= 'students-sample.txt'
import student
import course
def main():
process_students()
process_courses()
option = get_option()
while option != 5:
if option == 1:
course_num = input('please input course number of course you wish to add: ')
add_course = new_student.add_course(course_num)
while add_course == FALSE:
print('The course requested for add does not exist. Please Try Again.')
course_num = input('Please input course number of course you wish to add: ')
if new_course.space_available() == TRUE:
new_course.enroll_student()
else:
print('Class requested does not have any seats available.')
if option == 2:
course_num = input('please input course number of course you wish to drop: ')
drop_course = new_student.drop_course(course_num)
while drop_course == FALSE:
print('The enrolled course requested for drop does not exist. Please Try Again.')
course_num = input('Please input course number of course you wish to drop: ')
new_course.drop_student()
if option == 3:
print_student_info(student_dict)
if option == 4:
print_course_schedule(course_dict)
option = get_option()
write_updated('students-updated.txt',student_dict)
write_updated('courses-updated.txt',course_dict)
def print_menu():
print("1. Add course")
print("2. Drop course")
print("3. Print student's schedule")
print("4. Print course schedule")
print("5. Done")
print("")
def get_option():
print_menu()
choice = input("What would you like to do? ")
while choice not in range(1,6):
print_menu()
choice = input("Choice is invalid. What would you like to do? ")
return choice
def process_students():
student_dict = {}
student_info = open(STUDENTS_INPUT,"r")
for student in student_info:
info_list = student.split(":")
new_id = info_list[0]
first_name = info_list[1]
last_name = info_list[2]
course_list = info_list[3:]
new_student = student.Student(new_id, first_name, last_name, course_list)
print(new_student.line_for_file())
student_dict[new_eid] = new_student
student_info.close()
def process_courses():
course_dict = {}
course_info = open(COURSES_INPUT,"r")
for course in course_info:
info_list = course.split(";")
unique_num = info_list[0]
class_name = info_list[1]
prof = info_list[2]
seats = info_list[3]
capacity = info_list[4]
new_course = course.Course(unique_num, class_name, prof, seats, capacity)
course_dict[unique_num] = new_course
course_info.close()
def print_course_schedule(course_dict):
for value in course_dict:
print(value)
def print_student_info(student_dict):
for value in student_dict:
print(value)
def get_id():
eid = input("What is the UT EID? ")
while eid not in student_dict:
eid = input("Invalid UT EID. Please re-enter: ")
return eid
def get_unique():
unique = input("What is the course unique number? ")
while unique not in course_dict:
unique = input("Invalid unique number. Please re-enter: ")
return unique
def write_updated(filename,dictionary):
output_file = open(filename,'w')
for key in dictionary:
output_file.write(dictionary[key])
output_file.close()
main()
Error Message:
Traceback (most recent call last):
File "C:\Users\njung\Downloads\MIS 304 Final Project\untitled folder\Nguyen_Calvin_Jung_Nicholas-FP.py", line 132, in <module>
main()
File "C:\Users\njung\Downloads\MIS 304 Final Project\untitled folder\Nguyen_Calvin_Jung_Nicholas-FP.py", line 24, in main
process_students()
File "C:\Users\njung\Downloads\MIS 304 Final Project\untitled folder\Nguyen_Calvin_Jung_Nicholas-FP.py", line 83, in process_students
new_student = student.Student(new_id, first_name, last_name, course_list)
AttributeError: 'str' object has no attribute 'Student'
>>>
I also have the classes used stored in separate files (was required for the program) and have imported the modules containing these classes into main as you can see at the top.
Your error is AttributeError: 'str' object has no attribute 'Student'
Looking at the trace, it seems that it originates from this code:
student_info = open(STUDENTS_INPUT,"r")
for student in student_info:
info_list = student.split(":")
new_id = info_list[0]
first_name = info_list[1]
last_name = info_list[2]
course_list = info_list[3:]
new_student = student.Student(new_id, first_name, last_name, course_list)
Here you've opened a file. student_info is the file, and you iterate over the lines in the file. Each line student is a string.
You later call student.Student(new_id, first_name, last_name, course_list), but since student is just a string, it naturally does not contain a Student method.
I have 2 dict. One with local player data and one listing the players with subdictionaries:
class GameData:
def __init__(self):
self.player = {'id' : 1453339642,
'positionX' : 123,
'positionY' : 0
}
self.players = {1453339642:
{'name' : "Admin"}
}
gameData = GameData()
Then I print out just to check if everything works:
for x in gameData.player:
print (str(x),':',gameData.player[x])
print("\n\n")
for x in gameData.players:
print (str(x))
for y in gameData.players[x]:
print (' ',y,':',gameData.players[x][y])
print("\n\n")
This results in:
id : 1453339642
positionY : 0
positionX : 123
1453339642
name : Admin
When I now want to access the player's id in players for instance with
#print(str(type(gameData.player)))
#print(str(type(gameData.players)))
print(str(type(gameData.players[1453339642])))
I get KEYERROR as a result. Why?
If I put this in a file, it works:
class GameData:
def __init__(self):
self.player = {'id' : 1453339642,
'positionX' : 123,
'positionY' : 0
}
self.players = {1453339642:
{'name' : "Admin"}
}
gameData = GameData()
print(str(type(gameData.players[1453339642])))
Only indentation differs from your code. There must be something happening to gameData between instantiation and the final print.