group_by owners
My code:
def group_by_owners(files):
for key in files.keys():
if files[key] in files:
return files[key],key
files = {
'Input.txt': 'Randy',
'Code.py': 'Stan',
'Output.txt': 'Randy'
}
print(group_by_owners(files))
Expected output:
{'Randy': ['Input.txt','Output.txt'], 'Stan':['Code.py']}
def group_by_owners(files):
new_dic = dict() # Our new dict.
for key, val in files.items():
if val in new_dic: # If we have this name
new_dic[val].append(key) # Then just add the file
else:
new_dic[val] = [key] # Else create a new list with this file for this name
return new_dic
Output:
{'Randy': ['Input.txt', 'Output.txt'], 'Stan': ['Code.py']}
Related
I modified some scripts to delete all the "delete marker" placeholders from some bucket in s3 but I never found/developed something where I can delete the "delete marker" filtered by datetime.
My scope is to create a script to run after a "delete_object_job" failure, so I can run a new script where set the datatime of the failure and delete all the "delete-marker" just from that datetime.
Actually from this code I can delete all the "data marker" from some buckets but without a datetime filter:
#!/usr/bin/env python
import json
from datetime import datetime
from boto3 import Session
BUCKET_NAME = "germano"
prefix = ''
session = Session(region_name='eu-south-1', profile_name='default')
bucket = session.client('s3')
MAX_KEYS = 10000
def get_bucket_versions(version_id, key):
return bucket.list_object_versions(Bucket=BUCKET_NAME,
MaxKeys=MAX_KEYS,
Prefix=prefix,
VersionIdMarker=version_id,
KeyMarker=key)
class DateTimeEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, datetime):
return o.isoformat()
return json.JSONEncoder.default(self, o)
#ottengo gli attributes della classe creata DateTimeEncoder
#DateTimeEncoder = DateTimeEncoder()
#attributes_of_DateTimeEncoder = dir(DateTimeEncoder)
#print(attributes_of_DateTimeEncoder)
def objects_to_restore(versions):
return [
{
'VersionId': marker['VersionId'],
'Key': marker['Key']
} for marker in versions.get('DeleteMarkers') if marker['IsLatest']
]
def restore_s3_objects(version_markers, count):
markers_to_delete = objects_to_restore(version_markers)
print(f"Will restore {len(markers_to_delete)} items during request number: {count}")
if not markers_to_delete:
return 0
bucket.delete_objects(Bucket=BUCKET_NAME, Delete={'Objects': markers_to_delete})
return len(markers_to_delete)
obj_list = bucket.list_object_versions(Bucket=BUCKET_NAME,
MaxKeys=MAX_KEYS,
Prefix=prefix)
_next_version_id = obj_list.get('NextVersionIdMarker')
_next_key_marker = obj_list.get('NextKeyMarker')
counter = 1
total_restored = restore_s3_objects(obj_list, counter)
while _next_version_id and _next_key_marker:
counter += 1
another_list_of_versions = get_bucket_versions(_next_version_id, _next_key_marker)
_next_version_id = another_list_of_versions.get('NextVersionIdMarker')
_next_key_marker = another_list_of_versions.get('NextKeyMarker')
total_restored += restore_s3_objects(another_list_of_versions, counter)
print(f"Total Restored: {total_restored}")
i solved just modifing a bit the function "objects_to_restore":
def objects_to_restore(versions, last_modified_timestamp="2022-04-28 09:19:56.986445+00:00"):
print (versions.get('DeleteMarkers'))
#print (versions.get('Versions'))
return [
{
'VersionId': marker['VersionId'],
'Key': marker['Key'],
} for marker in versions.get('DeleteMarkers')
if marker['IsLatest']
if str(marker["LastModified"]) >= str(last_modified_timestamp)
I have to process an item report CSV file every 1 hour. The CSV contains 150k+ records for 1 account and there are multiple accounts in my system. I was working previously on rails and there was active record gem to handle this use case very efficiently. I am looking for an alternate to this gem in Django or any built in method that will be helpful to import such large data in bulk.
So far I have tried this code.
class ItemReportService:
def call(self, file_url):
with open(file_url, 'r') as file:
reader = csv.DictReader(file)
products = []
for row in reader:
product = self.process_product(row)
products.append(product)
self.update_products(products)
def process_product(self, row):
print(f'Processing sku: {row["SKU"]}')
product = Product.objects.filter(
sku=row['SKU']).first() or Product(sku=row['SKU'])
product.listing_title = row['Product Name']
product.listed_price = row['Price']
product.buy_box_price = row['Buy Box Item Price'] + \
row['Buy Box Shipping Price']
product.status = row['Lifecycle Status']
return product
def update_products(self, products):
Product.objects.bulk_update(
products,
[
'listing_title',
'listed_price',
'buy_box_price',
'Lifecycle Status'
]
)
It is raising this exception because when there is a new product it doesn't have primary key assigned to it
ValueError: All bulk_update() objects must have a primary key set.
Django 4.1 has new parameters for bulk_create(update_conflicts=bool and update_fields=[])
If your model has a field UNIQUE usually Django would ignore it when creating new data. But if you set the update_conflicts parameter to True, the fields inside update_fields will be updated.
You are not saving the product in the database before applying bulk_update.
I have checked your code for this purpose, you can use bulk_insert with an additional parameter
Model.objects.bulk_create(self.data, ignore_conflicts=True)
or
columns = ['column1', 'column2']
obj = Model.objects.filter(column1="sku").first()
if not obj:
obj = Model.objects.create(column1="sku")
obj.column1 = row["column1"] or obj.column1
obj.column2 = row["column2"] or obj.column2
items_to_be_inserted.append(obj)
In the end, you can do bulk update like
Model.objects.bulk_update(items_to_be_inserted, columns)
This will solve your problem.
I made this class function which can be used on any Django model in a project.
from django.db import models
class BaseModel(models.Model):
#classmethod
def bulk_create_or_update(
cls, uniques: list[str],
defaults: list[str],
data: list[dict]
):
# Get existing object list
data_dict, select = {}, None
for entry in data:
sub_entry, key = {}, ''
for uniq in uniques:
sub_entry[uniq] = entry[uniq]
key += str(entry[uniq])
data_dict[key] = entry
if not select:
select = models.Q(**sub_entry)
continue
select |= models.Q(**sub_entry)
records = cls.objects.filter(select).values('pk', *uniques)
existing = {}
for rec in records:
key = ''
for uniq in uniques:
key += str(rec[uniq])
existing[key] = rec
# Split new objects from existing ones
to_create, to_update = [], []
for key, entry in data_dict.items():
obj = cls(**entry)
if key not in existing:
to_create.append(obj)
continue
obj.pk = existing[key]['pk']
to_update.append(obj)
cls.objects.bulk_create(to_create, batch_size=1000)
cls.objects.bulk_update(to_create, defaults, batch_size=1000)
Let take an usage example
class Product(BaseModel)
price = models.IntegerField()
name = models.CharField(max_length=128, unique=True)
status = models.CharField(max_length=128)
if __name__ == '__main__':
data = [
{'price': 50, 'name': 'p1', 'status': 'New'},
{'price': 33, 'name': 'p2', 'status': 'Old'}
]
Product.bulk_create_or_update(uniques=['name'], defaults=['price', 'status'], data=data)
Any improvement suggestion of the code is welcome.
My file:
Goal: to read a txt file and return dictionaries
What I have:
def load_snacks(snack_file: TextIO) -> Tuple[Dict[str, List[str]],
Dict[str, List[str]]]:
"""Return a two-item tuple containing a "healthysnack_to_junkfood" dictionary
and a "healthysnack_to_healthysnack" dictionary with the data from snack_file.
"""
snack_H2J = {}
snack_H2H = {}
line = snack_file.readline().strip()
while line != '':
# due to structure of the file, line contains a healthy snack name
healthysnack_name = line
# properly format the 1st healthy snack name, use helper fcn (see below for helper fcn)
flip_name_and_del_comma(healthysnack_name)
healthy_list = []
junk_list = []
line = snack_file.readline().strip()
while line != '\n':
if ',' in line:
snack_H2J[healthysnack_name] = line
a = flip_name_and_del_comma(line)
healthy_list.append(a)
else:
snack_H2H[healthysnack_name] = line
junk_list.append(line)
line = snack_file.readline().strip()
return (snack_H2J, snack_H2H)
Below is my helperfcn; I have verified that this works
def flip_name_and_del_comma(s: str) -> str:
""" Retrun a new str that reverses the format name order from 'colour, healthy snack name' to
'healthy snack name to colour'
>>> flip_name_and_del_comma('orange, carrot')
'carrot orange'
>>> flip_name_and_del_comma('yellow, mango')
'mango yellow'
"""
s_reversed = ', '.join(reversed(s.split(', ')))
s_comma_delete = s_reversed.replace(', ', ' ')
return s_comma_delete
Any help is much appreciated! Thanks
I have a dictionary made up of lines extracted from a file like this:
Danny Shalev, 050-1111111, aaa#aaa.com
Gil Rom, 050-2222222, bbb#bbb.com
Tal Yakir, 050-3333333, ccc#ccc.com
Edit: my goal is for the dict to be printed out like this:
Danny Shalev - 050-1111111 - aaa#aaa.com
Gil Rom - 050-2222222 - bbb#bbb.com
Tal Yakir - 050-3333333 - ccc#ccc.com
The first name is the key, and the rest are the values.
I have written the code for converting the file lines into a dict, and I want to print out all values from my dictionary in a specific format, which would be line by line, separated by "-". I have already written the function print_person, to print it out in this format, I just want to apply this function (from the previous class) into my dict.
Here's the code:
class Person:
def __init__(self, name, phone,email):
self.name = name
self.phone = phone
self.email = email
def print_person(self):
return (str(self.name)+" - "+str(self.phone)+" - "+str(self.email))
class AddressBook:
def __init__ (self):
self.contactsdict = {}
def add(self, newContact):
self.contactsdict[newContact.name] = newContact.phone + " - " + newContact.email
def search(self, name):
return (self.contactsdict.get(name))
def addFromFile(self, fileName):
f = open("contacts.txt")
for line in f:
(key, val, val2) = line.split(",")
self.contactsdict[key] = val + " - " + val2
f.close
def printAddressBook(self):
for key, val in self.contactsdict.items():
Person.print_person
address = AddressBook() # make an instance
p1=Person("Danny Shalev","050-1111111","aaa#aaa.com")
print (p1.print_person())
address.add(p1)
address.addFromFile("contacts.txt")
address.printAddressBook()
I believe the problem is in this section, since I don't know how to use the method:
def printAddressBook(self):
for key, val in self.contactsdict.items():
Person.print_person
This
for key, val in self.contactsdict.items():
Person.print_person
deconstructs all your dictionary entries into 2 variables, one the key, the other the value. The second line is incorrrect - Person is your class, you need an instance of the class to use the defined print method on it.
You can call val.print_person() on each instance of the class Person to print each IF you store Persons in your inner dictionary. Classes are "templates" how a class is constructed - the instance must be used to call its functions. Currently your code only stores string in the internal dictionary.
To add persons to your internal Dict replace
for line in f:
(key, val, val2) = line.split(",")
self.contactsdict[key] = val + " - " + val2
with
for line in f:
(key, val, val2) = line.split(",")
self.contactsdict[key] = Person(key,val,val2) # create instances of Persons
# and store them in the dictionary by there name
# you get collisions if your file contains persons with identical names
Fixed code (this and some other errors marked with comments):
class Person:
def __init__(self, name, phone,email):
self.name = name
self.phone = phone
self.email = email
def print_person(self):
return (str(self.name) + " - " + str(self.phone) + " - " + str(self.email))
class AddressBook:
def __init__(self):
self.contactsdict = {}
def add(self, newContact):
self.contactsdict[newContact.name] = newContact # store the Person instance
# under its name as key
def search(self, name):
return (self.contactsdict.get(name))
def addFromFile(self, fileName):
f = open("contacts.txt")
for line in f:
(key, val, val2) = line.split(",")
self.add(Person(key,val,val2)) # create Person and use own add-Function
# to add it to internal dictionary
f.close
def printAddressBook(self):
for key, val in self.contactsdict.items():
print( val.print_person() ) # you need to print the output
# of print_person() - it currently only
# returns the string and does not print it
address = AddressBook() # make an instance
p1 = Person("Danny Shalev","050-1111111","aaa#aaa.com")
print(p1.print_person())
address.add(p1)
address.addFromFile("contacts.txt")
address.printAddressBook()
Search returns a person, so you can use it to change persons inside your dict:
print("")
p = address.search("Danny Shalev")
p.name = "Hallo" # change the name of the found person (the key will still be "Danny Shalev")
address.printAddressBook()
Output:
Danny Shalev - 050-1111111 - aaa#aaa.com
Gil Rom - 050-2222222 - bbb#bbb.com
Tal Yakir - 050-3333333 - ccc#ccc.com
Hallo - 050-1111111 - aaa#aaa.com # after change of searched person
Gil Rom - 050-2222222 - bbb#bbb.com
Tal Yakir - 050-3333333 - ccc#ccc.com
I'm wondering: how do you print the keys or the values individually from a dictionary in a function?
Example .txt file
00000000;Pikachu Muchacho;region1
11111111;SoSo good;region2
22222222;Marshaw williams;region3
33333333;larry Mikal Carter;region3
Code
test_file = open("test.txt", "r")
customer = {}
def dictionary():
for line in test_file:
entries = line.split(";")
key = entries[0]
values = entries[1]
customer[key] = values
def test():
print(customer)
print(customer[key])
def main():
dictionary()
test()
main()
As #jamesRH commented, you can use customer.keys() and customer.values():
test_file = open("test.txt", "r")
customer = {}
def dictionary():
for line in test_file:
entries = line.split(";")
key = entries[0]
values = entries[1]
customer[key] = values
def test():
# Print all the keys in customer
print(customer.keys())
# Print all the values in customer
print(customer.values())
def main():
dictionary()
test()
main()
This gives the output:
['00000000', '22222222', '33333333', '11111111']
['Pikachu Muchacho', 'Marshaw williams', 'larry Mikal Carter', 'SoSo good']
Your original code causes an error because key is not within the scope of test().