How can I upsert using class Session() of sqlalchemy when I have some uniqueconstraint into the tables?
I have this function in pyarchinit_dbmanager.py:
def insert_data_session(self, data):
Session = sessionmaker(bind=self.engine, autoflush=False)
session = Session()
session.add(data)
session.commit()
session.close()
And the code in pyarchinit_Config.py:
if mapper_class_write == 'PYUS' :
for sing_rec in range(len(data_list_toimp)):
try:
data = self.DB_MANAGER_write.insert_pyus(
self.DB_MANAGER_write.max_num_id(mapper_class_write,
id_table_class_mapper_conv_dict[mapper_class_write]) + 1,
data_list_toimp[sing_rec].area_s,
data_list_toimp[sing_rec].scavo_s,
data_list_toimp[sing_rec].us_s,
data_list_toimp[sing_rec].stratigraph_index_us,
data_list_toimp[sing_rec].tipo_us_s,
data_list_toimp[sing_rec].rilievo_originale,
data_list_toimp[sing_rec].disegnatore,
data_list_toimp[sing_rec].data,
data_list_toimp[sing_rec].tipo_doc,
data_list_toimp[sing_rec].nome_doc,
data_list_toimp[sing_rec].coord,
data_list_toimp[sing_rec].the_geom)
self.DB_MANAGER_write.insert_data_session(data)
value = (float(sing_rec)/float(len(data_list_toimp)))*100
self.progress_bar.setValue(value)
QApplication.processEvents()
except Exception as e :
QMessageBox.warning(self, "Errore", "Error ! \n"+ str(e), QMessageBox.Ok)
return 0
area_s, scavo_s, and us_s are the unique constraint.
So if I want to import the data without conclict it is OK but if I want to ignore the duplicate but just insert new data I have the problem with unique contraint.
At the moment I solved like that:
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.sql.expression import Insert
#compiles(Insert)
def _prefix_insert_with_ignore(insert, compiler, **kw):
return compiler.visit_insert(insert.prefix_with('OR IGNORE'), *kw)
def insert_data_session(self, data):
Session = sessionmaker(bind=self.engine, autoflush=False)
session = Session()
session.add(data)
session.commit()
session.close()
So I can skip the uniqconstarint and add new data and ignore the others.
Related
I have a model, User, in which the primary unique field is email. I have a separate Organization model which allows users to map to organizations through a many-to-many mapping.
I want the serializer to allow users to be created if they have an existing email but no organization association (organization is taken from the user making the request so is not in the payload).
The standard ModelSerializer includes a check against the field in to_internal_value(). I am consequently trying to override it like so:
def to_internal_value(self, data):
"""
Dict of native values <- Dict of primitive datatypes.
"""
fields = self._writable_fields
for field in fields:
validate_method = getattr(self, 'validate_' + field.field_name, None)
primitive_value = field.get_value(data)
try:
if validate_method == self.validate_email:
if User.objects.filter(email=primitive_value).exists():
if not self.active_organization.members.filter(email=primitive_value).exists():
continue
else:
validated_value = field.run_validation(primitive_value)
if validate_method is not None:
validated_value = validate_method(validated_value)
except ValidationError as exc:
errors[field.field_name] = exc.detail
except DjangoValidationError as exc:
errors[field.field_name] = get_error_detail(exc)
except SkipField:
pass
else:
set_value(ret, field.source_attrs, validated_value)
return super().to_internal_value(data)
This works, but the error that is returned if the object already has a record in User and Organization does not correctly map as a dictionary. For example, the validation error shows this:
[ErrorDetail(string='User with this Email already exists.', code='unique')]
Instead of what it should show, this:
{'email': [ErrorDetail(string='User with this Email already exists.', code='unique')]}
I tested by overriding the method and trying both my custom called code vs. the original, and it replicates the above findings:
def to_internal_value(self, data):
"""
Dict of native values <- Dict of primitive datatypes.
"""
try:
print('trying custom')
for field in self._writable_fields:
if field.field_name == 'email':
print(field.run_validation)
validated_value = field.run_validation(field.get_value(data))
except Exception as e:
print('error custom')
print(str(e))
try:
print('Trying original')
value = super().to_internal_value(data)
except Exception as e:
print('Exception - original')
print(str(e))
Output:
trying custom
<bound method CharField.run_validation of EmailField(max_length=254, validators=[<UniqueValidator(queryset=User.objects.all())>])>
error custom
[ErrorDetail(string='User with this Email already exists.', code='unique')]
Trying original
Exception - original
{'email': [ErrorDetail(string='User with this Email already exists.', code='unique')]}
Can anyone help me understand why this is happening please? I'm really stuck as to how this is happening.
I was not able to figure out why exactly the errors generated by to_internal_value(self, data) change when I copy the code from the underlying ModelSerializer vs calling super(). However, I did find a kind-of hackish workaround.
Because I strip out all the other related info (nested serializers, kind of) before hand, I'm really just worried about the email. Therefore, I traverse the _writable_fields twice: the first time I look for an existing email where it isn't in the organization. If those conditions are true, I return a dictionary and proceed to the create() method. If they fail, I call super. So far this seems to work. Here is my simplified class.
def to_internal_value(self, data):
"""
Dict of native values <- Dict of primitive datatypes.
"""
try:
for field in self._writable_fields:
if field.field_name == 'email':
primitive_value = field.get_value(data)
validator = EmailValidator()
validator(primitive_value)
if User.objects.filter(email=primitive_value).exists():
if not self.active_organization.members.filter(email=primitive_value).exists():
self.email_already_exists = True
return {'email': primitive_value}
except:
pass
return super().to_internal_value(data)
For reference, this is the complete ViewSet and Serializer I'm using:
class AdminUsersSerializer(serializers.ModelSerializer):
"""User serializer for the admin user view"""
groups = GenericGroupSerializer(source='group_set', many=True, required=False)
permission_sets = GenericPermissionSetSerializer(source='permissionset_set', many=True, required=False)
active_organization = None
email_already_exists = False
class Meta:
model = User
fields = ['pk', 'email', 'groups', 'permission_sets'] # , 'permissions']
def set_active_organization(self, organization):
self.active_organization = organization
def create(self, validated_data):
if not self.email_already_exists:
user = User.objects.create(
email=validated_data['email']
)
user.set_password(None)
user.save()
else:
user = User.objects.get(email=validated_data['email'])
return user
def to_internal_value(self, data):
"""
Dict of native values <- Dict of primitive datatypes.
"""
try:
for field in self._writable_fields:
if field.field_name == 'email':
primitive_value = field.get_value(data)
validator = EmailValidator()
validator(primitive_value)
if User.objects.filter(email=primitive_value).exists():
if not self.active_organization.members.filter(email=primitive_value).exists():
self.email_already_exists = True
return {'email': primitive_value}
except:
pass
return super().to_internal_value(data)
class AdminUsersViewSet(viewsets.ModelViewSet):
queryset = User.objects.all()
serializer_class = AdminUsersSerializer
permission_classes = [account_permissions.IsAdminRequired]
http_method_names = ['get', 'post']
active_organization = None
def create(self, request, *args, **kwargs):
self.active_organization = self.request.user.activeorganization.organization
groups = None
permission_sets = None
serializer = self.get_serializer(data=request.data)
serializer.set_active_organization(self.active_organization)
if 'groups' in serializer.initial_data:
if serializer.initial_data.get('groups'):
groups = serializer.initial_data.pop('groups')
else:
serializer.initial_data.pop('groups')
if 'permission_sets' in serializer.initial_data:
if serializer.initial_data.get('permission_sets'):
permission_sets = serializer.initial_data.pop('permission_sets')
else:
serializer.initial_data.pop('permission_sets')
# Normal method functions
serializer.is_valid(raise_exception=True)
self.perform_create(serializer)
headers = self.get_success_headers(serializer.data)
# Created user
user = User.objects.get(email=serializer.data['email'])
# Add user to organization
self.active_organization.members.add(user)
# Make sure to format the ajax groups data properly
if groups:
try:
group = Group.objects.get(
id=groups,
organization=self.active_organization
)
group.members.add(user)
except:
pass
if permission_sets:
try:
permission_set = PermissionSet.objects.get(
id=permission_sets,
organization=self.active_organization
)
permission_set.members.add(user)
except:
pass
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
def get_queryset(self):
return self.queryset.filter(
userorganizationmembership__organization=self.request.user.activeorganization.organization
).all()
It does feel hackish, but it seems to be getting the job done.
I'm using Scrapy to grab domains and their creation date using the Whois module. I am then adding them to a MySQL database using SqlAlchemy but I get the below error when adding the creation date to the database because the data type is <class 'datetime.datetime'>
sqlalchemy.orm.exc.UnmappedInstanceError: Class 'datetime.datetime' is not mapped
I tried to convert the date into a string but then I get another error.
pipelines.py:
class SaveDomainsPipeline(object):
def __init__(self):
engine = db_connect()
create_table(engine)
self.Session = sessionmaker(bind=engine)
def process_item(self, item, spider):
session = self.Session()
domain = Domains(**item)
domain_item = item['domain']
domain_whois = whois.query(domain_item)
creation_date = domain_whois.creation_date
try:
session.add_all([domain, creation_date])
session.commit()
models.py
class Domains(Base):
__tablename__ = "domains"
id = Column(Integer, primary_key=True)
date_added = Column(DateTime(timezone=True), server_default=func.now())
domain = Column('domain', Text())
creation_date = Column('creation_date', DateTime(timezone=True))
#creation_date = Column('creation_date', Text()) -- I also tried this
I made a rookie mistake in my original code.
As I initiated an instance of the class "Domains", I had to refer to it when populating the columns which I had originally missed. The working code can be found below.
class SaveDomainsPipeline(object):
def __init__(self):
engine = db_connect()
create_table(engine)
self.Session = sessionmaker(bind=engine)
def process_item(self, item, spider):
session = self.Session()
domains = Domains() #initiate instance of Domains class.
domains.domain = item['domain'] #Add the item "domain" from Items to DB
domain_whois = whois.query(domains.domain)
domains.creation_date = domain_whois.creation_date #Add the creation date to DB
try:
#save the instance which saves both the domain item and creation date.
session.add(domains)
session.commit()
hi I am a new in python i tried to create a new class that handle with sqlite3 in my read method i try to print some var but is does not print anything can you help me?
here is the code can you fix it and tell me what the problem is
class SQLite_class() :
file_name=''
table_sql_name =''
# con = sqlite3.connect(self.c)
def creat_file_name(self):
self.file_name = input("enter the SQL fille name !")
self.file_name = self.file_name+'.sqlite'
return self.file_name
def conncection(self):
conn = sqlite3.connect(self.file_name)
return conn
def creat_cursor(self):
conn = self.conncection()
cur = conn.cursor()
return cur
def del_table(self):
cur = self.creat_cursor()
cur.execute('DROP TABLE IF EXISTS '+self.table_sql)
def creat_table(self):
cur = self.creat_cursor()
#adding a new option (name of table of more we need to add some {} and use the format function)
cur.execute( '''CREATE TABLE IF NOT EXISTS {} (data TEXT, number INTEGER)'''.format(self.table_sql_name))
self.commiting()
def insrt(self):
cur = self.creat_cursor()
cur.execute('''INSERT INTO {} (data, number) VALUES (?, ?)'''.format(self.table_sql_name) ,('Thunderstruck', 20))
def close(self):
conn = self.conncection()
conn.close()
def commiting(self):
conn = self.conncection()
conn.commit()
def read(self):
cur = self.creat_cursor()
cur.execute('''SELECT data, number FROM {}'''.format(self.table_sql_name))
for row in cur: print(row)
test = SQLite_class()
test.creat_file_name()
test.table_sql_name = 'Track'
test.creat_table()
test.insrt()
test.commiting()
test.read()
test.commiting()
test.close()
This comment # con = sqlite3.connect(self.c) indicates you know what has to be done. I suspect when that didn't work, you went down the rabbit hole of creating connections everywhere. Program should make one db connection. Every time it makes a connection, it loses what came before.
It could declare conn = '', then call conncection() once (and remove it from the other methods; they'll probably need to take conn as an argument). It'll take some try, try again, but it will get you to a better place.
And don't forget to fetch the rows once the SELECT is executed.
I have a code like this,
# SWAMI KARUPPASWAMI THUNNAI
import jwt
import graphene
from flask import request
from auth.helper import medease_token
from database.get_connection import get_connection
from flask_graphql import GraphQLView
class CredentialInformation(graphene.ObjectType):
"""
graphene object type to get the personal information about the user
"""
country_code = graphene.String()
phone = graphene.String()
verified = graphene.Int()
#medease_token
def resolve_country_code(self, root):
customer_token = request.headers["x-access-token"]
decoded_token = jwt.decode(customer_token, verify=False)
customer_id = decoded_token["customer_id"]
try:
connection = get_connection()
cursor = connection.cursor()
cursor.execute("select country_code from customer_credential where id=%s limit 1", (customer_id, ))
result = cursor.fetchone()
return result["country_code"]
finally:
cursor.close()
connection.close()
#medease_token
def resolve_phone(self, root):
customer_token = request.headers["x-access-token"]
decoded_token = jwt.decode(customer_token, verify=False)
customer_id = decoded_token["customer_id"]
try:
connection = get_connection()
cursor = connection.cursor()
cursor.execute("select phone from customer_credential where id=%s limit 1", (customer_id, ))
result = cursor.fetchone()
return result["phone"]
finally:
cursor.close()
connection.close()
#medease_token
def resolve_verified(self, root):
customer_token = request.headers["x-access-token"]
decoded_token = jwt.decode(customer_token, verify=False)
customer_id = decoded_token["customer_id"]
try:
connection = get_connection()
cursor = connection.cursor()
cursor.execute("select verified from customer_credential where id=%s limit 1", (customer_id,))
result = cursor.fetchone()
return result["verified"]
finally:
cursor.close()
connection.close()
def credential_information_wrapper():
return GraphQLView.as_view("graphql", schema=graphene.Schema(query=CredentialInformation))
which uses flask-graphql and graphene for Python graphql. The code works absolutely fine but I think I am missing something here because I need to open new connections in every resolver, and I need to write same query again so there is a data duplication, so is this the right way of doing or am I missing something?
Any help would be highly appreciated! Thank you in advance.
Opening a new connection on every query (resolver in this case) is okay. You could also set a connection pool to minimize the cost of opening connections.
Just curious, what database adapter or ORM are you using? Would be great if you could refactor the connection and cursor steps into a single function to call on every resolver and only have to pass the SQL query string.
import lot_details, car_details
import numpy as np
import sqlite3
conn = sqlite3.connect('parkingLot.db')
c = conn.cursor()
class Parking(object):
"""
Parking class which has details about parking slots
as well as operation performed on parking are present here
"""
def __init__(self):
self.slots = {}
def create_parking_lot(self, no_of_slots):
try:
c.execute('CREATE TABLE IF NOT EXISTS parkingTable(slot_no REAL, reg_no TEXT, colour TEXT, total_time TEXT,charge TEXT)')
except Exception as ex:
print("Couldn't make a table in DB")
no_of_slots = int(no_of_slots)
if len(self.slots) > 0:
print ("Parking Lot already created")
return
if no_of_slots > 0:
for i in range(1, no_of_slots+1):
temp_slot = lot_details.PSlot(slot_no=i, available=True)
self.slots[i] = temp_slot
print ("Created a parking lot with {} slots" .format(no_of_slots))
else:
print ("Number of slots provided is incorrect.")
return
Using above code I am trying to create a database(which is successful) but now I am not able to create the Table inside it using the above command.
I tried doing it separately its working there. But, I am not able to create it using above code.
Depending on how and where you are creating the Parking object I suspect this could be a scoping issue, either pass the db connection into the constructor or just create it in the constructor itself. Code modified for brevity. The following works for me.
import sqlite3
class Parking(object):
def __init__(self):
self.slots = {}
self.conn = sqlite3.connect('parkingLot.db')
def create_parking_lot(self, no_of_slots):
try:
cur = self.conn.cursor()
cur.execute('CREATE TABLE IF NOT EXISTS parkingTable(slot_no REAL, reg_no TEXT, colour TEXT, total_time TEXT,charge TEXT)')
except Exception as ex:
print("Couldn't make a table in DB")
return
parking = Parking()
parking.create_parking_lot(5)