How can I upsert using class Session() of sqlalchemy when I have some uniqueconstraint into the tables?
I have this function in pyarchinit_dbmanager.py:
def insert_data_session(self, data):
Session = sessionmaker(bind=self.engine, autoflush=False)
session = Session()
session.add(data)
session.commit()
session.close()
And the code in pyarchinit_Config.py:
if mapper_class_write == 'PYUS' :
for sing_rec in range(len(data_list_toimp)):
try:
data = self.DB_MANAGER_write.insert_pyus(
self.DB_MANAGER_write.max_num_id(mapper_class_write,
id_table_class_mapper_conv_dict[mapper_class_write]) + 1,
data_list_toimp[sing_rec].area_s,
data_list_toimp[sing_rec].scavo_s,
data_list_toimp[sing_rec].us_s,
data_list_toimp[sing_rec].stratigraph_index_us,
data_list_toimp[sing_rec].tipo_us_s,
data_list_toimp[sing_rec].rilievo_originale,
data_list_toimp[sing_rec].disegnatore,
data_list_toimp[sing_rec].data,
data_list_toimp[sing_rec].tipo_doc,
data_list_toimp[sing_rec].nome_doc,
data_list_toimp[sing_rec].coord,
data_list_toimp[sing_rec].the_geom)
self.DB_MANAGER_write.insert_data_session(data)
value = (float(sing_rec)/float(len(data_list_toimp)))*100
self.progress_bar.setValue(value)
QApplication.processEvents()
except Exception as e :
QMessageBox.warning(self, "Errore", "Error ! \n"+ str(e), QMessageBox.Ok)
return 0
area_s, scavo_s, and us_s are the unique constraint.
So if I want to import the data without conclict it is OK but if I want to ignore the duplicate but just insert new data I have the problem with unique contraint.
At the moment I solved like that:
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.sql.expression import Insert
#compiles(Insert)
def _prefix_insert_with_ignore(insert, compiler, **kw):
return compiler.visit_insert(insert.prefix_with('OR IGNORE'), *kw)
def insert_data_session(self, data):
Session = sessionmaker(bind=self.engine, autoflush=False)
session = Session()
session.add(data)
session.commit()
session.close()
So I can skip the uniqconstarint and add new data and ignore the others.
I am unable to run scrapy through my pipeline to my local database. I have already installed mysql-connector-python 8.0.19 and am able to write data to the database within the same project but outside of a Scrapy pipeline . Can someone please help i can't figure out why it isn't working.
When i try to send data via scrapy pipeline i get the following error:
[twisted] CRITICAL: Unhandled error in Deferred:
File "C:\Users\Viking\PycharmProjects\Indigo_Scrp\IndgoScrp\IndgoScrp\pipelines.py", line 7, in <module>
from mysql.connector import (connection)
ModuleNotFoundError: No module named 'mysql
Here is my code for the pipeline :
from mysql.connector import (connection)
from mysql.connector import errorcode
class IndgoscrpPipeline(object):
def __init__(self):
self.create_connection()
self.create_table()
def create_connection(self):
self.conn = connection.MySQLConnection(
host='127.0.0.1',
user='root',
passwd='',
database='Python'
)
self.curr = self.conn.cursor()
def open_spider(self, spider):
print("spider open")
def process_item(self, item, spider):
print("Saving item into db ...")
self.save(dict(item))
return item
def close_spider(self, spider):
self.mysql_close()
##########################################################################
def mysql_connect(self):
try:
return self.curr.connect(**self.conf)
except self.curr.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
print("Something is wrong with your user name or password")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
print("Database does not exist")
else:
print(err)
#########################################
def create_table(self):
self.curr.execute(""" DROP TABLE IF EXISTS indigo""")
self.curr.execute(""" Create table indigo(
Product_Name text,
Product_Author text,
Product_Price text,
Product_Image text
)""")
def process_item(self, item, spider):
self.store_db(item)
def store_db(self, item):
self.curr.execute("""Insert Into indigo values (%s,%s,%s,%s)""",
(item['Product_Name'][0],
item['Product_Author'][0],
item['Product_Price'][0],
item['Product_Image'][0],
)
)
self.conn.commit()
return item
self.conn.close()
*
Here is my code from my spider
import scrapy
from ..items import IndScrItem
class IndgoSpider(scrapy.Spider):
name = 'Indgo'
start_urls = ['https://www.chapters.indigo.ca/en-ca/books/?link-usage=Header%3A%20books&mc=Book&lu=Main']
def parse(self, response):
items = IndScrItem()
Product_Name= response.css('.product-list__product-title-link--grid::text').getall(),
Product_Author= response.css('.product-list__contributor::text').getall(),
Product_Price= response.css('.product-list__price--orange::text').getall(),
Product_Image= response.css('.product-image--lazy::attr(src)').getall()
items['Product_Name'] = Product_Name
items['Product_Author'] = Product_Author
items['Product_Price'] = Product_Price
items['Product_Image'] = Product_Image
yield items
This is the line in the settings file that i have to enable pipelines
ITEM_PIPELINES = {
'IndgoScrp.pipelines.IndgoscrpPipeline': 100,
}
I actually found the issue was tied to having previously pip installed the wrong version of mysql-connector even though through my ide pycharm i had installed the correct one python was confused. After uninstalling both and reinstalling mysql-connector-python it was able to run.
[I am not good at English. Please understand. :
) ]
Sometimes even though using implicate timeout, chrome driver does not end.
To prevent this, I'm using the Timeout Decorator for Windows.
The timeout decorator works well,
But then, the Chrome driver not be shut down.
I also checked if it was the same object, but the object was the same.
What's the reason?
It seems to be using the timeout decorator...(the Chrome driver is also the latest version.)
self.driver.quit() <---- There is a problem with this method.
#timeout(10)
def driver_quit(self):
self.driver.quit()
#timeout(120)
def driver_get(self, url):
self.driver.get(url)
def call_url(self, url):
try:
self.driver_get(url)
except Exception as e:
try:
self.driver_quit()
except Exception as e:
pass
def timeout(timeout):
from threading import Thread
import functools
def deco(func):
#functools.wraps(func)
def wrapper(*args, **kwargs):
res = [Exception('function [%s] timeout [%s seconds] exceeded!' % (func.__name__, timeout))]
def newFunc():
try:
res[0] = func(*args, **kwargs)
except Exception as e:
res[0] = e
t = Thread(target=newFunc)
t.daemon = True
try:
t.start()
t.join(timeout)
except Exception as je:
print('error starting thread')
raise je
ret = res[0]
if isinstance(ret, BaseException):
raise ret
return ret
return wrapper
return deco
=============== Modified code ===============
WebDriverException occurs on finally,
but Chamedriver shut down this line ==> driver.close().
def call_url(self, url):
try:
self.driver_get(url)
except:
try:
self.driver_quit()
except:
pass
finally:
self.driver.close()
self.driver.quit()
A workaround is to call both driver.quit() and driver.close().
To do so you can put the commands in a finally: statement.
You will wrap all your automation with a try: and except: then use the finally: statement at the end.
try:
# do my automated tasks
except:
pass
finally:
driver.close()
driver.quit()
EDIT
If you find this does not help you just report a bug to selenium and the webdriver maintainers.
Hope this helps you!
In the following Database class if the connection fails my error is printed. But, a system error is also printed because the __exit__ code fires.
AttributeError: 'Database' object has no attribute '_conn'
How should I handle this type of situation?
try/except Or is there a better way?
import psycopg2
import mysecrets as ms
from time import time
class Database(object):
def __init__(self, localhost=False):
try:
print(f"ATTEMPTING DATABASE CONNECTION")
self._conn = psycopg2.connect(
f"""
dbname = {ms.LOCALDB_NAME if localhost else ms.DB_NAME}
port = {ms.LOCALDB_PORT if localhost else ms.DB_PORT}
user = {ms.LOCALDB_USER if localhost else ms.DB_USER}
password = {ms.LOCALDB_PASS if localhost else ms.DB_PASS}
host = {ms.LOCALDB_HOST if localhost else ms.DB_HOST}
"""
)
self._cursor = self._conn.cursor()
print(f"CONNECTED TO DATABASE\n")
except Exception as error:
print(f"UNABLE TO CONNECT TO DATABASE\n{error}")
def __enter__(self):
return self
def __del__(self):
self.connection.close()
def __exit__(self, exc_type, exc_value, traceback):
self.connection.close()
#property
def connection(self):
return self._conn
#property
def cursor(self):
return self._cursor
def commit(self):
self.connection.commit()
def query(self, sql, params=None):
self.cursor.execute(sql, params or ())
def fetchall(self):
return self.cursor.fetchall()
def fetchone(self):
return self.cursor.fetchone()
if __name__ == "__main__":
start_time = time()
with Database() as db:
elapsed_time = round(time() - start_time, 2)
print(f"Testing database connection took {elapsed_time} seconds.")
In this case you can initialize self._conn first thing, before the try, as self._conn = None, which makes it exist. Then, in the __exit__, close, etc. guard it with
if self._conn is not None:
self._conn.close()
self._conn = None
However, I don't think it's a good design the catch and ignore the general Exception error. It might be something other than a connection error. Even if it is, you end up with an uninitialized object. That's not useful, and will cause later errors. Better to catch them early and let it just pass from the __init__, and let whatever instantiated it deal with it.
First, catching Exception instead of a specific error is an antipattern.
In Python, you should know what exceptions are thrown from your functions and handle them accordingly.
Second, if your call to psycopg2.connect or _conn.cursor() throws any error, then your code catches the exception and your instance variables _conn and _cursor are never set.
That is the cause of your AttributeError.
IMO, the right way to handle it would be to raise an error and do not continue with execution if your database class cannot connect to a database or let whatever code is using this class handle the error.
class Database(object):
def __init__(self, localhost=False):
try:
print(f"ATTEMPTING DATABASE CONNECTION")
self._conn = psycopg2.connect(
f"""
dbname = {ms.LOCALDB_NAME if localhost else ms.DB_NAME}
port = {ms.LOCALDB_PORT if localhost else ms.DB_PORT}
user = {ms.LOCALDB_USER if localhost else ms.DB_USER}
password = {ms.LOCALDB_PASSWORD if localhost else ms.DB_PASSWORD}
host = {ms.LOCALDB_HOST if localhost else ms.DB_HOST}
"""
)
self._cursor = self._conn.cursor()
print(f"CONNECTED TO DATABASE\n")
except psycopg2.Error as error:
raise ValueError(f"UNABLE TO CONNECT TO DATABASE\n{error}") from None
There's probably a better exception than ValueError to use.
I have a list
mylist = ["18/10/2018","35029128","4T1BF28B6YU095317","TOYOTA","AVALON XL","2000","525","CA","HAYWARD","CA - HAYWARD","3","1200","PDT","Automobile","SEDAN 4D","GRAY","MINOR DENT/SCRATCHES"," ","CA","DQ","YES","D","240778","E","0","0","3.0L 6","Front-wheel Drive","AUTOMATIC","GAS","6","Run & Drive Verified","On Minimum Bid","0","","94545","USA","USD","SF007","N","0","XL","AUCTION DEALER SERVICES LLC"]
and function that inserts this list into database:
def insert_data(self, data):
sql_insert_data = """INSERT INTO copart(salesdata,lot,vin,make,model,year,saleprice,locstate,loccity,yardname,yardNumber,saletime,timezone,vehicle,bodystyle,color,damage,seconddamage,saletitlestate,saletitletype,haskey,lotcond,odometr,odometrtype,estvalue,repair,engine,drive,transmision,fule,cylinders,runsdrives,salestatus,startbid,specialnotes,loczip,loccountry,currency,gridrow,offer,buynow,trim,seller) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
"""
cur = self.conn.cursor()
try:
cur.execute(sql_insert_data, data)
except Error as e:
print(e)
print("inserted")
when i execute script, it ends without error but no data is inserted in database.
what is wrong in my code?
You need to commit after executing the sql.
Ex:
def insert_data(self, data):
sql_insert_data = """INSERT INTO copart(salesdata,lot,vin,make,model,year,saleprice,locstate,loccity,yardname,yardNumber,saletime,timezone,vehicle,bodystyle,color,damage,seconddamage,saletitlestate,saletitletype,haskey,lotcond,odometr,odometrtype,estvalue,repair,engine,drive,transmision,fule,cylinders,runsdrives,salestatus,startbid,specialnotes,loczip,loccountry,currency,gridrow,offer,buynow,trim,seller) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
"""
cur = self.conn.cursor()
try:
cur.execute(sql_insert_data, data)
self.conn.commit()
except Error as e:
print(e)
print("inserted")