I would like to insert multiple records from pandas dataframe into MS Azure table via python:
lst = [
('111', 'AA', 'AAA'),
('222', 'BB', 'BBB'),
('333', 'CC', 'CCC')
]
cursor = conn.cursor()
cursor.fast_executemany = True
SQLCommand = ("INSERT INTO [dbo].[ServiceRequestCollection_tmp] (ObjectID,BuyerMainContactPartyID,BuyerMainContactPartyName) VALUES (?,?,?);")
cursor.execute(SQLCommand, lst)
cursor.commit()
Unfortunately, system generates the following error. Can I kindly ask you to help me
pyodbc.ProgrammingError: ("A TVP's rows must all be the same size.", 'HY000')
The above exception was the direct cause of the following exception:
SystemError: <built-in function utf_16_le_encode> returned a result with an error set
The above exception was the direct cause of the following exception:
SystemError: encoding with 'utf-16le' codec failed (SystemError: <built-in function utf_16_le_encode> returned a result with an error set)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:/Users/103925alf1/PycharmProjects/p10/p11.py", line 35, in <module>
cursor.execute(SQLCommand, lst)
SystemError: <class 'pyodbc.Error'> returned a result with an error set
Process finished with exit code 1
You can directly set cursor.fast_executemany = False.
Also, you can upgrade to pyodbc 4.0.24 (or newer) and use setinputsizes to specify the parameter type, etc..
sql = "INSERT INTO #issue295 (txt, dec) VALUES (?, ?)"
params = [('Ώπα', 3.141)]
# explicitly set parameter type/size/precision
crsr.setinputsizes([(pyodbc.SQL_WVARCHAR, 50, 0), (pyodbc.SQL_DECIMAL, 18, 4)])
crsr.fast_executemany = True
crsr.executemany(sql, params)
For more details, you could refer to this wiki.
Related
I want to delete rows based on a datetime filter.
I created a table with DateTime column without timezone using similar script.
class VolumeInfo(Base):
...
date: datetime.datetime = Column(DateTime, nullable=False)
Then I try to delete rows using such filter
days_interval = 10
to_date = datetime.datetime.combine(
datetime.datetime.utcnow().date(),
datetime.time(0, 0, 0, 0),
).replace(tzinfo=None)
from_date = to_date - datetime.timedelta(days=days_interval)
query = delete(VolumeInfo).where(Volume.date < from_date)
Unexpectedly, sometimes there is no error, but sometimes there is the error:
Traceback (most recent call last):
...
File "script.py", line 381, in delete_volumes
db.execute(query)
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 1660, in execute
) = compile_state_cls.orm_pre_session_exec(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py", line 1843, in orm_pre_session_exec
update_options = cls._do_pre_synchronize_evaluate(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py", line 2007, in _do_pre_synchronize_evaluate
matched_objects = [
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py", line 2012, in <listcomp>
and eval_condition(state.obj())
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/evaluator.py", line 211, in evaluate
return operator(eval_left(obj), eval_right(obj))
TypeError: can't compare offset-naive and offset-aware datetimes
Using python3.10 in docker (image python:3.10-slim) and postgreSQL database with psycopg2 driver.
I have already tried all possible options, but this error appears every once in a while
How can i solve this? or where I made a mistake?
UPD1:
I keep getting an error with Pandas' 0.25.0 read_sql(). The code below is supposed to establish a couple connections, check if tables need to be created, and then begin extracting the result set from Oracle.
Stack Trace
RESTART: C:\xxxx\xxxx\AppData\Local\Programs\Python\Python36\xxxx\main.py
Creating connections
Attempting to connect
Connection to box_db Successful
Attempting to connect
Connection to JDE Successful
Box table already existed
BoxStaging table already existed
LotStartSiteMap table already existed
Extracting from JDE
Traceback (most recent call last):
File "C:\xxxx\xxxx\AppData\Local\Programs\Python\Python36\xxxx\main.py", line 42, in <module>
main()
File "C:\xxxx\xxxx\AppData\Local\Programs\Python\Python36\xxxx\main.py", line 26, in main
df = data.box_data_extract(queries.BOX_QUERY, jde_conn)
File "C:\xxxx\xxxx\AppData\Local\Programs\Python\Python36\xxxx\data.py", line 46, in box_data_extract
pd.read_sql_query(query, connection)
File "C:\xxxx\xxxx\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\sql.py", line 332, in read_sql_query
chunksize=chunksize,
File "C:\xxxx\xxxx\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\sql.py", line 1645, in read_query
cursor = self.execute(*args)
File "C:\xxxx\xxxx\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\sql.py", line 1590, in execute
cur = self.con.cursor()
AttributeError: module 'connection' has no attribute 'cursor'
main.py
import connection
import credential
import data
import emailer
import queries
import tables
def main():
# Create connection to box_db file and JDE
#
print("Creating connections")
sqlite_conn = connection.box_db_connect(tables.DB_PATH)
jde_conn = connection.mrap0680_connect(credential.USERNAME, credential.MRAP0680_PASSWORD)
# If new path has been chosen create sqlite db and tables accordingly
#
tables.create_box_table(sqlite_conn)
tables.create_box_staging_table(sqlite_conn)
tables.create_lot_startsite_map_table(sqlite_conn)
# Extract & insert JDE Box Query results into BoxStaging table
#
print("Extracting from JDE")
df = data.box_data_extract(queries.BOX_QUERY, jde_conn) #############LINE CAUSING ERROR
print("Storing in box_db")
data.box_data_insert(df, sqlite_conn)
print("BoxStaging updated")
data.py
# Extract the box data from JDE
#
def box_data_extract(query, connnection):
try:
df = pd.read_sql(query, con=connection)
print("JDE Box Data extracted")
return df
except Exception as e:
print(repr(e))
connection.py
def xxxx_connect(username, password):
# Connect to JDE database
try:
print("Attempting to connect")
conn = cx_Oracle.connect(user=username, password=password, dsn=DSN_TNS)
print("Connection to JDE Successful")
return conn
except Exception as e:
print(repr(e))
I am able to extract the data if running manually.
Please help. Thanks.
I found the error! The issue was in data.py. In the parameter list of data_box_extract, connection is spelled as "connnection". Inside the function, "connection" was being used so a cursor couldn't be instantiated. Not sure why it didn't throw an error about "connection" not being instantiated. Oh well, I'm writing the program in IDLE, maybe it's time to switch to an IDE that will catch those mistakes...
Thanks to all who viewed!
I encoded a value from input file and inserted into Sqlite DB
cur.execute('''INSERT INTO Locations (address, geodata)
VALUES ( ?, ? )''', (memoryview(address.encode()), memoryview(data.encode()) ) )
Now I'm trying to decode it but I'm getting
Traceback (most recent call last):
File "return.py", line 9, in
print(c.decode('utf-8'))
AttributeError: 'tuple' object has no attribute 'decode'
My code looks like this:
import sqlite3
conn = sqlite3.connect('geodata.sqlite')
cur = conn.cursor()
cur.execute('SELECT address FROM Locations')
for c in cur:
print(c.decode('utf-8'))
Regardless of how many columns are selected, rows are returned as tuples. You would get the first element of the tuple the usual way.
It seems like all no error in the code but no idea why I'm getting this.
I was creating a simple GUI app which store user details to a database(postgresql) and also they will be able to search for entries in database. This particular error ocures in this search() function so I haven't added the rest of the code. If necessary I can add them.
Hope I will get some solutions from this community.
def search(id):
conn = psycopg2.connect(dbname="postgres",user="postgres",password="1018",host="localhost",port="5432")
mycursor = conn.cursor()
query = '''select * from demotab where id=%s '''
mycursor.execute(query, (id))
row = mycursor.fetchone()
print(row)
conn.commit()
conn.close()
Getting this error below
Exception in Tkinter callback
Traceback (most recent call last):
File "c:\programdata\anaconda3\lib\tkinter\__init__.py", line 1702, in __call__
return self.func(*args)
File "appwithDB.py", line 51, in <lambda>
search_button = Button(newframe, text="Search", command=lambda : search(entry_search.get()))
File "appwithDB.py", line 71, in search
mycursor.execute(query, (id))
TypeError: not all arguments converted during string formatting
The second argument to mycursor.execute must be an iterable containing the values to insert in the query
You can use a list: mycursor.execute(query, [id])
or a one-element tuple: mycursor.execute(query, (id,))
Notice the comma. (id) is the same than id. In python, the comma is making the tuple, not the parenthesis.
I am trying to pass a variable in mysql Query but its keep giving error.
I am referign to this
https://dev.mysql.com/doc/connector-python/en/connector-python-example-cursor-select.html
mydb = mysql.connector.connect(
host="111.222.333.444.555",
user="user",
passwd="pass",
database="database_aname"
)
#print(mydb)
mycursor = mydb.cursor()
mycursor = mydb.cursor(buffered=True)
#lon=((data['hits']['hits'][i]['_source']['#timestamp']))
query = ("SELECT * from device_stock join customers on device_stock.user_id=customers.id WHERE device_stock.imei=%s")
#query = ("SELECT * from device_stock join customers on device_stock.user_id=customers.id WHERE device_stock.imei=0351777090784867")
mycursor.execute(query,(imei))
for row in mycursor:
print (row)
but it gives bellow error
Traceback (most recent call last):
mycursor.execute(query,(imei))
File "C:\Program Files\Python37\lib\site-packages\mysql\connector\cursor_cext.py", line 248, in execute
prepared = self._cnx.prepare_for_mysql(params)
File "C:\Program Files\Python37\lib\site-packages\mysql\connector\connection_cext.py", line 537, in prepare_for_mysql
raise ValueError("Could not process parameters")
ValueError: Could not process parameters
What can I do?
Thanks
try
mycursor.execute(query,(imei,))
(imei) is not taken as a tuple.