I am using DataFrame to read data from each postgres table and using to_sql() method to insert data into the oracle. The problem I am facing is that It gets stuck after copying a few records to oracle. Jupyter Notebook gets busy but does nothing.
def duplicateData(conn, conn2, session):
query1 = "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'"
all_tables = session.execute(query1)
count = 0
for index,tables in enumerate(all_tables):
count += 1
# getting rid of comma and parathesis
for i, table in enumerate(tables):
print("\n"+table+" - NO: "+str(count)+"\n")
query2 = "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '" + table + "'"
columns = session.execute(query2)
cols = []
for col in columns:
cols.append(col[0])
query3 = "SELECT * FROM " + table
df = pd.read_sql(query3, conn)
alias = (table[:30] + '') if len(table) > 30 else table
df.to_sql(alias, conn2, index=False, schema="PMS")
print("\nDONE\n")
Related
I am writing a code to copy rows of matching columns from CSV to PostgreSQL table. I am using python and qgis for the same.Code is as follows
connection=psycopg2.connect(host=host, port=port, dbname=dbname, user=name_user, password=password)
cursor = connection.cursor ()
cursor.execute("""SELECT Count(*) FROM INFORMATION_SCHEMA.Columns where TABLE_NAME = 'table'""")
csv1 = pd.read_csv(self.dlg.lineEdit_5.text())
csvfile = open(self.dlg.lineEdit_5.text(),'r')
columnnames = csv1.columns.values
table=self.dlg.comboBox.currentText()
table_name = table.strip(' ' ' ')
self.dlg.lineEdit_6.setText(str(table))
with open(self.dlg.lineEdit_5.text(), 'r') as f:
reader = csv.reader(f)
next(reader) # This skips the 1st row which is the header.
for x in columnnames:
column = x.strip(' ' ' ')
#self.dlg.lineEdit_6.setText(str(column))
sql_insert = """INSERT INTO table_name(x) VALUES(%s)"""
for record in reader:
cursor.execute(sql_insert,[record])
connection.commit()
I am getting error as follows
psycopg2.errors.UndefinedTable: relation "table_name" does not exist
LINE 1: INSERT INTO table_name(x) VALUES(ARRAY['501','mah','A'])
How to resolve this error?. table_name exists in the database.
It was a silly mistake. I was taking table name from a variable in a python code. so, query need to be written as follows.
table=self.dlg.comboBox.currentText()
table_name = table.strip(' ' ' ')
sql_insert = """INSERT INTO %(table_name)s (x) VALUES(%s);"""
cursor.execute(sql_insert,[value])
connection.commit()
Not able to extract table names used within with clause, I'm using presto-parser version 0.226.
SqlParser sqlParser = new SqlParser();
String sql = "WITH dataset AS ( SELECT ROW('Bob', 38) AS users from tabb ) SELECT * FROM dataset";
Query query = (Query)sqlParser.createStatement(sql, ParsingOptions.builder().build());
QuerySpecification body = (QuerySpecification)query.getQueryBody();
System.out.println("From = " + body.getFrom().get());
/* Output
From = Table{dataset}
*/
Expected output
From = Table{dataset, tabb}
conn = pyodbc.connect("DRIVER={SQL Server};"
"SERVER="+server+";"
"UID="+username+";"
"PWD="+password,
autocommit=True)
cursor = conn.cursor()
database= "abcd"
sql_create = (
"DECLARE #sql AS NVARCHAR(MAX);"
"SET #sql = 'if not exists(select * from sys.databases where name = ' + QUOTENAME(?) + ')' + ' CREATE DATABASE ' + QUOTENAME(?);"
"EXEC sp_executesql #sql")
cursor.execute(sql_create,database,database)
Getting error msg like pyodbc.ProgrammingError: ('42S22', u"[42S22] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid column name 'abcd'. (207) (SQLExecDirectW)")"
Don't use QUOTENAME and concatenation for the WHERE clause parameter. Also, avoid using the legacy SQL Server ODBC driver that ships with Windows to access Azure SQL Database. Instead, download and use a newer ODBC driver. Below is an example with these changes.
conn = pyodbc.connect("DRIVER={ODBC Driver 17 for SQL Server};"
"SERVER="+server+";"
"UID="+username+";"
"PWD="+password,
autocommit=True)
cursor = conn.cursor()
database= "abcd"
sql_create = (
"DECLARE #sql AS NVARCHAR(MAX);"
"SET #sql = N'if not exists(select * from sys.databases where name = #DatabaseName)' + N' CREATE DATABASE ' + QUOTENAME(?) + N';';"
"EXEC sp_executesql #sql, N'#DatabaseName sysname', #DatabaseName = ?;")
cursor.execute(sql_create,database,database)
You could also declare a T-SQL variable for the database name and assign it to the parameter value so that you only need to pass a single parameter:
sql_create = (
"DECLARE #sql AS NVARCHAR(MAX);"
"DECLARE #DatabaseName sysname = ?;"
"SET #sql = N'if not exists(select * from sys.databases where name = #DatabaseName)' + N' CREATE DATABASE ' + QUOTENAME(#DatabaseName) + N';';"
"EXEC sp_executesql #sql, N'#DatabaseName sysname', #DatabaseName = #DatabaseName;")
cursor.execute(sql_create,database)
What am I doing wrong here? It run's without error, it has created table, but rows are empty. Why?
import sqlite3
sqlite_file = (r"C:\Users\Dragan\PycharmProjects\MyProject\ArchLib2.db")
conn = sqlite3.connect(sqlite_file)
cursor = conn.cursor()
table_name = 'Archive'
sql = 'CREATE TABLE IF NOT EXISTS ' + table_name + '("first_name" varchar NOT NULL, "second_name" varchar NOT NULL)'
cursor.execute(sql)
sql = 'INSERT INTO ' + table_name + '(first_name,second_name) VALUES ("value1","value2");'
cursor.execute(sql)
cursor.close()
Ok so I found why it didn't INSERT data into table.
data in sql = string didnt have good formating ( this ' must be replaced with this "
second if you have string value like "value1" it has to have backslash on both sides like this "\value1\"
third and most important after insert execution line you have to add this line conn.commit()
Final code looks like this:
import sqlite3
sqlite_file = (r"C:\Users\Dragan\PycharmProjects\MyProject\ArchLib2.db")
conn = sqlite3.connect(sqlite_file)
cursor = conn.cursor()
table_name = 'Archive'
sql = "CREATE TABLE IF NOT EXISTS " + table_name + "(first_name varchar NOT NULL, datetime)"
cursor.execute(sql)
sql = "INSERT INTO " + table_name + "(first_name,datetime) VALUES (\"value1\",CURRENT_TIMESTAMP)"
cursor.execute(sql)
conn.commit()
cursor.close()
I have two tables, one in ms sql and other one in sqlite3
I need to update a field in mssql table to the related value from sqlite3 table.
here is my code:
import pypyodbc
import sqlite3
connection = pypyodbc.connect('Driver={SQL Server};'
'Server=****'
'Database=****;'
'uid=****;pwd=****')
cursor = connection.cursor()
SQLCommand = ("select ObjectId, ObjectColumnName,Label from LocalizedLabel "
"where LanguageId = 1065")
sqlUpdate = ("UPDATE LocalizedLabel "
"SET Label = ? "
"WHERE ObjectId = ? and ObjectColumnName = ? and LanguageId = 1065")
cursor.execute(SQLCommand)
results = cursor.fetchall()
srcDB = sqlite3.connect('crmLLDB')
for result in results:
val = (result[0], result[1])
print('SELECT "1065" FROM crm '
'WHERE UPPER(ObjectID)= UPPER(%s) AND ObjectColumnName = %s' % val)
srcCursor = srcDB.execute('SELECT "1065" FROM crm '
'WHERE UPPER(ObjectID)= UPPER(?) AND ObjectColumnName = ?', val)
if srcCursor.fetchone() is None:
print("No translation found for " + result[2])
else:
translation = srcCursor.fetchone()[0]
updateVals = (translation, result[0], result[1])
cursor.execute(sqlUpdate, updateVals)
connection.close()
srcDB.close()
objectId is a GuID and other fields are all strings
the print function returns:
SELECT "1065" FROM crm WHERE UPPER(ObjectID)= UPPER(b'0340B506-2341-DB11-898A-0007E9E17EBD') AND ObjectColumnName = DisplayName
while the watcher gives:
result[0] = 'b\\'0340B506-2341-DB11-898A-0007E9E17EBD\\''
result[1] = 'DisplayName'
this query of course returns this error:
[1] [SQLITE_ERROR] SQL error or missing database (near "'0340B506-2341-DB11-898A-0007E9E17EBD'": syntax error)
while this query :
SELECT "1065" FROM crm WHERE UPPER(ObjectID)= UPPER('0340B506-2341-DB11-898A-0007E9E17EBD') AND ObjectColumnName = 'DisplayName'
return the perfect answer,
can someone point out my problem, please?
appearaently the problem was with pypyodbc and its problem with GuId,
I used pyodbc and everything works fine now!
my final code, if some googler in the future passes by:
import pyodbc
import sqlite3
startTme = datetime.now()
connection = pyodbc.connect('Driver={SQL Server};'
'Server=*****;'
'Database=****;'
'uid=sa;pwd=*****')
cursor = connection.cursor()
SQLCommand = ("select ObjectId, ObjectColumnName,Label from LocalizedLabel "
"where LanguageId = 1065")
sqlUpdate_p = ("UPDATE LocalizedLabel "
"SET Label = ? "
"WHERE ObjectId = ? and ObjectColumnName = ? and LanguageId = 1065")
cursor.execute(SQLCommand)
results = cursor.fetchall()
srcDB = sqlite3.connect('crmLLDB')
jobLength = str(len(results))
i = 0
for result in results:
val = (result[0], result[1])
srcCursor = srcDB.execute('SELECT "1065" FROM crm '
'WHERE UPPER(ObjectID)= UPPER(?) AND ObjectColumnName = ?', val)
trans = srcCursor.fetchall()
for tr in trans:
updateVals = (tr[0], result[0], result[1])
cursor.execute(sqlUpdate_p, updateVals)
i += 1
connection.commit()
connection.close()
srcDB.close()