I wrote the following script for uploading a csv file to postgresql databse.
import psycopg2
import keys
con = psycopg2.connect(
host = keys.keys['host'],
database = keys.keys['database'],
user = keys.keys['user'],
password = keys.keys['password'])
#cursor
cur = con.cursor()
#execute query
#Already created ___#cur.execute("CREATE TABLE accounts (user_id serial PRIMARY KEY, username VARCHAR ( 50 ) UNIQUE NOT NULL, password VARCHAR ( 50 ) NOT NULL, email VARCHAR ( 255 ) UNIQUE NOT NULL, created_on TIMESTAMP NOT NULL, last_login TIMESTAMP)")
cur.execute("""\COPY "MyData" FROM 'C:\FILES\TestData.csv' DELIMITER ',' CSV HEADER;""")
#commit the transcation
con.commit()
#close the cursor
cur.close()
#close the connection
con.close()
But it returned the following error:-
SyntaxError: syntax error at or near "\"
LINE 1: \COPY "MyData" FROM 'C:\FILES\TestData.csv' DELIMITER ',' C...
I'm not a root user, so I could not directly use the COPY command.
Well.
You can use psycopg2's copy_from -> https://www.psycopg.org/docs/cursor.html#cursor.copy_from
So your code would look something like:
import psycopg2
import keys
con = psycopg2.connect(
host = keys.keys['host'],
database = keys.keys['database'],
user = keys.keys['user'],
password = keys.keys['password'])
#cursor
cur = con.cursor()
#execute query
#Already created ___#cur.execute("CREATE TABLE accounts (user_id serial PRIMARY KEY, username VARCHAR ( 50 ) UNIQUE NOT NULL, password VARCHAR ( 50 ) NOT NULL, email VARCHAR ( 255 ) UNIQUE NOT NULL, created_on TIMESTAMP NOT NULL, last_login TIMESTAMP)")
with open('C:\\Files\\TestData.csv', 'r') as acc:
next(acc) # This will skip the header
cur.copy_from(acc, 'accounts', sep=',')
#commit the transcation
con.commit()
#close the cursor
cur.close()
#close the connection
con.close()
Hope this answers your question.
Related
I am trying to insert values of a dataframe to postgres database. for that i am treating a temporary csv file and load the csv file, load the csv in buffer and using copy_from() i am inserting it into the database.
But Here I am getting an error [WinError 32] The process cannot access the file because it is being used by another process: 'temp.csv'
No other application is open except pycharm. Can some please help!
def copy_from_datafile(conn, df_s, table):
"""
save the given dataframe on disk as a csv file,
load the csv file in buffer and use copy_from() to copy it to the pgsql database table
"""
if table == "symbols_option":
command= '''CREATE TEMPORARY TABLE IF NOT EXISTS _symbol (
instrument_token INTEGER NOT NULL, exchange_token INTEGER, tradingsymbol TEXT NOT NULL, name TEXT,
expiry TIMESTAMP, strike DOUBLE PRECISION, lot_size INTEGER,instrument_type TEXT, segment TEXT NOT NULL, exchange TEXT, PRIMARY KEY (instrument_token,
tradingsymbol));'''
query_main = '''INSERT INTO {table}(instrument_token, exchange_token, tradingsymbol, name, expiry, strike, lot_size,instrument_type, segment, exchange )
SELECT * FROM _symbol
ON CONFLICT (instrument_token, tradingsymbol)
DO NOTHING
'''.format(table=table)
if table == "symbols_future":
command= '''CREATE TEMPORARY TABLE IF NOT EXISTS _symbol (
instrument_token INTEGER NOT NULL,exchange_token INTEGER, tradingsymbol TEXT NOT NULL,name TEXT,
expiry TIMESTAMP,lot_size INTEGER,instrument_type TEXT, segment TEXT NOT NULL, exchange TEXT, PRIMARY KEY (instrument_token,
tradingsymbol));'''
query_main = '''INSERT INTO {table}(instrument_token, exchange_token, tradingsymbol,name, expiry,lot_size,instrument_type, segment, exchange )
SELECT * FROM _symbol
ON CONFLICT (instrument_token, tradingsymbol)
DO NOTHING
'''.format(table=table)
tmp_df = 'temp.csv'
df_s.to_csv(tmp_df, header=False, index=False)
f = open(tmp_df, 'r')
cursor = conn.cursor()
try:
if table == "symbols_option":
# cursor.execute(command)
cursor.execute(command)
cursor.copy_from(f, '_symbol', sep=",", columns=["instrument_token", "exchange_token", "tradingsymbol", "name", "expiry", "strike", "lot_size","instrument_type", "segment", "exchange" ])
cursor.execute(query_main)
print("Data inserted using copy_from_datafile() successfully....")
else:
cursor.execute(command)
cursor.copy_from(f, '_symbol', sep=",", columns=[
"instrument_token", "exchange_token", "tradingsymbol", "name", "expiry", "lot_size",
"instrument_type", "segment", "exchange"])
cursor.execute(query_main)
print("Data inserted using copy_from_datafile() successfully....")
except (Exception, psycopg2.DatabaseError) as err:
os.remove(tmp_df)
print(err)
cursor.close()
else:
os.remove(tmp_df)
I'm using psycopg2 to connect to postgre DB, and to export the data into CSV file.
This is how I made the export DB to csv:
def export_table_to_csv(self, table, csv_path):
sql = "COPY (SELECT * FROM %s) TO STDOUT WITH CSV DELIMITER ','" % table
self.cur.execute(sql)
with open(csv_path, "w") as file:
self.cur.copy_expert(sql, file)
But the data is just the rows - without the column names.
How can I export the data with the column names?
P.S. I am able to print the column names:
sql = '''SELECT * FROM test'''
self.cur.execute(sql)
column_names = [desc[0] for desc in self.cur.description]
for i in column_names:
print(i)
I want the cleanest way to do export the DB with columns name (i.e. I prefer to do this in one method, and not rename columns In retrospect).
As I said in my comment, you can add HEADER to the WITH clause of your SQL:
sql = "COPY (SELECT * FROM export_test) TO STDOUT WITH CSV HEADER"
By default, comma delimiters are used with CSV option so you don't need to specify.
For future Questions, you should submit a minimal reproducible example. That is, code we can directly copy and paste and run. I was curious if this would work so I made one and tried it:
import psycopg2
conn = psycopg2.connect('host=<host> dbname=<dbname> user=<user>')
cur = conn.cursor()
# create test table
cur.execute('DROP TABLE IF EXISTS export_test')
sql = '''CREATE TABLE export_test
(
id integer,
uname text,
fruit1 text,
fruit2 text,
fruit3 text
)'''
cur.execute(sql)
# insert data into table
sql = '''BEGIN;
insert into export_test
(id, uname, fruit1, fruit2, fruit3)
values(1, 'tom jones', 'apple', 'banana', 'pear');
insert into export_test
(id, uname, fruit1, fruit2, fruit3)
values(2, 'billy idol', 'orange', 'cherry', 'strawberry');
COMMIT;'''
cur.execute(sql)
# export to csv
fid = open('export_test.csv', 'w')
sql = "COPY (SELECT * FROM export_test) TO STDOUT WITH CSV HEADER"
cur.copy_expert(sql, fid)
fid.close()
And the resultant file is:
id,uname,fruit1,fruit2,fruit3
1,tom jones,apple,banana,pear
2,billy idol,orange,cherry,strawberry
I made a postrgesql database on heroku. Then I tried to access it via a python script. For that I wrote the following code.
import psycopg2
#connect to the db
con = psycopg2.connect(
host = "an_aws_ec2_instance",
database="d57kjtuarj5li8",
user = "mmpywqodjzxlzr",
password = "************************************")
#cursor
cur = con.cursor()
#execute query
cur.execute("CREATE TABLE accounts (user_id serial PRIMARY KEY, username VARCHAR ( 50 ) UNIQUE NOT NULL, password VARCHAR ( 50 ) NOT NULL, email VARCHAR ( 255 ) UNIQUE NOT NULL, created_on TIMESTAMP NOT NULL, last_login TIMESTAMP)")
cur.execute = ("""SELECT * FROM accounts""")
rows = cur.fetchall()
for r in rows:
print (r)
#commit the transcation
con.commit()
#close the cursor
cur.close()
#close the connection
con.close()
But on executing the code, I got the following error.
---> 15 cur.execute = ("""SELECT * FROM accounts""")
16 rows = cur.fetchall()
AttributeError: 'psycopg2.extensions.cursor' object attribute 'execute' is read-only
When you comapare your two SQL lines, there is a major difference.
This calls the method execute on the object cur, with the SQL statement as parameter:
cur.execute("CREATE TABLE accounts (user_id serial PRIMARY KEY, username VARCHAR ( 50 ) UNIQUE NOT NULL, password VARCHAR ( 50 ) NOT NULL, email VARCHAR ( 255 ) UNIQUE NOT NULL, created_on TIMESTAMP NOT NULL, last_login TIMESTAMP)")
This here is something different. Here you are trying to assign a string (the SQL statement) to an attribute (execute), which is read-only on that object.
cur.execute = ("""SELECT * FROM accounts""")
# THIS
if remove the = from the second statement, you're fine.
I created one-to-many relationship table and according to the sqlite3 documentation I can't insert value into the child table if the referenced table column value in the parent table does not exist.
import sqlite3
class Database:
def __init__(self, database_name):
self.database_name = database_name
def create_table(self, table_name, *columns):
columns = ", ".join(columns)
conn = sqlite3.connect(self.database_name)
cursor = conn.cursor()
_SQL = f"CREATE TABLE IF NOT EXISTS {table_name}({columns})"
cursor.execute(_SQL)
conn.commit()
cursor.close()
conn.close()
def insert_values(self, table_name, values, *columns):
dynamic_values = ('?, ' * len(columns))[0:-2]
columns = ", ".join(columns)
conn = sqlite3.connect(self.database_name)
cursor = conn.cursor()
_SQL = f"INSERT INTO {table_name}({columns}) VALUES ({dynamic_values})"
cursor.execute(_SQL, values)
conn.commit()
cursor.close()
conn.close()
def view_values(self, table_name, *columns):
columns = ", ".join(columns)
conn = sqlite3.connect(self.database_name)
cursor = conn.cursor()
_SQL = f"SELECT {columns} FROM {table_name}"
cursor.execute(_SQL)
the_data = cursor.fetchall()
cursor.close()
conn.close()
return the_data
data = Database("games.db")
#
# data.create_table("supplier_groups", "group_id integer PRIMARY KEY", "group_name text NOT NULL")
#
data.insert_values("supplier_groups", ("Domestic", ), "group_name")
# data.create_table("suppliers ", "supplier_id INTEGER PRIMARY KEY",
# "supplier_name TEXT NOT NULL",
# "group_id INTEGER NOT NULL, "
# "FOREIGN KEY (group_id) REFERENCES supplier_groups (group_id)")
data.insert_values("suppliers", ('ABC Inc.', 9), "supplier_name", "group_id")
as you see on this line: data.insert_values("supplier_groups", ("Domestic", ), "group_name") - I'm inserting a value into supplier_groups table
and then right here: data.insert_values("suppliers", ('ABC Inc.', 9), "supplier_name", "group_id") - I'm inserting value into suppliers table with the group_id that does not exist in the group_suppliers table. Python executes it successfully and adds value to the database, however when attemping to execute this command in SQLITE browser I get this error:
Execution finished with errors. Result: FOREIGN KEY constraint failed which is what python should also have done instead of adding it into the database.
So, could anyone explain me what's going on here? Do I understand something in the wrong way? Help would be appreciated
From Section 2. Enabling Foreign Key Support in the sqlite doc:
Assuming the library is compiled with foreign key constraints enabled, it must still be enabled by the application at runtime, using the PRAGMA foreign_keys command. For example:
sqlite> PRAGMA foreign_keys = ON;
What am I doing wrong here? It run's without error, it has created table, but rows are empty. Why?
import sqlite3
sqlite_file = (r"C:\Users\Dragan\PycharmProjects\MyProject\ArchLib2.db")
conn = sqlite3.connect(sqlite_file)
cursor = conn.cursor()
table_name = 'Archive'
sql = 'CREATE TABLE IF NOT EXISTS ' + table_name + '("first_name" varchar NOT NULL, "second_name" varchar NOT NULL)'
cursor.execute(sql)
sql = 'INSERT INTO ' + table_name + '(first_name,second_name) VALUES ("value1","value2");'
cursor.execute(sql)
cursor.close()
Ok so I found why it didn't INSERT data into table.
data in sql = string didnt have good formating ( this ' must be replaced with this "
second if you have string value like "value1" it has to have backslash on both sides like this "\value1\"
third and most important after insert execution line you have to add this line conn.commit()
Final code looks like this:
import sqlite3
sqlite_file = (r"C:\Users\Dragan\PycharmProjects\MyProject\ArchLib2.db")
conn = sqlite3.connect(sqlite_file)
cursor = conn.cursor()
table_name = 'Archive'
sql = "CREATE TABLE IF NOT EXISTS " + table_name + "(first_name varchar NOT NULL, datetime)"
cursor.execute(sql)
sql = "INSERT INTO " + table_name + "(first_name,datetime) VALUES (\"value1\",CURRENT_TIMESTAMP)"
cursor.execute(sql)
conn.commit()
cursor.close()