I'm converting a postgres script into asyncpg.
im getting "asyncpg.exceptions.PostgresSyntaxError: syntax error at or near "%""
i assume my placeholder format is incorrect but i cant find an example of a correct format.
Original working psycopg2 code:
async def commit_trade_postgres(response_data_input):
conn = await psycopg2.connect(
"dbname='postgres' user='postgres' password = 'postgres123' host='localhost' port= '5432'")
cur = conn.cursor()
cur.execute(
"CREATE TABLE IF NOT EXISTS trade_{symbol} (time timestamptz NOT NULL ,side text, size float, price float, tick_direction text)".format(**response_data_input))
conn.commit()
cur.execute(
"SELECT create_hypertable('trade_{symbol}', 'time', if_not_exists => TRUE)".format(**response_data_input))
conn.commit()
cur.execute("INSERT INTO trade_{symbol} (time, side, size, price, tick_direction) VALUES (now(), %(side)s, %(size)s, %(price)s, %(tick_direction)s)".format(
**response_data_input), (response_data_input))
conn.commit()
print("commited trade")
My attempt as per the example code supplied int he docs:
async def commit_trade_postgres(response_data_input):
conn = await asyncpg.connect(database='postgres', user='postgres', password='postgres123', host='localhost', port='5432')
await conn.execute(
"CREATE TABLE IF NOT EXISTS trade_{symbol} (time timestamptz NOT NULL ,side text, size float, price float, tick_direction text)".format(**response_data_input))
await conn.execute(
"SELECT create_hypertable('trade_{symbol}', 'time', if_not_exists => TRUE)".format(**response_data_input))
await conn.execute("INSERT INTO trade_{symbol} (time, side, size, price, tick_direction) VALUES (now(), %(side)s, %(size)s, %(price)s, %(tick_direction)s)".format(
**response_data_input), (response_data_input))
print("commited trade")
EDIT: Sample Query, Which i'm extracting 'data' as a dict.
response_dict_instrument = {'topic': 'instrument.BTCUSD', 'data': [{'symbol': 'BTCUSD', 'mark_price': 12367.29, 'index_price': 12360.1}]}
You're formatting query by yourself. You never should do that. Also I would suggest you to create table for every incoming symbol beforehand, do not do this dynamically.
Asyncpg template uses $ sign with number to substitute values to query for you. doc
So, syntax should be like this, if input is dictionary.
async def save_input(input):
# create connection
conn = ...
trade_symbol = input['symbol']
query = "create table if not exists trade_{trade_symbol} ... ".format(trade_symbol=trade_symbol) # your column names go here
await conn.execute(query)
query = "SELECT create_hypertable('trade_{trade_symbol} ...".format(trade_symbol=trade_symbol)
await conn.execute(query)
# i'm not copyng your exact keys, you should do it yourself
values = (input['key1'], input['key2'], input['key3'])
query = "insert into trade_{trade_symbol} (key1, key2, key3) values ($1, $2, $3);".format(trade_symbol=trade_symbol)
await conn.execute(query, *values)
await conn.close()
Related
I receive a json message thru rabbitMQ and I want to store it to the database.
Message looks like this:
message ={
"ticket": "ticket_12334345",
"items" : ["item1","item2","item3"],
"prices" : [10,20,15],
"full price" : [45]
}
Storing to database looks like this:
def callback(ch, method, properties, body):
print("%r" % body)
body = json.loads(body)
conn = sqlite3.connect('pythonDB.db')
c = conn.cursor()
c.execute('CREATE TABLE IF NOT EXISTS Table_3 (ticket TEXT, items TEXT, prices INTEGER,'
'FullPrice INTEGER)')
c.execute("INSERT INTO Table_3 VALUES(?,?,?,?)", (body["ticket"],
body["items"], body["prices"], body["full price"],
))
conn.commit()
I get an error sqlite3.InterfaceError: Error binding parameter 0 - probably unsupported type.
I assume it's because I'm not storing list data correctly. I want in the column all the values from the list in one cell.
Use str() function, like this:
def callback(ch, method, properties, body):
print("%r" % body)
body = json.loads(body)
conn = sqlite3.connect('pythonDB.db')
c = conn.cursor()
c.execute('CREATE TABLE IF NOT EXISTS Table_3 (ticket TEXT, items TEXT, prices INTEGER,'
'FullPrice INTEGER)')
c.execute("INSERT INTO Table_3 VALUES(?,?,?,?)", (body["ticket"],
str(body["items"]), str(body["prices"]), str(body["full price"][0]),
))
conn.commit()
I created one-to-many relationship table and according to the sqlite3 documentation I can't insert value into the child table if the referenced table column value in the parent table does not exist.
import sqlite3
class Database:
def __init__(self, database_name):
self.database_name = database_name
def create_table(self, table_name, *columns):
columns = ", ".join(columns)
conn = sqlite3.connect(self.database_name)
cursor = conn.cursor()
_SQL = f"CREATE TABLE IF NOT EXISTS {table_name}({columns})"
cursor.execute(_SQL)
conn.commit()
cursor.close()
conn.close()
def insert_values(self, table_name, values, *columns):
dynamic_values = ('?, ' * len(columns))[0:-2]
columns = ", ".join(columns)
conn = sqlite3.connect(self.database_name)
cursor = conn.cursor()
_SQL = f"INSERT INTO {table_name}({columns}) VALUES ({dynamic_values})"
cursor.execute(_SQL, values)
conn.commit()
cursor.close()
conn.close()
def view_values(self, table_name, *columns):
columns = ", ".join(columns)
conn = sqlite3.connect(self.database_name)
cursor = conn.cursor()
_SQL = f"SELECT {columns} FROM {table_name}"
cursor.execute(_SQL)
the_data = cursor.fetchall()
cursor.close()
conn.close()
return the_data
data = Database("games.db")
#
# data.create_table("supplier_groups", "group_id integer PRIMARY KEY", "group_name text NOT NULL")
#
data.insert_values("supplier_groups", ("Domestic", ), "group_name")
# data.create_table("suppliers ", "supplier_id INTEGER PRIMARY KEY",
# "supplier_name TEXT NOT NULL",
# "group_id INTEGER NOT NULL, "
# "FOREIGN KEY (group_id) REFERENCES supplier_groups (group_id)")
data.insert_values("suppliers", ('ABC Inc.', 9), "supplier_name", "group_id")
as you see on this line: data.insert_values("supplier_groups", ("Domestic", ), "group_name") - I'm inserting a value into supplier_groups table
and then right here: data.insert_values("suppliers", ('ABC Inc.', 9), "supplier_name", "group_id") - I'm inserting value into suppliers table with the group_id that does not exist in the group_suppliers table. Python executes it successfully and adds value to the database, however when attemping to execute this command in SQLITE browser I get this error:
Execution finished with errors. Result: FOREIGN KEY constraint failed which is what python should also have done instead of adding it into the database.
So, could anyone explain me what's going on here? Do I understand something in the wrong way? Help would be appreciated
From Section 2. Enabling Foreign Key Support in the sqlite doc:
Assuming the library is compiled with foreign key constraints enabled, it must still be enabled by the application at runtime, using the PRAGMA foreign_keys command. For example:
sqlite> PRAGMA foreign_keys = ON;
I am trying to convert my sql query output into a list to look a certain way.
Here is my code:
def get_sf_metadata():
import sqlite3
#Tables I want to be dynamically created
table_names=['AcceptedEventRelation','Asset', 'Book']
#SQLIte Connection
conn = sqlite3.connect('aaa_test.db')
c = conn.cursor()
#select the metadata table records
c.execute("select name, type from sf_field_metadata1 limit 10 ")
print(list(c))
get_sf_metadata()
Here is my output:
[('Id', 'id'), ('RelationId', 'reference'), ('EventId', 'reference')]
Is there any way to make the output looks like this:
[Id id, RelationId reference, EventId reference]
You can try
print(["{} {}".format(i[0], i[1]) for i in list(c)])
That will print you
['Id id', 'RelationId reference', 'EventId reference']
I am trying to send data to PostgreSQL, data is a tuple of strings i.e (time, price).
The problem is when I send data using a simple query (not parametrized) it works fine!
Following is the simple query working perfectly.
cur.execute("INSERT INTO paxos (date,price) VALUES ('2020-04-09 14:39:58.145804', '$1,664.08');");
But as those values aren't fixed, so I want to store them in variable and use a parameterized query for sending data, but the parameterized query isn't working for me. Here is the parameterized query.
cur.execute("INSERT INTO paxos (date, price) values (?, ?)",(time, price))
Here is the complete function I am trying to implement:
def insert_data(time, price):
con = psycopg2.connect(database="", user="", password="", host="", port="5432")
print("Database opened successfully")
cur = con.cursor()
data_tuple = (time, price)
cur.execute("insert into paxos (date, price) values (?, ?)",(time, price))
con.commit()
print("Record inserted successfully")
con.close()
insert_data("2020-04-09 14:39:58.145804", "$1,664.08")
Here is the error message:
It seems like this is a python syntax error. I think you should use the format method of the string object (see codesnippet below).
I can't test it right now, but according to some old code of mine, I always "built" a query string first and then passed the string object to the cursor. Try something like that:
artist = "Aphex Twin"
title = "Windowlicker"
query = '''SELECT EXISTS(SELECT * FROM tracks
WHERE artist ILIKE \'{}\'AND
title ILIKE \'{}\')'''.format(artist, title)
cursor.execute(query)
I made new column with NULL values called 'id' in table. Now I want to add data to it from list. It holds about 130k elements.
I tried with insert, it returned error:
conn = create_connection(xml_db)
cursor = conn.cursor()
with conn:
cursor.execute("ALTER TABLE xml_table ADD COLUMN id integer")
for data in ssetId:
cursor.execute("INSERT INTO xml_table(id) VALUES (?)", (data,))
conn.commit()
I also tried with update:
conn = create_connection(xml_db)
cursor = conn.cursor()
with conn:
cursor.execute("ALTER TABLE xml_table ADD COLUMN id INTEGER")
for data in ssetId:
cursor.execute("UPDATE xml_table SET ('id' = ?)", (data,))
conn.commit()
What is incorrect here ?
EDIT for clarification.
The table was already existing, filled with data. I want to add column 'id' with custom values to it.
Heres an example similar to yours which may be useful.
import sqlite3
conn = sqlite3.connect("xml.db")
cursor = conn.cursor()
with conn:
# for testing purposes, remove this or else the table gets dropped whenever the file is loaded
cursor.execute("drop table if exists xml_table")
# create table with some other field
cursor.execute("create table if not exists xml_table (other_field integer not null)")
for other_data in range(5):
cursor.execute("INSERT INTO xml_table (other_field) VALUES (?)", (other_data,))
# add id field
cursor.execute("ALTER TABLE xml_table ADD COLUMN id integer")
# make sure the table exists
res = cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
print("Table Name: {}".format(res.fetchone()[0]))
# add data to the table
for data in range(5):
cursor.execute("UPDATE xml_table SET id = ? WHERE other_field = ?", (data, data))
# if you must insert an id, you must specify a other_field value as well, since other_field must be not null
cursor.execute("insert into xml_table (id, other_field) VALUES (? ,?)", (100, 105))
# make sure data exists
res = cursor.execute("SELECT id, other_field FROM xml_table")
for id_result in res:
print(id_result)
conn.commit()
conn.close()
As I stated in the comment below, since one of your rows has a NOT NULL constraint on it, no rows can exist in the table that have that column NULL. In the example above other_field is specified NOT NULL, therefore there can be no rows that have NULL values in the column other_field. Any deviation from this would be an IntegrityError.
Output:
Table Name: xml_table
(0, 0)
(1, 1)
(2, 2)
(3, 3)
(4, 4)
(100, 105)