SQLAlchemy Core. Values method does not limit columns in insert statement - python-3.x

SQLALchemy Core insert expressions documentation says:
Notice above that the INSERT statement names every column in the users table. This can be limited by using the values() method, which establishes the VALUES clause of the INSERT explicitly:
With that in mind, I wrote the following snippet, which returns unexpected results.
from datetime import datetime
import sqlalchemy
from sqlalchemy import types
from sqlalchemy.dialects import postgresql
metadata = sqlalchemy.MetaData()
users = sqlalchemy.Table(
"users",
metadata,
sqlalchemy.Column(
"id",
postgresql.UUID(as_uuid=True),
default=uuid.uuid4(),
primary_key=True,
),
sqlalchemy.Column("email", types.String, unique=True, index=True),
sqlalchemy.Column(
"created_at",
types.TIMESTAMP(timezone=True),
default=datetime.utcnow(),
),
sqlalchemy.Column(
"updated_at",
types.TIMESTAMP(timezone=True),
default=datetime.utcnow(),
onupdate=datetime.utcnow(),
),
)
email = "god#olympus.org"
query = users.insert().values(email=email)
# (Pdb) print(query)
# INSERT INTO users (id, email, created_at, updated_at) VALUES (:id, :email, :created_at, :updated_at)
#
# (Pdb) print(query.compile().params)
# {'id': None, 'email': 'god#olympus.org', 'created_at': None, 'updated_at': None}
I expected the query to be INSERT INTO users (email) VALUES (:email)
Is there anything I'm missing?
I'm using SQLAlchemy==1.3.20 by the way.

The issue does not relate to SQLAlchemy, but rather to encode/databases.
Support for "default" parameter in sqlalchemy.Column

Related

Django rest framework cast foreign mysql foreign table column

please help me to convert cast foreign table(customers) first name
class LoandetailListSearch(generics.ListAPIView):
serializer_class = LoandetailSerializer
def get_queryset(self):
"""
Optionally restricts the returned loan details to a search list,
by filtering against a `fields` in query parameter in the URL.
"""
queryset = Loandetails.objects.all().exclude(broker_name__isnull=True).\
extra(
{
'staff': "CONVERT(CAST(CONVERT(CONCAT(staff ) using latin1) as binary) using utf8)",
'customer__first_name': 'SELECT CONVERT(CAST(CONVERT(CONCAT(first_name ) using latin1) as binary) using utf8) as first_name FROM customers WHERE customers.id = loan_details.customer_id'
}).\
extra(select={'customer__first_name': 'SELECT CONVERT(CAST(CONVERT(CONCAT(first_name ) using latin1) as binary) using utf8) as first_name FROM customers WHERE customers.id = loan_details.customer_id' })
return queryset
I tried to convert and cast both methods first_name, But no luck. Thanks

Insert statement returns A value is required for bind parameter <x>

I'm trying to insert a row from a CSV file into a Postgres database that lives on Heroku website, the code access the database, opens CSV file and access tha data, but also returns a InvalidRequestError.
The code fetches the data from the CSV file correctly, I don't know what I am doing wrong.
the data format is in CSV file:
1416949658,The Dark Is Rising,Susan Cooper,1973
Table structure:
CREATE TABLE books(
id SERIAL PRIMARY KEY,
isbn VARCHAR NOT NULL,
title VARCHAR NOT NULL,
author_name VARCHAR NOT NULL,
publish_year INTEGER NOT NULL
);
for isbn, title, author_name, publish_year in reader:
db.execute("INSERT INTO books (isbn, title, author_name, publish_year) VALUES (:isbn, :title, :author_name, :publish_year)",\
{isbn: isbn, title: title, author_name: author_name, publish_year: publish_year})
print(f"Added book {title} written by {author_name} with isbn {isbn} on year {publish_year}")
db.commit()
sqlalchemy.exc.StatementError: (sqlalchemy.exc.InvalidRequestError) A
value is required for bind parameter 'isbn' [SQL: INSERT INTO books
(isbn, title, author_name, publish_year) VALUES (%(isbn)s, %(title)s,
%(author_name)s, %(publish
_year)s);] [parameters: [{'1416949658': '1416949658', 'The Dark Is Rising': 'The Dark Is Rising', 'Susan Cooper': 'Susan Cooper', '
1973': '1973'}]]
As you see it fetches the data but returns an error still.
To import the data from csv run the following snippet
import csv
import os
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
engine = create_engine(os.getenv("DATABASE_URL"))
db = scoped_session(sessionmaker(bind=engine))
def main():
f = open("books.csv")
reader = csv.reader(f)
for isbn, title, author, year in reader:
db.execute("INSERT INTO books (isbn, title, author, year) VALUES (:isbn, :title, :author, :year)",
{"isbn": isbn, "title": title, "author": author, "year": year})
print(
f"Added book of ISBN {isbn} having Title: {title} written by {author} in the year {year}.")
db.commit()
if __name__ == "__main__":
main()
This should solve your problem. You are missing the {"key": value} format that is needed to be supplied for a dictionary in python.

Unable to create table in Amazon redshift using Psycopg2

I am trying to make a simple script in python, which will fetch data from an endpoint convert it into a dataframe and write it to an Amazon redshift cluster and then automate the script using a cronjob from aws. I am using psycopg2 for connecting to the redshift cluster and the script executes the commands pretty well (creates table in redshift and writes the data as well). But when I try to see the table from a sql client the table doesnt show up
from pandas.io.json import json_normalize
import json
import pandas as pd
import requests
import psycopg2
mm_get = requests.get('endpoint link'})
mm_json=mm_get.json()
data_1 = json_normalize(data = mm_json['data'],
record_path = ['courses','modules'],
record_prefix = 'courses.modules.',
meta = [['courses', 'id'],
['courses', 'title'],
'activated',
'createdAt',
'email',
'employeeId',
'firstName',
'group',
'id',
'lastName',
'phone',
'teams'
]
)
data_2 = json_normalize(data = mm_json['data'],
record_path = 'lessons',
record_prefix = 'lessons.',
meta = 'id',
meta_prefix = 'user.'
)
data_3 = data_1.merge(
data_2,
how = 'outer',
left_on = ['courses.modules.id', 'id'],
right_on = ['lessons.moduleId', 'user.id']
)
cols = data_3.columns
cols = cols.tolist()
cols = pd.DataFrame(cols)
re_cols = pd.DataFrame(cols.loc[:,0].str.replace('.','_').tolist(),index=cols.index)
data_3.teams = data_3.teams.astype(str)
data_3.teams = data_3.teams.str.replace('[','')
data_3.teams = data_3.teams.str.replace(']','')
data_3.teams = data_3.teams.str.replace("'","")
con=psycopg2.connect(dbname='name',
host='hostname',
port='xxxx',user='username',password='password')
cur = con.cursor()
cur.execute('create table testing_learn.test (courses_modules_completionDate DATE, courses_modules_id int, courses_modules_status TEXT,courses_modules_title TEXT, courses_id int,courses_title TEXT, activated bool, createdAt TIMESTAMP, email TEXT, employeeId TEXT, firstName TEXT, group_name TEXT, id TEXT, lastname TEXT, phone int8, teams TEXT, lessons_courseId int, lessons_date DATE, lessons_id int, lessons_lessonNumber int, lessons_moduleId int,lessons_score TEXT, lessons_title TEXT,user_id int);')
cur.close()
data_mat = data_3.as_matrix()
str_mat = b','.join(cur.mogrify('(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',x) for x in tuple(map(tuple,data_mat)))
cur = con.cursor()
cur.execute('insert into testing_learn.test VALUES '+str_mat.decode('utf-8'))
I am able to see the data when I query the same table from python using psycopg2, but the same table doesnt show up. It would be of great help if anyone could help with what I am doing wrong here. Thank in advance.
According to Psycopg2-2.7.5 official documentation, the main entry points of Psycopg2 includes:
The class connection encapsulates a database session. It allows to:
create new cursor instances using the cursor() method to execute database commands and queries,
terminate transactions using the methods commit() or rollback().
Therefore, you need to call con.commit() every time after you call cur.execute() to make the changes to the database persistent. Otherwise your table won't show up in the database.

How to query jsonb array with sqlalchemy

I have data stored in a jsonb field like so:
class Test(Base):
__tablename__ = 'test'
id = Column(Integer, primary_key=True)
data = Column(JSONB)
In the data column there is a json of the form:
{depth: [0.0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06]}
I want to determine the max depth for each record and came up with the following query in raw SQL which does the job:
SELECT test.id, test.name,
(SELECT max(elem::float)
FROM jsonb_array_elements_text(test.data -> 'depth') As elem
) AS maxdepth
FROM test
ORDER BY maxdepth DESC
As I am using SQLAlchemy ORM in my application I want to write this query with SQLAlchemy ORM, but I cannot come up with the proper form.
I was thinking I need something like this:
subq = session.query(
func.max().label('maxdepth')).\
select_from(func.jsonb_array_elements(Test.data['depth'])).\
subquery()
stmnt = session.query(
Test.id, subq.c.maxdepth).\
order_by(subq.c.maxdepth)
But this obviously doesn't work as I don't know how to query from the fields extracted by jsonb_array_elements
[NOTE: AS OF SQLAlchemy 1.0, Oct. 26 2015. This may change in a future release] these special PG syntaxes are not built in to SQLAlchemy right now, please see the recipe at https://bitbucket.org/zzzeek/sqlalchemy/issues/3566/figure-out-how-to-support-all-of-pgs#comment-22842678 which illustrates your query.

Pycassa and Cassandra: doing a select based on columns only

I'm new to both technologies and I'm trying to do the following:
select * from mytable where column = "col1" or column="col2"
So far, the documentation says I should use the get method by using:
family.get('rowid')
But I do not have the row ID. How would I run the above query?
Thanks
In general I think you're mixing two ideas. The query you've written is in CQL, and Pycassa doesn't support CQL (at least to my knowledge).
However, in general regardless of used query interface, if you don't know the row key, you will have to create Secondary Indexes on the queried columns.
You can do just that in Pycassa, consider following code fragment:
from pycassa.columnfamily import ColumnFamily
from pycassa.pool import ConnectionPool
from pycassa.index import *
from pycassa.system_manager import *
sys = SystemManager('192.168.56.110:9160')
try:
sys.drop_keyspace('TestKeySpace')
except:
pass
sys.create_keyspace('TestKeySpace', SIMPLE_STRATEGY, {'replication_factor': '1'})
sys.create_column_family('TestKeySpace', 'mycolumnfamily')
sys.alter_column('TestKeySpace', 'mycolumnfamily', 'column1', LONG_TYPE)
sys.alter_column('TestKeySpace', 'mycolumnfamily', 'column2', LONG_TYPE)
sys.create_index('TestKeySpace', 'mycolumnfamily', 'column1', value_type=LONG_TYPE, index_name='column1_index')
sys.create_index('TestKeySpace', 'mycolumnfamily', 'column2', value_type=LONG_TYPE, index_name='column2_index')
pool = ConnectionPool('TestKeySpace')
col_fam = ColumnFamily(pool, 'mycolumnfamily')
col_fam.insert('row_key0', {'column1': 10, 'column2': 20})
col_fam.insert('row_key1', {'column1': 20, 'column2': 20})
col_fam.insert('row_key2', {'column1': 30, 'column2': 20})
col_fam.insert('row_key3', {'column1': 10, 'column2': 20})
# OrderedDict([('column1', 10), ('column2', 20)])
print col_fam.get('row_key0')
## Find using index: http://pycassa.github.io/pycassa/api/pycassa/
column1_expr = create_index_expression('column1', 10)
column2_expr = create_index_expression('column2', 20)
clause = create_index_clause([column1_expr, column2_expr], count=20)
for key, columns in col_fam.get_indexed_slices(clause):
print "Key => %s, column1 = %d, column2 = %d" % (key, columns['column1'], columns['column2'])
sys.close
However maybe you can think if it's possible to design your data in a way that you can use row keys to query your data.

Resources