a datebase contained 80 tables ,every table is a 18000x4 matrix ,how can I translate these to a 80x18000x4 numpy array?
the db : https://drive.google.com/open?id=0B3bNSNFik5wGdm1DYnJwNHBJMVU
I wrote a function. Do you have any better idea?
import sqlite3 as sql
import os
def db_to_array(db,r,c):
db = sql.connect(db)
cursor = db.cursor()
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
tables = cursor.fetchall()
wd=os.getcwd()
if not os.path .exists(wd +'/temp/'):
os.makedirs(wd +'/temp/')
for table_name in tables:
table_name = table_name[0]
table = pd.read_sql_query("SELECT * from %s" % table_name, db)
table.to_csv(wd +'/temp/'+ table_name + '.csv', index_label='index')
ss=os.listdir(wd +'/temp/')
print(len(ss))
dd=np.empty([len(ss),r,c])
for i in range(len(ss)):
ddd=np.array(pd.read_csv(wd +'/temp/'+ss[i]))
print(i)
print(ddd.shape)
dd[i,:,:]=ddd[:,0:r]
return dd
Related
The below query makes a result set in the variable 'result'
I need to insert that into the iconndest (the new MySQL server). But I have no idea how to insert the query result into the new table? I just want to do Insert into DB.TBL SELECT * FROM RESULT. But I am not sure how?
import mysql.connector
import pandas as pd
from sqlalchemy import create_engine
import multiprocessing as mp
from multiprocessing import cpu_count
try:
engine_source = create_engine("CONN STRING")
iconn = engine_source.connect()
result = iconn.execute('SELECT QUERY')
print('EXTRACT COMPLETE')
engine_dest = create_engine("CONN STRING")
iconndest = engine_dest.connect()
iconndest.execute('SELECT * from ')
engine_source.dispose()
engine_dest.dispose()
except Exception as e:
print('extract: ' + str(e))
What you describe is very simple if we use .mappings() to convert the list of Row objects to a list of RowMapping objects when we retrieve the results. RowMapping objects behave like dict objects when passed as parameter values:
import sqlalchemy as sa
source_engine = sa.create_engine("mssql+pyodbc://scott:tiger^5HHH#mssql_199")
destination_engine = sa.create_engine("sqlite://")
with source_engine.begin() as conn:
results = (
conn.exec_driver_sql(
"""\
SELECT 1 AS id, N'foo' AS txt
UNION ALL
SELECT 2 AS id, N'bar' AS txt
"""
)
.mappings()
.all()
)
print(results)
# [{'id': 1, 'txt': 'foo'}, {'id': 2, 'txt': 'bar'}]
destination_engine.echo = True
with destination_engine.begin() as conn:
conn.exec_driver_sql("CREATE TABLE t (id int, txt varchar(10))")
conn.execute(
sa.text("INSERT INTO t (id, txt) VALUES (:id, :txt)"), results
)
"""SQL emitted:
INSERT INTO t (id, txt) VALUES (?, ?)
[generated in 0.00038s] ((1, 'foo'), (2, 'bar'))
"""
Python 3.8.10
streamlit==1.9.0
pandas==1.4.2
psycopg2-binary==2.9.3
Loading a Postgres table directly into a Pandas DataFrame with the following code.
df = pd.DataFrame(run_query("SELECT * FROM schema.tablename;"))
Displaying it with either streamlit.dataframe(df) or streamlit.write(df) loses the column names.
In order to capture the column names, I use this kluge.
# Initialize connection.
#st.experimental_singleton
def init_connection():
return psycopg2.connect(**st.secrets["postgresservername"])
conn = init_connection()
# Perform query.
#st.experimental_memo(ttl=600)
def run_query(query):
with conn.cursor() as cur:
cur.execute(query)
return cur.fetchall()
def load_table_as_dataframe(table):
# This is super klugy.
data = run_query("SELECT * FROM schema.{};".format(str(table)))
columns = run_query("SELECT *FROM information_schema.columns WHERE table_schema = 'schema' AND table_name = '{}';".format(str(table)))
# Fish out the actual column names
columns = [c[3] for c in columns]
df = pd. DataFrame(data, columns = columns)
return df
df = load_table_as_dataframe("tablename")
Which works...
Is there a better way to collect the needed data (and columns names) into a Pandas DataFrame within Postgres and Streamlit?
Using...
df = pd.read_sql("SELECT * FROM schema.{};".format(str(table)), conn)
...solved the issue. (Thx #parfait)
I am saving sequences with different ids associated with them as two column in sqlite3 DB where sequence is a column and ids_string are another column. I have problem with retrieving from the database
The dictionary is created as uniqe_sequence = [list of ids]
the sequence is a string of roughly 7000 characters or less and the list of ids could be up to 1 million characters
import sys
from Bio import SeqIO
from Bio.Seq import Seq
import time
import sqlite3
conn = sqlite3.connect('Sausql_012419.db')
c = conn.cursor()
c.execute("create table Sau (sequence text, list_of_ids text)")
for record in sequences_dict_d:
c.execute("insert into Sau values (?,?)", (record,'._/'.join(sequences_dict_d[record])))
conn.commit()
c.execute("SELECT COUNT(*) FROM Saureus_panproteome")
sql_count = c.fetchall()
print("saved sql database of {} proteins in --- {:.3f} seconds ---".format(sql_count[0][0],time.time() - start_time))
c.close()
conn.close()
#retrieval exact sequence match
for record in SeqIO.parse(queryfile, _format):
conn = sqlite3.connect('Sausql_012419.db')
c = conn.cursor()
c.execute('select list_of_ids from Sau where sequence = str(record.seq)')
print(c.fetchall()) # or do something with the list_of_ids
New to using sqlite with python. I am trying to insert two date values into two date columns in sqlite db via Python.
import sqlite3
def create_connection(db_file):
# Create a database connection to a SQLite database
# Param: db_file as str. Return: connection objects or None
try:
conn = sqlite3.connect(db_file)
cur = conn.cursor()
return conn, cur
except Error as e:
print (e)
return None
my_conn, my_cur = create_connection(dpd_sqlite_db_dir)
def create_sqlite_table_if_nonexist(conn, table_name):
sql = 'create table if not exists '+table_name+' (data_download_date datetime, script_executed_date datetime)'
conn.execute(sql)
create_sqlite_table_if_nonexist(my_conn, 'df_timestamp_en')
def insert_timestamp(conn, timestamp):
# execute insert into db
sql = ''' INSERT INTO df_timestamp_en (data_download_date, script_executed_date) VALUES(?,?) '''
cur = conn.cursor()
cur.execute(sql, timestamp)
return cur
timestamp_info = ('2018-10-30', '2018-11-30')
insert_timestamp(my_conn, timestamp_info)
It runs and created the table with two date columns, but it doesn't insert the timestamp_info's values.
Using python, how do i get to delete data based on lastrowid. The code i have deletes all the rows
CODE:
import re
import sys
import difflib
import sqlite3
def main():
while True:
name = input ('Please Type your Question: ').lower().split()
name2 = name[:]
import sqlite3
for item in name2:#break
conn = sqlite3.connect("foods.db")
cursor = conn.cursor()
cursor.execute("INSERT INTO INPUT33 (NAME) VALUES (?);", (name2,))
cursor.execute("select MAX(rowid) from [input33];")
conn.commit()
for rowid in cursor:break
for elem in rowid:
m = elem
print(m)
cursor.execute("DELETE FROM INPUT33 (NAME) WHERE NAME = name")
To get the last inserted rowid, use the cursor's lastrowid attribute.
To delete a record with a specific rowid, use that column in the WHERE condition:
cursor.execute("INSERT INTO input33(Name) VALUES(?)", ("whatever",))
rowid = cursor.lastrowid
cursor.execute("DELETE FROM input33 WHERE rowid = ?", (rowid,))