I am trying to insert data in a "Dummy" table in postgres SQL using psycopg2 and faker library. This is a table that I have created only for learning purpose. It has only one column Student_name which is of type char[]. Below is my Python script
import psycopg2
from faker import Faker
fake = Faker()
conn = psycopg2.connect(database="kreiotdb", user="****", password="*****", host="127.0.0.1", port="5432")
print("Connected Successfuly")
cur = conn.cursor()
for i in range (10):
name = fake.name()
cur.execute(""" INSERT INTO "Dummy" ("Student_name") VALUES (%s);""",[name])
It is giving me the following error when I run the script. The connection is successful
Fri Nov 02 12:16:07 gaurav ~ $ python3 /Users/gaurav/Desktop/populate.py
Connected Successfuly
Traceback (most recent call last):
File "/Users/gaurav/Desktop/populate.py", line 11, in <module>
cur.execute(""" INSERT INTO "Dummy" ("Student_name") VALUES (%s);""",[name])
psycopg2.DataError: malformed array literal: "Brent Allison"
LINE 1: INSERT INTO "Dummy" ("Student_name") VALUES ('Brent Allison...
^
DETAIL: Array value must start with "{" or dimension information.
Why is it giving me this error and should I do ?
Please help.
import os
import csv
import sys
import psycopg2
import json
import csv
#import xlsxwriter
#import configparser
import psycopg2.extras
import psycopg2.extensions
#import logging
#import logging.config
import datetime
import zipfile
from subprocess import call
def db_connect():
dbconn = None
#if conf_section in config == False:
# print("Given section -> {0} is not exists in conf file.".format(conf_section))
# return None
dbhost = ""
dbport = ""
dbname = ""
dbuser = ""
dbpass = ""
try:
dbconn = psycopg2.connect(host=dbhost, port=dbport, dbname=dbname, user=dbuser, password=dbpass)
dbconn.autocommit = True
except Exception as e:
print(e)
return None
finally:
return dbconn
def execute_query(dbconn, query):
nrows = cursor = None
colnames = result = []
try :
cursor = dbconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cursor.execute(query)
except Exception as e:
print(e)
return (0, colnames, result)
nrows = cursor.rowcount
colnames = [desc[0] for desc in cursor.description]
result = cursor.fetchall()
#cursor.close()
return (nrows)
def parse_csv(default_data):
with open('key.csv') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
tid=row['TID']
mid=row ['MID']
key=row ['Exported Key ']
kcv=row ['KCV']
serial_no=row['HarwardSerialNo']
print ("TID="+tid+" MID="+mid+" EXPORTED KEY="+key+" KCV="+kcv)
request_data=default_data+key
request_data.replace(" ", "")
print (request_data)
cmd="/home/siva/HSM_REQ/hsm_comms.out 192.168.5.51 4000"+request_data
response_data=os.system(cmd)
print (response_data)
dbconn = db_connect()
query="select * from hsm_keys where serial_no ='"+serial_no+"'";
rows=execute_query(dbconn, query)
print (rows)
if (rows==0):
query="INSERT ";
print (query)
return()
def main():
header="0101303200"
head_len="1D"
fun_code="EE0200"
fun_mod="00"
key_len="05"
key_spc="081002"
key_index="0004"
key_type="0500"
len_of_key="10"
default_data=header+head_len+fun_code+fun_mod+key_len+key_spc+key_index+key_type+len_of_key
print (default_data)
parse_csv(default_data)
if __name__ == '__main__':
main()
Related
Trying to write a multiprocessing code using the import get pass module
import time
from multiprocessing import Pool
from multiprocessing import freeze_support
import getpass
import jaydebeapi
import pandas as pd
import numpy as np
pw = getpass.getpass(prompt="Password", stream=False)
# establishing database to the ODS database
ODS = jaydebeapi.connect(
'com.ibm.db2.jcc.DB2Driver',
'jdbc:db2://he3qlxvtdbs351.fhlmc.com:50001/DB2QLTY',
['f408195', pw],
'C:/JDBC/db2jcc.jar')
# Allows SQL statements between the ODS database
ODS = ODS.cursor()
# creating the password needed to establish PML database connection
pw_2 = getpass.getpass(prompt="Password", stream=False)
# establishing database to the PML database
PML = jaydebeapi.connect(
'com.ibm.db2.jcc.DB2Driver',
'jdbc:db2://he3qlxvtdbs957.fhlmc.com:50001/PMLFDB2',
['f408195', pw_2],
'C:/JDBC/db2jcc.jar')
# Allows SQL statements between the PML database
PML = PML.cursor()
def test(first_evnt, last_evnt):
PML_loan_Query = "select b.id_lpa_alt_loan from udbadm.pml_lst_cmpltd_trans_mtch a join udbadm.lpa_altv_loan_idtn b on a.id_evnt = b.id_evnt where b.cd_lpa_alt_loan_idtn = 'HewlettPackardGeneratedTransaction' and a.id_evnt BETWEEN ? AND ?"
PML.execute(PML_loan_Query,(first_evnt, last_evnt))
loan_records = PML.fetchall()
df = pd.DataFrame()
for x in loan_records:
# Populating the ODS table
#borr_query = "SELECT nbr_aus, CAST(NULLIF(NULLIF(cd_idx, -9999), 0.000000) AS VARCHAR(100)) AS cd_idx, CAST(rate_curr_int AS INT) AS rate_curr_int, CAST(NULLIF(rate_gr_mrtg_mrgn,0) AS INT) AS rate_gr_mrtg_mrgn, CAST(rate_loln_max_cap AS INT) AS rate_loln_max_cap, CAST(NULLIF(rate_perdc_cap,0) AS INT) AS rate_perdc_cap FROM DB2MANT.I_LP_TRANS WHERE nbr_trans_aus BETWEEN ? AND ?"
borr_query = 'SELECT nbr_aus, CAST(NULLIF(NULLIF(cd_idx, -9999), 0.000000) AS VARCHAR(10)) AS cd_idx, CAST(rate_curr_int AS VARCHAR(10)) AS rate_curr_int, CAST(NULLIF(rate_gr_mrtg_mrgn,0) AS VARCHAR(10)) AS rate_gr_mrtg_mrgn, CAST(rate_loln_max_cap AS VARCHAR(10)) AS rate_loln_max_cap, CAST(NULLIF(rate_perdc_cap,0) AS VARCHAR(10)) AS rate_perdc_cap FROM DB2MANT.I_LP_TRANS WHERE nbr_trans_aus IN (?)'
#borr_query = "SELECT DISTINCT nbr_aus FROM DB2MANT.I_LP_TRANS WHERE nbr_trans_aus BETWEEN ? AND ?"
ODS.execute(borr_query, x)
#ODS.execute(ODS_list)
ODS_records = ODS.fetchall()
ODS_records = df.append(pd.DataFrame(ODS_records, columns = ['nbr_aus', 'cd_idx', 'rate_curr_int', 'rate_gr_mrtg_mrgn', 'rate_loln_max_cap', 'rate_perdc_cap']))
return ODS_records
if __name__ == '__main__':
freeze_support()
first_evnt = 155643917
last_evnt = 155684481
p = Pool()
result = p.map(test, [first_evnt, last_evnt])
print(result)
p.close()
p.join()
I saved this script into a .py file and I tried to run it in the command prompt, it asked for my password for my ODS database, then my PML database, and then it seems to keep running the getpass command over and over again.
Below in a picture for reference.
enter image description here python script terminal
If I have a table on my server and I am producing a connection string to it, how can I, using Vaex, load it to a dataframe?
Here is what I am doing but with Pandas:
from sqlalchemy import types, create_engine, text
import pandas as pd
import pymysql
def connect_to_data(driver='mysql+pymysql://', conn_string=''):
try:
conn = create_engine(driver + conn_string)
print("MySQL Connection Successfull!")
except Exception as err:
print("MySQL Connection Failed!")
print(err)
return conn
# Connect to the db:
conn_string = 'xxxxxxxx'
conn = connect_to_data(conn_string=conn_string)
# Get all requests from the db:
query = '''SELECT * FROM table_name'''
result = conn.execute(text(query))
# Desired dataframe:
df = pd.read_sql_query(query, conn)
How can I do the same with Vaex (because of it's high performance)?
For now at least, you can't do it directly. But vaex can easily read a pandas dataframe so you can
# Following your example..
pandas_df = pd.read_sql_query(query, conn)
df = vaex.from_pandas(pandas_df)
I have a script that opens up an excel worksheet (1.353 kB), updates row based on values and inserts new rows based on an index. Unfortunately, I get a MemoryError. Besides, the script is consuming a lot of RAM. Is this based on the code or this based on the library? Thank you for your support.
The error message is as follows:
Traceback (most recent call last):
File "update_worksheet.py", line 42, in <module>
update.add_row(idx)
File "update_worksheet.py", line 30, in add_row
self.__ws.insert_rows(idx)
File ".\Python\Python38-32\lib\site-packages\openpyxl\worksheet\worksheet.py", line 713, in insert_rows
self._move_cells(min_row=idx, offset=amount, row_or_col="row")
File ".\Python\Python38-32\lib\site-packages\openpyxl\worksheet\worksheet.py", line 706, in _move_cells
self._move_cell(row, column, row_offset, col_offset)
File ".\Programs\Python\Python38-32\lib\site-packages\openpyxl\worksheet\worksheet.py", line 802, in _move_cell
self._cells[new_row, new_col] = cell
MemoryError
from openpyxl import load_workbook
import pandas as pd
import time
class UpdateWorksheet():
def __init__(self,path):
self.__wb = load_workbook(path, read_only = False)
self.__ws = self.__wb["WP_Planning"]
self.__df = pd.DataFrame(self.__ws.values)
def search_value_in_col_idx(self, search_string, col_idx=1):
try:
idx = self.__df[self.__df[col_idx]==search_string].index.item()
return idx
except ValueError:
return None
def update_status(self,row_idx, col_idx, status):
self.__ws.cell(row = row_idx, column = col_idx).value = status
def find_last_row(self, key, status):
idx = None
try:
idx = self.__df[self.__df[1]=="HMI"].index.tolist()
except ValueError:
idx = None
return idx[-1]
def add_row(self, idx):
self.__ws.insert_rows(idx)
def save_workbook(self,path):
self.__wb.save(path)
print("finish")
if __name__ == "__main__":
path = './data/TrackingSheet.xlsx'
update = UpdateWorksheet(path)
index = update.search_value_in_col_idx("PFIVEDAIEX-4244",4)
update.update_status(index,7,"Mikey")
idx = update.find_last_row("PFIVEDAIEX-4244","Fixed")
update.add_row(idx)
Edit1:
from openpyxl import load_workbook
from openpyxl import Workbook
import pandas as pd
class UpdateWorksheet():
def __init__(self,path):
self.__path = path
def load_file(self, read_only):
wb = load_workbook(self.__path, read_only)
return wb
# Search line to be updated
def search_value_in_col_idx(self, search_string, col_idx, ws):
df = pd.DataFrame(ws.values)
try:
idx = df[df[col_idx]==search_string].index.item()
return idx
except ValueError:
return None
# Update line attributes
def update_status(self,row_idx, col_idx, status, ws):
if not row_idx is None:
ws.cell(row = row_idx, column = col_idx).value = status
# Find last row for new rows
def find_last_row(self,ws):
df = pd.DataFrame(ws.values)
idx = None
try:
idx = df[df[1]=="HMI"].index.tolist()
except ValueError:
idx = None
return idx[-1]
# Add new rows
def add_row(self,idx,ws):
if idx is not None:
ws.insert_rows(idx, 5)
# Save changes made to workbook
def save_workbook(self,wb):
wb.save(self.__path)
if __name__ == "__main__":
path = './data/TrackingSheet.xlsx'
update = UpdateWorksheet(path)
wb = update.load_file(read_only=True)
idx = update.search_value_in_col_idx("PFIVEDAIEX-4244",4,wb["WP_Planning"])
last_row = update.find_last_row(wb["WP_Planning"])
wb.close()
wb = update.load_file(read_only=False)
update.update_status(idx,7,"Mikey",wb["WP_Planning"])
update.add_row(200,wb["WP_Planning"])
pdate.save_workbook(wb)
wb.close()
Below is my directory structure:
/home
|___ /sub_directory
|__abc.py
|__ xyz.py
Below is my xyz.py code:
from flask import Flask, request, redirect, url_for,send_from_directory, jsonify, render_template
import mysql.connector
from mysql.connector import Error
app = Flask(__name__)
try:
connection = mysql.connector.connect(host='127.0.0.1',database='test',user='root',password='')
if connection.is_connected():
db_Info = connection.get_server_info()
cursor = connection.cursor()
cursor.execute("select id,name from skill_category;")
record = cursor.fetchall()
out = [item for t in record for item in t]
except Error as e:
print("Error while connecting to MySQL",e)
#app.route('/', methods=['GET'])
def dropdown():
val = record
return render_template('Rankcv.html', val = val)
#app.route('/get-subskills', methods=['POST'])
def get_subskills():
skills = request.form['select_skills']
cursor.execute("SELECT skill_items.name FROM skill_items WHERE skill_items.category_id = " + skills + " ;")
record = cursor.fetchall()
out = [item for t in record for item in t]
...........
...........
...........
return jsonify(something)
if __name__ == "__main__":
app.run(debug=True)
Now I have to use the value of variable out and skills in abc.py.
I tried importing xyz directly and tried to retrieve the value using function name (get_subskills), but it didnt work. Can someone please explain how to solve this?
Import the abc function into xyz.
I want to write a python script from which I can execute multiple sql queries and the output of that query is saved in excel.
Suppose I have 4 sql query i.e Script1, Script2, Script3 & Script4 and I want to save the generated excel workbook in E:\Test, In that workbook sheet1 contains Script1 output, sheet2 contains Script2 output, sheet3 contains Script3 output, and so on. I have written a query but its working for only one script.
By using this script I am able to generate excel sheet with Test name, but How I run the remaining script so that their output will show in other sheet of same workbook
Please Help
import psycopg2
import sys
import pprint
import pandas as pd
import os
import openpyxl.cell
COMMASPACE = ', '
def main():
conn_string = "dbname='abc' user='qwerty' host='pqr' password='******' port='1234'"
script1 = """
select * From something1
"""
script2 = """
select * From something2
"""
script3 = """
select * From something3
"""
script4 = """
select * From something4
"""
pprint.pprint ('Making connection to the Database...')
con1 = psycopg2.connect(conn_string)
cur = con1.cursor()
pprint.pprint ('Execution Start')
cur.execute(script)
if not cur.rowcount:
pprint.pprint ('Oops! Error Occured')
else:
columns = [desc[0] for desc in cur.description]
data = cur.fetchall()
df = pd.DataFrame(list(data), columns=columns)
df.columns = map(str.upper, df.columns)
writer = pd.ExcelWriter('E:\\Test.xlsx')
df.to_excel(writer, sheet_name='Sheet1')
def hide_column(ws, column_id):
if isinstance(column_id, int):
assert column_id >= 1, "Column numbers must be 1 or greater"
column_id = openpyxl.cell.get_column_letter(column_id)
column_dimension = ws.column_dimensions[column_id]
column_dimension.hidden = True
writer.save()
print ("END of extraction")
if __name__ == "__main__":
main()
try using pandas read_sql with Sql Alchemy.
from openpyxl import load_workbook
from sqlalchemy import create_engine
import pandas as pd
# Parameters for SQL Alchemy
ServerName = "your_Server_Name"
Database = "Your_Database"
Driver = "Your_Driver"
# Create the connection
engine = create_engine('mssql+pyodbc://' + ServerName + '/' + Database + "?" + Driver)
# reading in the dataframes
df1 = pd.read_sql_query("select * from somewhere", engine)
df2 = pd.read_sql_query("select * from somewhere_else", engine)
# Using openpyxl to write to excel sheets
file = 'Your_file_path_Here'
book = load_workbook(file)
writer = pd.ExcelWriter(file, engine='openpyxl')
writer.book = book
# now start writing them to sheets
df1.to_excel(writer, index=None, sheet_name='SQL1')
df1.to_excel(writer, index=None, sheet_name='SQL2')