But I need to extract data from multiple tables in one running of the code
Here is the code which I wrote:
import matplotlib.pyplot as plt
import pyodbc
import pandas as pd
database_file = input("Enter the path to the database file: ")
conn = pyodbc.connect(
r'DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};'
r'DBQ=' + database_file + ';'
)
while True:
print("MENU:\n 1.Query for checking the number of masterCodes in each WBID\n 2.Plot with this data")
num1 = int(input("Enter your response: "))
if num1 == 1:
table_name = input("Enter the name of the table: ")
column_to_count = input("Enter the name of the column to count the variables in: ")
condition_column = input("Enter the name of the column to use as a condition: ")
condition_value = input("Enter the value to use as a condition: ")
pd.options.display.max_rows = None
#The problem in the line below, I have to get access not only one table, but 4. How I can do that?
query = "SELECT " + column_to_count + ", " + condition_column + " FROM "+ table_name+ " WHERE " + condition_column + " = '" + condition_value + "'"
df = pd.read_sql(query, conn)
if num1 == 2:
print("Going to next steps")
break
# Count the number of occurrences of each variable in the column
counts = df[column_to_count].value_counts()
print(counts)
#query = "SELECT year, result,wbid FROM rawData1 WHERE (wbid = '1573A' and masterCode = 'DO')"
while True:
print("MENU:\n 1.QUERY\n 2.EXIT")
num = int(input("Enter your response: "))
if num == 1:
row = input("Enter row : ")
column = input("Enter column : ")
condition = input("Enter condition : ")
query = "SELECT " + row + ", " + column + " FROM rawData1 Where (" + condition + ")"
#query = input("Enter your query: ")
if num == 2:
print("Program is completed")
break
print(query)
df = pd.read_sql(query, conn)
print(df)
df = df.sort_values(by='year')
x = df['year']
y = df['result']
plt.plot(x, y)
plt.xlabel(row)
plt.ylabel(column)
plt.title("Plot Title")
plt.show()
The problem in the line below, I have to get access not only one table, but 4. How I can do that?
query = "SELECT " + column_to_count + ", " + condition_column + " FROM "+ table_name+ " WHERE " + condition_column + " = '" + condition_value + "'"
table_name give me access only to one table in MS Access.
So I am new to SQL lite and Python in general...
I'm creating a search function that should return an else statement when the input receives an empty query. Not understanding while I fufill the else statement the whole database is returned instead of the print statement I called.
def search_task():
askjeeves = input("Enter Search Term: ")
cur = con.cursor()
if askjeeves != "" or askjeeves != " ":
cur.execute("SELECT * FROM books WHERE author LIKE'%'||?||'%' ", (askjeeves,))
finalsearch = cur.fetchall()
print(finalsearch)
search_task()
else:
print("No value inputted, program ended. ")
Also tried:
def search_task():
askjeeves = input("Enter Search Term: ")
cur = con.cursor()
if askjeeves != None:
cur.execute("SELECT * FROM books WHERE author LIKE'%'||?||'%' ", (askjeeves,))
finalsearch = cur.fetchall()
print(finalsearch)
search_task()
else:
print("No search input")
input("Do you want to search again? Input 1 for Yes, 0 for No:")
if input == 1:
search_task()
else:
print("Search Ended")
You have a circular dependency here. You are calling the function again in the if.
When it is empty, you have a problem with your checks.
Empty strings in python are False. So
def search_task():
askjeeves = input("Enter Search Term: ")
cur = con.cursor()
if askjeeves and askjeeves != " ":
cur.execute("SELECT * FROM books WHERE author LIKE'%'||?||'%' ", (askjeeves,))
finalsearch = cur.fetchall()
print(finalsearch)
search_task()
else:
print("No search input")
input("Do you want to search again? Input 1 for Yes, 0 for No:")
if input == 1:
search_task()
else:
print("Search Ended")
I am writing a Azure python function which read emails and write them to a Azure SQL Database. The code runs fine untill it handles a email with a picture in the body. Then the program stop and return the message of :
Starting worker process:py "C:\ProgramData\chocolatey\lib\azure-functions-core-tools-3\tools\workers\python\3.8/WINDOWS/X64/worker.py" --host 127.0.0.1 --port 65101 --workerId 773c7b91-5c39-4d22-8509-2baea30124bb --requestId 0260fedc-f842-4dba-a2e9-6c7630f7228a --grpcMaxMessageLength 2147483647
Than the Azure python function hangs on the type of email and stops working. I try to build Try and Exception with continue to ensures it pickups the next email aswell a check on the length of the email_body. But none of these workarounds is working.
My question is how can i solve / skip this problem so it can also handle / recognize email which has a picture in the email body.
ReadEmail.py
import logging
import email
import imaplib
import pyodbc
from datetime import datetime
from raFunctions import uitvullenDatum
# def readEmail(EMAIL_ADRES,EMAIL_PASSWORD,EMAIL_SERVER,DB_SERVER,DB_DATABASE,DB_USER,DB_PASSWORD):
logging.info('ReadEmail start')
# logging.info('ReadEmail pars:' + EMAIL_ADRES + ' ' + EMAIL_PASSWORD + ' ' + EMAIL_SERVER + ' ' + DB_SERVER + ' ' + DB_DATABASE + ' ' + DB_USER + ' ' + DB_PASSWORD )
#Configurations
EMAIL = 'xxxxxxx'
PASSWORD = 'xxxxxx'
SERVER = 'xxxxxx'
server = 'xxxxxx'
database = 'xxxxx'
username = 'xxxx'
password = 'xxxxx'
driver= '{ODBC Driver 17 for SQL Server}'
# Connection settings to Database and Email Server
mail = imaplib.IMAP4_SSL(SERVER)
mail.login(EMAIL, PASSWORD)
# cnxn = pyodbc.connect('DRIVER='+driver+';SERVER='+server+';PORT=1433;DATABASE='+database+';UID='+username+';PWD='+ password +';Authentication=ActiveDirectoryPassword')
cnxn = pyodbc.connect('DRIVER='+driver+';SERVER='+server+';PORT=1433;DATABASE='+database+';UID='+username+';PWD='+ password +'; autocommit=True' )
cursor = cnxn.cursor()
cursor_max_mail_date = cnxn.cursor()
max_mail_date = ""
try:
_sql_max_datum = "select format(max(o.ReceivingDate),'dd-MMM-yyyy') from wp_ra.ontvangenmail o "
cursor_max_mail_date.execute(_sql_max_datum)
rows = cursor_max_mail_date.fetchall()
for row in rows:
datum = str(row[0])
max_mail_date = datum
except Exception as e:
print('Max_Mail_Date Error:')
logging.error(str(e))
try:
mail.select('inbox')
# status, data = mail.search(None, 'ALL')
logging.info(max_mail_date)
status, data = mail.search(None, '(SINCE "19-May-2020")')
mail_ids = []
for block in data:
try:
mail_ids += block.split()
except:
print('Error in block in data')
continue
for i in mail_ids:
try:
status, data = mail.fetch(i, '(RFC822)')
for response_part in data:
if isinstance(response_part, tuple):
try:
message = email.message_from_bytes(response_part[1])
mail_from = message['From']
mail_subject = message['Subject']
mail_date = message['Date']
print('mail_date: ' + mail_date + 'mail_subject: ' + mail_subject)
if message.is_multipart():
mail_content = ''
for part in message.get_payload():
if part.get_content_type() == 'text/plain':
mail_content += part.get_payload()
else:
mail_content = part.get_payload()
else:
mail_content = message.get_payload()
maildate = email.utils.parsedate(mail_date)
maildate = uitvullenDatum(str(maildate[0]),str(maildate[1]),str(maildate[2]),str(maildate[3]),str(maildate[4]))
print("Check Mail Content")
if(len(mail_content) >= 1147483647 ):
print("Niet in te lezen email: maildate: " + str(mail_date) + ' mail subject: ' + mail_subject )
else:
values = (maildate, mail_from, mail_subject, mail_content)
sql = "EXEC wp_ra.invoerenmail ?, ?, ?, ?"
cursor.execute(sql, (values))
cursor.commit()
logging.info("Data saved on the database")
except Exception as e:
print('Error reading mail:')
print(str(e))
continue
except:
print('Error i in mail_ids')
continue
except:
print("Fout bij het inlezen van de mail!")
continue
If you stop in some some email, you can try to use continue after output current exception to skip it to deal with other emails:
for i in s :
#Some logic here
try:
#Do something here
except:
print('print something in this place.')
continue
My project is to compare file names located on a folder with names saved in a PostgreSQL database.
I looked into different responses on the SO but nothing worked for me.
Please see below my comments within the source code.
try
cur = db_conn().cursor()
report query = "SELECT ev_report FROM stc_events_master WHERE
si_station_num = %s AND" +
" ev_ins_date BETWEEN to_date(" + "'" + "1/1/2018" + "'" + ",'mm/dd/yyyy')" + "
and TO_DATE('12/31/2018', 'mm/dd/yyyy')"
# files_dict content-> k = '123p34" , value = "DV03S_120124_5-7-2018.pdf"
for k, v in files_dict.items():
cur.execute(report_query, (k,))
# the reports are unique in the database table
query_result = cur.fetchmany(1) #incoming value:DV03S_120124_5-7-2018.pdf
# query returns:
sta = str(query_result[0])
# if a file is not found in the database, the exception happens
# and the code below is not executed. It jumps
# directly to Exception
if len(sta) < 1:
print("no report found in the database")
print("query_results" + sta)
else:
if v.lower() == q_ry.lower():
print("match " + v)
else:
print("no match " + v)
except Exception:
pass
print("In Exception area")
finally:
cur.close()
I want to continue the loop even if nothing is found the database and save a name of the file from the folder which caused the exception. Now, the code stops after the exception.
Thank you for your help.
for k, v in files_dict.items():
cur.execute(report_query, (k,v))
if cur.rowcount == 0:
print("The file is not in the database: " + v)
else:
query_result = cur.fetchmany(3)
sta = str(query_result[0])
file_name_length = len(sta)
q_ry = sta[2: file_name_length-3]
if v.lower() == q_ry.lower():
pass
else:
print("no match after comparing names" + v)
except (Exception, psycopg2.Error) as error:
print("Error fetching data from PG table " + k, error)
finally:
cur.close()
Tried to do a script that will pull all the clients on my meraki organization. An error occurred I think the file that I want to iterate cant be read by my code.
Thanks,
I tried changing the shard to specific tried installing modules that i think required but still same error
import sys, getopt, requests, json, time, datetime, os, sqlite3
#SECTION: GLOBAL VARIABLES: MODIFY TO CHANGE SCRIPT BEHAVIOUR
API_EXEC_DELAY = 0.21 #Used in merakirequestthrottler() to avoid hitting dashboard API max request rate
#connect and read timeouts for the Requests module in seconds
REQUESTS_CONNECT_TIMEOUT = 90
REQUESTS_READ_TIMEOUT = 90
#SECTION: GLOBAL VARIABLES AND CLASSES: DO NOT MODIFY
LAST_MERAKI_REQUEST = datetime.datetime.now() #used by merakirequestthrottler()
ARG_APIKEY = '' #DO NOT STATICALLY SET YOUR API KEY HERE
ARG_ORGNAME = '' #DO NOT STATICALLY SET YOUR ORGANIZATION NAME HERE
ORG_LIST = None #list of organizations, networks and MRs the used API key has access to
DEVICE_DB = None #SQLite3 database of all network devices
MAX_CLIENT_TIMESPAN = 2592000 #maximum timespan GET clients Dashboard API call supports
class c_Net:
def __init__(self):
id = ''
name = ''
shard = 'n132.meraki.com'
devices = []
class c_Organization:
def __init__(self):
id = ''
name = ''
shard = 'n132.meraki.com'
nets = []
#SECTION: General use functions
def merakirequestthrottler():
#makes sure there is enough time between API requests to Dashboard not to hit shaper
global LAST_MERAKI_REQUEST
if (datetime.datetime.now()-LAST_MERAKI_REQUEST).total_seconds() < (API_EXEC_DELAY):
time.sleep(API_EXEC_DELAY)
LAST_MERAKI_REQUEST = datetime.datetime.now()
return
def printhelp():
print(readMe)
#SECTION: Meraki Dashboard API communication functions
def getInventory(p_org):
#returns a list of all networks in an organization
merakirequestthrottler()
try:
r = requests.get('https://%s/api/v0/organizations/%s/inventory' % (p_org.shard, p_org.id), headers={'X-Cisco-Meraki-API-Key': ARG_APIKEY, 'Content-Type': 'application/json'}, timeout=(REQUESTS_CONNECT_TIMEOUT, REQUESTS_READ_TIMEOUT) )
except:
print('ERROR 01: Unable to contact Meraki cloud')
return(None)
if r.status_code != requests.codes.ok:
return(None)
return(r.json())
def getNetworks(p_org):
#returns a list of all networks in an organization
merakirequestthrottler()
try:
r = requests.get('https://%s/api/v0/organizations/%s/networks' % (p_org.shard, p_org.id), headers={'X-Cisco-Meraki-API-Key': ARG_APIKEY, 'Content-Type': 'application/json'}, timeout=(REQUESTS_CONNECT_TIMEOUT, REQUESTS_READ_TIMEOUT) )
except:
print('ERROR 07: Unable to contact Meraki cloud')
return(None)
if r.status_code != requests.codes.ok:
return(None)
return(r.json())
def getOrgs():
#returns the organizations' list for a specified admin, with filters applied
merakirequestthrottler()
try:
r = requests.get('https://n132.meraki.com/api/v0/organizations', headers={'X-Cisco-Meraki-API-Key': ARG_APIKEY, 'Content-Type': 'application/json'}, timeout=(REQUESTS_CONNECT_TIMEOUT, REQUESTS_READ_TIMEOUT) )
except:
print('ERROR 02: Unable to contact Meraki cloud')
return(None)
if r.status_code != requests.codes.ok:
return(None)
rjson = r.json()
orglist = []
listlen = -1
if ARG_ORGNAME.lower() == '/all':
for org in rjson:
orglist.append(c_Organization())
listlen += 1
orglist[listlen].id = org['id']
orglist[listlen].name = org['name']
else:
for org in rjson:
if org['name'] == ARG_ORGNAME:
orglist.append(c_Organization())
listlen += 1
orglist[listlen].id = org['id']
orglist[listlen].name = org['name']
return(orglist)
def getShardHost(p_org):
#Looks up shard URL for a specific org. Use this URL instead of 'api.meraki.com'
# when making API calls with API accounts that can access multiple orgs.
#On failure returns None
merakirequestthrottler()
try:
r = requests.get('https://n132.meraki.com/api/v0/organizations/%s/snmp' % p_org.id, headers={'X-Cisco-Meraki-API-Key': ARG_APIKEY, 'Content-Type': 'application/json'}, timeout=(REQUESTS_CONNECT_TIMEOUT, REQUESTS_READ_TIMEOUT) )
except:
print('ERROR 03: Unable to contact Meraki cloud')
return None
if r.status_code != requests.codes.ok:
return None
rjson = r.json()
return(rjson['hostname'])
def refreshOrgList():
global ORG_LIST
global DEVICE_DB
print('INFO: Starting org list refresh at %s...' % datetime.datetime.now())
flag_firstorg = True
orglist = getOrgs()
if not orglist is None:
for org in orglist:
print('INFO: Processing org "%s"' % org.name)
org.shard = 'n132.meraki.com'
orgshard = getShardHost(org)
if not orgshard is None:
org.shard = orgshard
netlist = getNetworks(org)
devlist = getInventory(org)
if not devlist is None and not netlist is None:
DEVICE_DB = sqlite3.connect(':memory:')
dbcursor = DEVICE_DB.cursor()
dbcursor.execute('''CREATE TABLE devices (serial text, name text, networkId text, mac text, type text, model text)''')
dbcursor.execute('''CREATE TABLE ouis (oui text)''')
DEVICE_DB.commit()
for device in devlist:
if not device['networkId'] is None:
devType = 'merakiDevice'
if device['model'][:2] in ['MR','MS','MX','Z1','Z3']:
devType = 'merakiNetworkDevice'
dbcursor.execute('''INSERT INTO devices VALUES (?,?,?,?,?,?)''', (device['serial'],device['name'],device['networkId'],device['mac'],devType,device['model']))
dbcursor.execute('''INSERT INTO ouis VALUES (?)''', (device['mac'][:8],))
DEVICE_DB.commit()
flag_firstnet = True
for net in netlist:
if net['type'] != 'systems manager': #ignore systems manager nets
dbcursor.execute('''SELECT serial, name, model FROM devices WHERE networkId = ? AND type = ?''', (net['id'],'merakiNetworkDevice'))
devicesofnet = dbcursor.fetchall()
if len(devicesofnet) > 0: #network has MR, MS, MX, Zx
if flag_firstnet:
if flag_firstorg:
ORG_LIST = []
lastorg = -1
flag_firstorg = False
ORG_LIST.append(org)
lastorg += 1
lastnet = -1
ORG_LIST[lastorg].nets = []
flag_firstnet = False
ORG_LIST[lastorg].nets.append(c_Net())
lastnet += 1
ORG_LIST[lastorg].nets[lastnet].id = net['id']
ORG_LIST[lastorg].nets[lastnet].name = net['name']
ORG_LIST[lastorg].nets[lastnet].shard = org.shard
ORG_LIST[lastorg].nets[lastnet].devices = []
for device in devicesofnet:
ORG_LIST[lastorg].nets[lastnet].devices.append(device)
LAST_ORGLIST_REFRESH = datetime.datetime.now()
print('INFO: Refresh complete at %s' % LAST_ORGLIST_REFRESH)
return None
def getclientlist(p_shardhost, p_serial, p_timespan):
merakirequestthrottler()
try:
r = requests.get('https://%s/api/v0/devices/%s/clients?timespan=%s' % (p_shardhost, p_serial, p_timespan), headers={'X-Cisco-Meraki-API-Key': ARG_APIKEY, 'Content-Type': 'application/json'}, timeout=(REQUESTS_CONNECT_TIMEOUT, REQUESTS_READ_TIMEOUT) )
except:
print('ERROR 04: Unable to contact Meraki cloud')
return(None)
if r.status_code != requests.codes.ok:
return(None)
return(r.json())
#SECTION: main
def main(argv):
global ARG_APIKEY
global ARG_ORGNAME
#initialize command line arguments
ARG_APIKEY = ''
ARG_ORGNAME = ''
arg_numresults = ''
arg_mode = ''
arg_filter = ''
#get command line arguments
try:
opts, args = getopt.getopt(argv, 'hk:o:m:')
except getopt.GetoptError:
printhelp()
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
printhelp()
sys.exit()
elif opt == '-k':
ARG_APIKEY = arg
elif opt == '-o':
ARG_ORGNAME = arg
elif opt == '-m':
arg_mode = arg
#check that all mandatory arguments have been given
if ARG_APIKEY == '':
printhelp()
sys.exit(2)
#set defaults for empty command line arguments
if ARG_ORGNAME == '':
ARG_ORGNAME = '/all'
refreshOrgList()
if ORG_LIST is None or DEVICE_DB is None:
print('ERROR 05: No organizations with network devices access points for the specified API key')
sys.exit(2)
DEVcursor = DEVICE_DB.cursor()
for org in ORG_LIST:
flag_firstNet = True
orgClientList = []
reportFileName = 'clients_' + org.name + '_' + str(datetime.datetime.now()).replace(':','.') + '.csv'
print ('INFO: Processing org "%s"' % org.name)
for net in org.nets:
print ('INFO: Processing net "%s"' % net.name)
for dev in net.devices:
clients = getclientlist(org.shard, dev[0], MAX_CLIENT_TIMESPAN)
for client in clients:
DEVcursor.execute('''SELECT oui FROM ouis WHERE oui = ?''', (client['mac'][:8],))
matchingMerakiOuis = DEVcursor.fetchall()
if len(matchingMerakiOuis) == 0: #client device is not, in fact, a Meraki device neighbour
if flag_firstNet:
flag_firstNet = False
print('INFO: Creating file "' + reportFileName + '"')
try:
f = open(reportFileName, 'w')
f.write('id,mac,description,mdnsName,dhcpHostname,ip,vlan,switchport,usageKBSentToClient,usageKBRecvFromClient,networkId,networkName,reportedByDevSerial,reportedByDevName,reportedByDevModel\n')
except:
print('ERROR 06: Unable to open file "' + reportFileName + '" for writing')
sys.exit(2)
try:
f.write(str(client['id']) + ',' +
str(client['mac']) + ',' +
str(client['description']) + ',' +
str(client['mdnsName']) + ',' +
str(client['dhcpHostname']) + ',' +
str(client['ip']) + ',' +
str(client['vlan']) + ',' +
str(client['switchport']) + ',' +
str(int(client['usage']['sent'])) + ',' +
str(int(client['usage']['recv'])) + ',' +
str(net.id) + ',' +
str(net.name) + ',' +
str(dev[0]) + ',' +
str(dev[1]) + ',' +
str(dev[2]) + '\n' )
except:
print('ERROR 08: Unable to write to file "' + reportFileName + '"')
sys.exit(2)
DEVICE_DB.close()
try:
f.close()
except:
print ('INFO: Unable to close file (not open?)')
if __name__ == '__main__':
main(sys.argv[1:])
Traceback error:
Traceback (most recent call last):
File "orgsclientcsv.py", line 351, in <module>
main(sys.argv[1:])
File "orgsclientcsv.py", line 308, in main
for client in clients:
TypeError: 'NoneType' object is not iterable
The result should be an csv file that will give me list of phones mac address serial number manufacturer but the actual output is only phones mac address like Iphone