I have a project I'm working on where I would like to query MariaDB via a file line by line. It is all written in python. The problem I'm having is it will only print the last line if found, not all matches. My expectation is that it prints to the terminal every time it finds a match. Any help is greatly appreciated.
import mysql.connector
import time
file1 = open("sample.unique.big", "r")
count = 0
mydb = mysql.connector.connect(
host="10.0.0.72",
user="admin",
password="#########",
database="#########"
)
start = time.time()
while True:
count += 1
# Get next line from file
line = file1.readline()
# if line is empty
# end of file is reached
if not line:
break
#print("{}".format(line.strip()))
#print("Line{}: {}".format(count, line.strip()))
mycursor = mydb.cursor()
sql = "SELECT * FROM ether2 use index (idx_privadd) WHERE privadd = %s"
adr = (line, )
mycursor.execute(sql, adr)
myresult = mycursor.fetchall()
for x in myresult:
print(x)
end = time.time()
print(end-start)
i achieved my desired results by first formating the readline. Secondly, an if statement for myresults. Completed code looks like
import mysql.connector
import time
file1 = open("sample.unique.big", "r")
count = 0
mydb = mysql.connector.connect(
host="10.0.0.72",
user="*****",
password="*****",
database="etherpro"
)
start = time.time()
while True:
count += 1
# Get next line from file
line = file1.readline()
#Format the line
search=("{}".format(line.strip()))
# if line is empty
# end of file is reached
if not line:
break
mycursor = mydb.cursor()
sql = "SELECT id FROM ether2 WHERE privadd = %s"
adr = (search, )
mycursor.execute(sql, adr)
myresults = mycursor.fetchone()
if myresults:
print("oh snap, the id is:",myresults)
end = time.time()
print(end-start)
Related
I was trying to make a basic data collector that will put information in excel and i was successful at doing so with the titles of the data sets but i cant figure out why it cant write the data entered into an excel file. Its far from done and i am fairly new to python (about a month into learning) so if my code looks terrible then my apologies but for any help thank you.
below is my full code and the error message along with it:
import xlsxwriter
import os
file = open("Datasheet.xlsx", "w+")
databook = xlsxwriter.Workbook("Datasheet.xlsx")
data = databook.add_worksheet()
titles = []
masterdata = {}
data_entry = 0
data_filled = 0
project_start = True
while project_start == True:
data_entires = input("How many data entries will you need? ")
if data_entires.isdigit() == False:
print("Sorry that is not a valid input please enter a numerical integer")
project_start = True
if data_entires.isdigit() == True:
project_start = False
while int(data_entires) != data_entry:
data_title = input("Please enter a title for a dataset ")
titles.append(data_title)
data_entry += 1
for value in range(len(titles)):
data.write(0, value + 1, titles[value])
# dictionary with created lists for the amount of titles made
for i in range(len(titles)):
dict = masterdata["list " + str(i + 1)] = []
for i in range(len(masterdata)):
data_in = True
while data_in == True:
data = input(f"Please input data for {titles[i]} ")
data_fill = masterdata["list " + str(i + 1)].append(data)
fin = input("Do you want to input more data? ")
if "yes" in fin:
data_in = True
if "no" in fin:
data_in = False
lis = [list(x) for x in masterdata.values()]
for part in lis:
l = list(str(part))
for value in l:
data.write(value + 1, value + 1, l[value])
#os.startfile("Datasheet.xlsx")
#databook.close()
Error message : "Traceback (most recent call last):
File "f'{filelocation}'", line 49, in
data.write(value + 1, value + 1, l[value])
AttributeError: 'str' object has no attribute 'write' "
experts
I meet an value missing error in my code , but I think the variable in function are claimed. I don't know why it is happens.
$ python check_rsg_V0312.py
2018-03-20 13:05:49 === Script Start ===
2018-03-20 13:05:49 Monitoring via remote logon
The authenticity of host 'rpahost0 ([127.0.0.1]:7000)' can't be established.
RSA key fingerprint is 2d:f5:67:75:84:b6:24:45:e6:48:60:65:61:ca:69:f7.
Are you sure you want to continue connecting
(yes/no)? yes
Warning: Permanently added 'rpahost0' (RSA) to the list of known hosts.
Password:
Traceback (most recent call last):
File "check_rsg_V0312.py", line 89, in <module>
label = ssh_cmd(nLocalport, rsg_target, ouser, lookip, opasw, command,)
File "check_rsg_V0312.py", line 79, in ssh_cmd
print (ssh.before.decode(),ssh.after().decode())
TypeError: __init__() missing 1 required positional argument: 'value'
Below is my code, it is really strange that one error happens in ssh_cmd
function. detailed please review comments in code.
import os,time,pexpect,re, subprocess, smtplib
from email import encoders
from email.header import Header
from email.mime.text import MIMEText
#-----------------------------------------
dir = os.environ['HOME']
ouser = 'x02d726'
opasw = 'qwe12'
nLocalport = 7000
lookip = '127.0.0.1'
rsg_target = "rpahost0"
command = "ls -ltrh | grep tunnel | tail"
nstage = 0
mail_addr = 'cheng.huang#qq.com'
otp_file = dir + '/otplist/C591260'
otp_list = []
rsg_file = dir + '/.rsg_hosts'
known_hosts = dir + '/.ssh/known_hosts'
rsg_port = "auto"
#-----------------------------------------
def printlog(prompt):
year, mon, mday, hour, min, sec, wday, yday, isdst = time.localtime()
print("%04d-%02d-%02d %02d:%02d:%02d %s" % (year, mon , mday, hour, min,
sec, prompt))
def get_egw_name(ref_arr, key):
for oneline in ref_arr:
if (re.search(key, oneline)):
templine = oneline
oneline = re.sub('^\s+|\s+$','',oneline)
egw_ssg = re.split('\s+',oneline)[2]
result = re.split(':',egw_ssg)[0]
return result
def sendworker(to_addr):
from_addr = 'itk-bj.ericsson.se'
smtp_server = 'smtp.eamcs.ericsson.se'
msg = MIMEText('There is no otp left ,please input new OTP list',
'plain', 'utf-8')
msg['From'] = from_addr
msg['To'] = to_addr
msg['Subject'] = Header(u'OTP List is Blank', 'utf-8')
server = smtplib.SMTP(smtp_server, 25)
#server.set_debuglevel(1)
server.sendmail(from_addr, to_addr, msg.as_string())
server.quit()
def ssh_cmd(port, target, user, ip, pasw, cmd ):
printlog("=== Script Start ===")
printlog("Monitoring via remote logon")
time.sleep(1)
ssh = pexpect.spawn('/usr/bin/ssh -p %s -o HostKeyAlias=%s %s#%s %s' %
(port, target, user, ip, cmd ),timeout=6000)
try:
i = ssh.expect(['Password: ', 'continue connecting (yes/no)?'],
timeout=15)
if i == 0 :
print(ssh.before.decode(),ssh.after.decode())
ssh.sendline(pasw)
elif i == 1:
print(ssh.before.decode(),ssh.after.decode())
ssh.sendline('yes')
ssh.expect('Password: ')
print(ssh.before.decode(),ssh.after.decode())
ssh.sendline(pasw)
except pexpect.EOF:
print ("no connection EOF,please check RSG tunnel")
except pexpect.TIMEOUT:
print ("your pexpect has TIMEOUT")
else:
ssh.expect(pexpect.EOF)
print (ssh.before.decode(),ssh.after().decode()) # if I disable this line, there will be no error.
flag = ssh.before.decode()
return flag
ssh.close()
if __name__ == '__main__':
if os.path.exists(os.environ['HOME'] + "/.ssh/known_hosts"):
os.remove(known_hosts)
else:
pass
label = ssh_cmd(nLocalport, rsg_target, ouser, lookip, opasw, command)
if re.search('tunnel_check', str(label)):
nstage = 1
if (nstage == 0):
printlog("Tunnel was down and will re-establish now\n")
rsg = open (rsg_file,'r')
rsg_in = rsg.readlines()
rsg.close()
egwname = get_egw_name(rsg_in, rsg_target)
try :
otp = open(otp_file, 'r')
otp_arrary = otp.readlines()
otp.close()
for ot in otp_arrary:
ot = ot.strip()
ot = ot.replace('^\s*|\s*$', '')
otp_list.append(ot)
otp_num = len(otp_list) + 1
if (otp_num > 0):
os.remove(rsg_file)
try :
new_otp = otp_list[0]
except IndexError:
printlog('There is no otp left ,please input new OTP list')
sendworker(mail_addr)
sys.exit()
else:
out = open(rsg_file, 'w')
for line in rsg_in :
line = line.strip()
line = line.replace('^\s+|\s+$','')
if (re.match(egwname, line)):
temp_line = line
old_otp = re.split('\s+',temp_line)[5]
old_otp = old_otp.replace('^\s+|\s+$', '')
line = line.replace(old_otp, new_otp).replace('\\','')
printlog(line + "\n")
out.write(line + "\n")
out.close()
os.remove(otp_file)
time.sleep(1)
outotp = open(otp_file , 'w+')
i = 0
while (i < len(otp_list)):
outotp.write(otp_list[i] + "\n")
i += 1
outotp.close()
os.system("pkill -9 -f \"ssh\.\*-L " + str(nLocalport) + "\"")
os.system("sleep 10")
os.system("pkill -9 -f \"rtunnel\.\*-p " + str(nLocalport) + "\"")
os.system("nohup /opt/ericsson/itk/bin/rtunnel -d -q -g -p " + str(nLocalport) + "-rp auto " + rsg_target + " &")
os.system("sleep 10")
printlog("Kill the rtunnel process\n");
printlog("Tunnel is re-established again\n");
else:
sendworker(mail_addr)
except IOError:
print ("File is not accessible.")
else:
printlog("Tunnel OK")
As you see , after disable the line in Try... else... block, the code will be OK.
else:
ssh.expect(pexpect.EOF)
print (ssh.before.decode(),ssh.after().decode()) # if I disable this line, there will be no error.
flag = ssh.before.decode()
return flag
If you look carefully your traceback, you will find the problem:
Instead of:
print (ssh.before.decode(),ssh.after().decode())
you should write
print (ssh.before.decode(),ssh.after.decode())
after is a method which return string, not a function.
BTW, I think you should put decoding/encoding in your pexpect constructor.
from geopy.geocoders import Nominatim
import openpyxl
wb = openpyxl.load_workbook('#######.xlsx')
ws = wb.active
geolocator = Nominatim(timeout=60)
for i in range(2,1810):
count1 = 0
count2 = 1
address = str(ws['B'+str(i)].value)
city = str(ws['C'+str(i)].value)
state = str(ws['D'+str(i)].value)
zipc = str(ws['F'+str(i)].value)
result = None
iden1 = address + ' ' + city + ' ' + state
iden2 = city + ' ' + zipc + ' ' + state
iden3 = city + ' ' + state
print(iden1, iden2, iden3)
print(geolocator.geocode(iden2).address)
try:
location1 = geolocator.geocode(iden1)
except:
pass
try:
location2 = geolocator.geocode(iden2)
except:
pass
try:
location3 = geolocator.geocode(iden3)
except:
pass
count = None
try:
county1 = str(location1.address)
county1_list = county1.split(", ")
#print(county1_list)
for q in county1_list:
if 'county' in q.lower():
if count == None:
count = q
except:
pass
try:
county2 = str(location2.address)
county2_list = county2.split(", ")
#print(county2_list)
for z in county2_list:
if 'county' in z.lower():
if count == None:
count = z
except:
pass
try:
county3 = str(location3.address)
county3_list = county3.split(", ")
#print(county3_list)
for j in county3_list:
if 'county' in j.lower():
if count == None:
count = j
except:
pass
print(i, count)
#ws['E'+str(i)] = count
if count == 50:
#wb.save("#####" +str(count2) +".xlsx")
count2 += 1
count1 = 0
Hello all, this code is pretty simple and uses geopy to extract county names using 3 different methods names iden1, iden2, and iden3 which are a combination of address, city, state, and zipcode. This ran fine for about 300 lines but began to repeat the same county, and after restarting the script, just spat out Nones. I put in the line print (geolocator.geocode(iden2).address) to find the error and got this error message.
Traceback (most recent call last):
File "C:/Users/#####/Downloads/Web content/#####/####_county.py",
line 19, in
print(geolocator.geocode(iden2).address) File "C:\Users#####\AppData\Local\Programs\Python\Python36-32\lib\site-packages\geopy\geocoders\osm.py",
line 193, in geocode
self._call_geocoder(url, timeout=timeout), exactly_one File "C:\Users#####\AppData\Local\Programs\Python\Python36-32\lib\site-packages\geopy\geocoders\base.py",
line 171, in _call_geocoder
raise GeocoderServiceError(message) geopy.exc.GeocoderServiceError: [WinError 10061] No connection could
be made because the target machine actively refused it
This script was working before but now does not. Is my IP being blocked from using goepy's database or something? Thanks for your help!
It looks like you're hitting their ratelimiting. It seems that they ask that you limit your API requests to 1/second. You can take a look here for their usage policy where they list alternatives to using their API as well as contraints.
I am trying to use the following code to filter a txt file based on the info in 89,90,91 and 92 index.
The problem is that output file is an empty file just with headers.The code is not giving any error-so am not sure how else to go about debugging it.
Thanks for helping!!
for line in fileHandle:
if firstLineFlag == 0: #to skip first line
firstLineFlag = 1
firstLineText = line #save the first line elsewhere
continue
parts = line.strip().split('\t')
try:
column13=float(parts[13-1])
except ValueError:
column13=0
if column13 < 0.01:
if parts[92] == "./.:.:.:.:.":
Nor_info_2 = parts[92].replace("./.:.:.:.:.", "00:1,1:1:1:1,1,1")
if parts[91] == "./.:.:.:.:.":
Nor_info = parts[91].replace("./.:.:.:.:.", "00:1,1:1:1:1,1,1")
if parts[90] == "./.:.:.:.:.":
Tu_info_2 = parts[90].replace("./.:.:.:.:.", "00:1,1:1:1:1,1,1")
if parts[89] == "./.:.:.:.:.":
Tu_info = parts[89].replace("./.:.:.:.:.", "00:1,1:1:1:1,1,1")
normalSplit_2 = parts[92].split(':')
normalSplit = parts[91].split(':')
tumorSplit_2 = parts[90].split(':')
tumorSplit = parts[89].split(':')
print(Nor_info_2)
try:
TD_Tumor_2 = float(tumorSplit_2[3-1])
except ValueError:
TD_Tumor = 0
try:
TD_Tumor = float(tumorSplit[3-1])
except ValueError:
TD_Tumor = 0
try:
TD_Normal_2 = float(normalSplit_2[3-1])
except ValueError:
TD_Tumor = 0
try:
TD_Normal = float(normalSplit[3-1])
except ValueError:
TD_Tumor = 0
if TD_Tumor_2 >= TD_Tumor and TD_Tumor_2 >= 7:
tumorAD=tumorSplit_2[2-1].split(',')
normalAD=normalSplit_2[2-1].split(',')
normalratio=float(normalAD[2-1])/TD_Normal_2
else:
tumorAD=tumorSplit[2-1].split(',')
normalAD=normalSplit[2-1].split(',')
normalratio=float(normalAD[2-1])/TD_Normal
tumorratio=float(tumorAD[2-1])/TD_Tumor_2
parts.append(tumorratio)
parts.append(normalratio)
data.append(parts)
dataz1 = sorted(data, key = itemgetter(91), reverse = True)
#with open('filtered/'+currentFile+'_filtered.txt', 'w') as fileHandle: ## to write your data in proper format
with open(currentFile+'_filtered.txt', 'w') as fileHandle: ## to write your data in proper format
fileHandle.write(firstLineText)
for item in data:
convert_first_to_generator = (str(w) for w in item)
string = '\t'.join(convert_first_to_generator)
string += '\n'
#print string
fileHandle.write(string)
command = 'mv '+currentFile+'_filtered.txt filtered/' ### to move edited files into a different folder
system(command)
a python program to read the rainfall.txt file and then write out a new file
called rainfallfmt.txt . The data should be grouped on the total annual rainfall field into
the following categories: [60-70], [71-80], [81-90],[91-]. Under each category, the new
file should format each line so that the city is right-justified in a field that is 25 characters wide, and the rainfall data should be printed in a field that is 5 characters wide with 1 digit to the right of the decimal point.
This is what I have so far;
the problem is I don't know how to categorize it?
Would you please help me and let me know how I can solve this problem?
================================================
# read the rainfall.txt then write out a new file called rainfall.txt
# the data should be grouped on the total annual rainfall field into the
# categories: [60-70], [71-80], [81,90], [91-]
import os.path
def main():
endofprogram = False
try:
InputFileName = input('Enter name of input file: ')
infile = open(InputFileName,'r')
OutputFileName = input('Enter name of output file: ')
# determine wether name exists
while True:
if os.path.isfile(OutputFileName):
OutputFileName = input('File Exists. Enter name again: ')
else:
outfile = open(OutputFileName,'w')
break
except IOError:
print("Error opening file - End of program")
endofprogram = True
#If there is not exception, start reading the input file
#Write the same data in formated form in new file.
if endofprogram == False:
data = []
for line in infile:
.
.# I dont know what to do in here!
.
outfile.write(data[0])
main()
# read the rainfall.txt then write out a new file called rainfall.txt
# the data should be grouped on the total annual rainfall field into the
# categories: [60-70], [71-80], [81,90], [91-]
import os.path
def main():
endofprogram = False
try:
InputFileName = input('Enter name of input file: ')
infile = open(InputFileName,'r')
OutputFileName = input('Enter name of output file: ')
# determine wether name exists
while True:
if os.path.isfile(OutputFileName):
OutputFileName = input('File Exists. Enter name again: ')
else:
outfile = open(OutputFileName,'w')
break
except IOError:
print("Error opening file - End of program")
endofprogram = True
#If there is not exception, start reading the input file
#Write the same data in formated form in new file.
if endofprogram == False:
cat_60 = []
cat_71 = []
cat_81 = []
cat_91 = []
for line in infile:
city, rain = line.split(' ')
rain = float(rain)
if 60 <= rain < 70:
cat_60.append((city, rain)) # Storing a tuple in the list
elif 70 <= rain < 80:
cat_71.append((city, rain))
elif 80 <= rain < 90:
cat_81.append((city, rain))
elif 90 <= rain :
cat_91.append((city, rain))
outfile.write("[60-70]"+'\n')
for i in range(len(cat_60)):
city = cat_60[i][0]
rain = cat_60[i][1]
outfile.write('%+25s'%(city)+'%5.1f'%(rain)+'\n')
outfile.write("[70-80]"+'\n')
for i in range(len(cat_71)):
city = cat_71[i][0]
rain = cat_71[i][1]
outfile.write('%+25s'%(city)+'%5.1f'%(rain)+'\n')
outfile.write("[80-90]"+'\n')
for i in range(len(cat_81)):
city = cat_81[i][0]
rain = cat_81[i][1]
outfile.write('%+25s'%(city)+'%5.1f'%(rain)+'\n')
outfile.write("[91-]"+'\n')
for i in range(len(cat_91)):
city = cat_91[i][0]
rain = cat_91[i][1]
outfile.write('%+25s'%(city)+'%5.1f'%(rain)+'\n')
main()
You can use split to separate the line in fields (it returns a list of strings), then take the rainfall, convert it to float, and use a list per category to store the data.
cat_60 = []
cat_71 = []
cat_81 = []
cat_91 = []
for line in infile:
city, rain = line.split(' ') # Spliting using blank as separator
rain = float(rain)
if 60 <= rain <= 70: # This works in Python as expected but don't in most other languages
cat_60.append((city, rain)) # Storing a tuple at the end of the list
elif 71 <= rain <= 80:
cat_71.append((city, rain))
# etc...
After that you can iterate in the categories and format the output with str.format() or using the % operator.
There are other things you can do to make your code more Pythonic, for example variable names should be snakecase, names in titlecase should be reserved for classes and uppercase for constants and instead of using a flag variable you can use and exit() to end the program.
You can do the following to call main() only if the file is invoked as a script:
if __name__ == '__main__':
main()