I am making a simple client/server that will pass strings back and forth. It is only slightly more advanced than an echo server. However, I am having some issue with passing the strings. I first ran into an issue where I was using the incorrect data type (not converting to utf-8), but am still having an issue.
Here is my server code:
import socket
import re
host = ''
port = 15008
backlog = 5
size = 1024
QUIT = bytes("QUIT" , "utf-8")
BYE = bytes("BYE" , "utf-8")
MATCH = bytes("MATCH" , "utf-8")
NO_MATCH = bytes("NO MATCH", "utf-8")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host,port))
s.listen(backlog)
while 1:
client, address = s.accept()
regex = client.recv(size)
if regex == QUIT:
client.send(BYE)
client.close()
break
string = client.recv(size)
if string == QUIT:
client.send(BYE)
client.close()
break
if re.match(regex, string):
client.send(MATCH)
else:
client.send(NO_MATCH)
and the client code:
import socket
host = 'localhost'
port = 15008
size = 1024
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,port))
s.send(bytes('[ws]', "utf-8"))
s.send(bytes('s', "utf-8"))
data = s.recv(size)
print('Should match: ' + data)
s.close()
Right now both the server and client just get hung-up.
Needed to use decode.
data = s.recv(size).decode("utf-8")
Related
Working with processing data that I receive as b64-encoded strings. For some reason I received data the other day where a set of the b64-encoded strings had four white space chars in them, making the decoded output incorrect. I made a small python script to brute force the missing characters that did the job, but I wanted to see if I can improve the speed by multiprocessing.
I'm not that familiar with using generators so it seems like this is throwing me off in the process of making it work together with multiprocessing.
Following sample code isn't the exact code I'm working with, but similar enough.
import base64
from maskprocessor import maskprocessor as maskproc
from tqdm import tqdm
import string
import multiprocessing as mp
NCORE = 8
KEY = b"some decryption key"
printable_chars = set(bytes(string.printable, 'ascii'))
def printable(data):
return True if all(char in printable_chars for char in data) else False
def decrypt(cipher):
try:
plain = base64.b64decode(cipher) #someDecryptAlgo(KEY, base64.b64decode(cipher))
return plain
except:
return None
def slow_generator(ciphers):
def gen_to_queue(input_q, ciphers):
for cipher in ciphers:
input_q.put(cipher)
for _ in range(NCORE):
input_q.put((None,None))
def process(input_q, output_q):
while True:
cipher = input_q.get()
if cipher is None:
output_q.put(None)
break
output_q.put((cipher, decrypt(cipher)))
input_q = mp.Queue(maxsize=NCORE * 2)
output_q = mp.Queue(maxsize=NCORE * 2)
gen_pool = mp.Pool(1, initializer=gen_to_queue, initargs=(input_q, ciphers))
pool = mp.Pool(NCORE, initializer=process, initargs=(input_q, output_q))
finished_workers = 0
while True:
cipher, plain = output_q.get()
if plain is None:
finished_workers += 1
if finished_workers == NCORE:
break
else:
yield cipher, plain
if __name__ == "__main__":
actual_msg = "All your base are belong to us"
actual_data_b64 = b'QWxsIHlvdXIgYmFzZSBhcmUgYmVsb25nIHRvIHVz' # "All your base are belong to us
received_data_b64 = b'QWxsI lvdXIgY FzZSBhcmUg mVsb25n HRvIHVz'
charset = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+/"
n_ws = received_data_b64.decode('utf-8').count(' ')
g = maskproc(received_data_b64.decode('utf-8').replace(' ', '?1'), custom_charset1=charset)
output = slow_generator(g)
for cipher, plain in zip(x[0], x[1]):
if printable(plain):
if plain.decode('utf-8') == actual_msg:
print(f"Found correct cipher: {cipher}")
print(f"Secret message: {plain}")
break
The decoded b64 data is then decrypted using a static key, so the function for checking if I've found the correct bytes is obviously more complex than in the sample above, but I think the sample represents the same problem I got, making the generator work with multiprocessing. There is no reading/writing files involved.
I'd like to avoid creating a list of all the candidates that the maskproc generator creates but so far I haven't been able to make multiprocessing work without doing this, and thus I wonder if anyone got some pointers for how to do this?
Edit: Say I want to run N (4/8/16/etc) processes in parallel, could it be an idea to make a list of length N with candidates that is then processed and in this way avoid having to make a list of length 64^4?
Edit2: Changed code to the one adopted from https://stackoverflow.com/a/58832140/853898
Edit3: Think I managed to get it working. Now to only find a way to implement a tqdm progress bar, if possible.
import base64
from maskprocessor import maskprocessor as maskproc
from tqdm import tqdm
import string
import multiprocessing as mp
NCORE = 8
KEY = b"some decryption key"
printable_chars = set(bytes(string.printable, 'ascii'))
def printable(data):
return True if all(char in printable_chars for char in data) else False
def decrypt(cipher):
try:
plain = base64.b64decode(cipher) #someDecryptAlgo(KEY, base64.b64decode(cipher))
return plain
except:
return None
def slow_generator(ciphers):
def gen_to_queue(input_q, ciphers):
for cipher in ciphers:
input_q.put(cipher)
for _ in range(NCORE):
input_q.put(None)
def process(input_q, output_q):
while True:
cipher = input_q.get()
if cipher is None:
output_q.put((None, None))
break
output_q.put((cipher, decrypt(cipher)))
input_q = mp.Queue(maxsize=NCORE * 2)
output_q = mp.Queue(maxsize=NCORE * 2)
gen_pool = mp.Pool(1, initializer=gen_to_queue, initargs=(input_q, ciphers))
pool = mp.Pool(NCORE, initializer=process, initargs=(input_q, output_q))
finished_workers = 0
while True:
cipher, plain = output_q.get()
if plain is None:
finished_workers += 1
if finished_workers == NCORE:
break
else:
yield cipher, plain
if __name__ == "__main__":
actual_msg = "All your base are belong to us"
actual_data_b64 = b'QWxsIHlvdXIgYmFzZSBhcmUgYmVsb25nIHRvIHVz' # "All your base are belong to us
received_data_b64 = b'QWxsI lvdXIgY FzZSBhcmUg mVsb25n HRvIHVz'
#charset = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+/"
charset = "HmYI"
n_ws = received_data_b64.decode('utf-8').count(' ')
g = maskproc(received_data_b64.decode('utf-8').replace(' ', '?1'), custom_charset1=charset)
output = slow_generator(g)
for cipher, plain in output:
if printable(plain):
if plain.decode('utf-8') == actual_msg:
print(f"Found correct cipher: {cipher}")
print(f"Secret message: {plain}")
break
I am trying to create a server that will recieve messages from the client and answer them properly.
When i ask for a random number(RAND) i get this error "a bytes-like object is required, not 'int'",
how can i fix it?
and another question, i tried to change the bytes in 'recv' function, and didn't succeed. Can somebody help me out ):?
import socket
import time
import random
server_socket = socket.socket()
server_socket.bind(('0.0.0.0', 8820))
server_socket.listen(1)
(client_socket, client_address) = server_socket.accept()
localtime = time.asctime( time.localtime(time.time()) )
ran = random.randint(0,10)
RUN = True
recieve = 1024
while RUN:
client_input = (client_socket.recv(recieve)).decode('utf8')
print(client_input)
if client_input == 'TIME':
client_socket.send(localtime.encode())
elif client_input == 'RECV':
recieve = client_socket.send(input("the current recieve amount is " + int(recieve) + ". Enter the recieve amount: "))
elif client_input == 'NAME':
client_socket.send(str("my name is SERVER").encode())
elif client_input == 'RAND':
client_socket.send(ran.encode())
elif client_input == 'EXIT':
RUN = False
else:
client_socket.send(str("I can only get 'TIME', 'NAME', 'RAND', 'EXIT'").encode())
client_socket.close()
server_socket.close()
the client code is:
import socket
my_socket = socket.socket()
my_socket.connect(('127.0.0.1', 8820))
while True:
user_input = input("Naor: ")
my_socket.send(user_input.encode())
data = my_socket.recv(1024)
print("Server: " + data.decode('utf8'))
my_socket.close()
The reason for this error is that in Python 3, strings are Unicode, but when transmitting on the network, the data needs to be bytes instead. So... a couple of suggestions:
Suggest using client_socket.sendall() instead of client_socket.send() to prevent possible issues where you may not have sent the entire msg with one call (see docs).
For literals, add a 'b' for bytes string: client_socket.sendallsend(str("I can only get 'TIME', 'NAME', 'RAND', 'EXIT'").encode())
For variables, you need to encode Unicode strings to byte strings (see below)
output = 'connection has been processed'
client_socket.sendall(output.encode('utf-8'))
I wrote a client that connects to a server written in Go. Then I send an 8 byte packet. As the response comes in, it gets passed through a function I built for unpacking it - which generates an error:
Error: line 38, in binary_response
tag, code = struct.unpack('!I I ', response[:8]) struct.error: unpack requires a buffer of 8 bytes
I've tried to take out the function and do everything in the while loop, but have the same error.
import socket
import struct
import textwrap
from time import sleep
TCP_IP = '127.0.0.1'
TCP_PORT = 4200
TAB_1 = '\t - '
TAB_2 = '\t\t - '
TAB_3 = '\t\t\t - '
TAB_4 = '\t\t\t\t - '
DATA_TAB_1 = '\t '
DATA_TAB_2 = '\t\t '
DATA_TAB_3 = '\t\t\t '
DATA_TAB_4 = '\t\t\t\t '
def main():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", 4200))
packet = struct.pack('!ii', 10, 4)
s.send(packet)
while True:
response = s.recv(65535)
bny = binary_response(response)
print(TAB_1 + 'Binary Response Segment:')
print('Response Code: ', bny[1])
print('Client ID: ', bny[0])
print('Message: ', response.decode('latin-1'))
# Unpack binary_response
def binary_response(response):
tag, code = struct.unpack('!I I ', response[:8])
return tag, code
main()
What am I missing?
I have the below script which is perfectly working, but when I read the status of the Cisco devices using a for loop and send mail using the user defined function send_mail(), I am receiving two mails instead of one mail, any suggestions to get a single mail, will be greatly appreciated:
username = 'cisco'
password = 'cisco'
hosts = ['10.0.10.100','10.0.10.101']
platform = 'cisco_ios'
def send_mail():
fromaddr = "#gmail.com"
toaddr = "#hotmail.com"
msg = MIMEMultipart()
msg['From'] = '#gmail.com'
msg['To'] = '#hotmail.com'
msg['Subject'] = "This is Health check"
msg.attach(MIMEText(status, 'plain'))
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(fromaddr, "password")
text = msg.as_string()
server.sendmail(fromaddr, toaddr, text)
server.quit()
for host in hosts:
#print(host)
connect = ConnectHandler(device_type=platform, ip=host, username=username, password=password)
output = connect.send_command('terminal length 0', expect_string=r'#')
output = connect.send_command('enable',expect_string=r'#')
host_name = connect.send_command('show run | in hostname',expect_string=r'#')
interface_status = connect.send_command(f'show ip int brief',expect_string=r'#')
#print(interface_status)
old_stdout = sys.stdout
interface_result = StringIO()
sys.stdout = interface_result
sys.stdout = old_stdout
data = pd.read_fwf(StringIO(interface_status), widths=[27, 16, 3, 7, 23, 8])
status = " "
for index, row in data.iterrows():
if row[4] == 'administratively down' or row[4] == 'down':
log = (f"\nInterface {row[0]} is down in {host_name}\n")
status += log
bgp_status = connect.send_command('show ip bgp summary | be N',expect_string=r'#')
old_stdout = sys.stdout
bgp_result = StringIO()
sys.stdout = bgp_result
sys.stdout = old_stdout
bgp_data = pd.read_fwf(StringIO(bgp_status), delim_whitespace=True, header=None)
for index, row in bgp_data.iterrows():
if row[9] == 'Down' or row[9] == 'Idle' or row[9] == 'Active':
bgp = (f"\nNeighbor {row[0]} is down in {host_name}\n")
status += bgp
send_mail()
I think I understand what your issue is. In short, you are calling send_mail() within each for loop, whereas you actually want to collect statuses for each host, and then send a final status report as one email. This is a short example to illustrate how you might do this - obviously you will have to adapt your code accordingly.
A couple of points:
Rather than having send_mail rely on a global status variable, pass in a status argument. This helps decouple functions from each other.
You might want to collect statuses for each host in a list, and then combine them in a logical manner when sending your status email.
String concatenation is slow. Better to build a list of strings, and join them using the string join function.
Here is some example code:
hosts = ['10.0.10.100','10.0.10.101']
def send_mail(status=''):
'''A placeholder function for testing'''
print("Sending email:\n{status}".format(status=status))
return
status_for_hosts = []
for host in hosts:
status_list = []
# An example of how you might collect statuses for a host
status_list.append("Host {host_ip} is up".format(host_ip=host))
# Use join() instead of string concatenation
full_status = ''.join(status_list)
status_for_hosts.append(full_status)
send_mail(status='\n'.join(status_for_hosts))
I have a single threaded program that simply executes commands over ssh and simply looks for output over ssh. But after a while I start getting extrodinarily strange behaviour:
ssh_cmd = 'ssh %s#%s %s' % (user, addr, options)
ssh = pexpect.spawn(ssh_cmd, timeout=60)
lgsuc = ['(?i)(password)')]
for item in loginsuccess:
lgsuc.append(item)
retval = ssh.expect(lgsuc)
for cmd in cmdlist:
time.sleep(0.1)
#this is regex to match anything. Essentially clears the buffer so you don't get an invalid match from before
ssh.expect(['(?s).*'])
ssh.sendline(cmd)
foundind = ssh.expect([re.escape("root#")], 30) #very slow
#repr escape all the wierd stuff so madness doesn't happen with ctr chars
rettxt = repr(ssh.before.decode("us-ascii") + "root:#")
print("We Found:" + rettxt
And it will be fine for about 20 commands or so then madness occurs Assume the right echo is blablabla each time:
We found 'blablabla \r\n\r\n[edit]\r\nroot#'
We found 'blablabla \r\n\r\n[edit]\r\nroot#'
We found 'blablabla \r\n\r\n[edit]\r\nroot#'
... about 20 commands...
We found 'bl\r\nroot#' # here it just missed part of the string in the middle
We found 'alala\r\nroot#'
here is the remainder of the echo from the previous command!!! and the echo of the current command will show up some time later!! and it gets worse and worse. The thing that is strange is it is in the middle of the return byte array.
Now there are some wierd control codes coming back from this device so if I replace:
rettxt = repr(ssh.before.decode("us-ascii") + "root:#")
with
rettxt = repr(ssh.before.decode("us-ascii") + "root:#")
then
print("We Found:" + rettxt)
returns:
root#e Found lala
Anyway there is really strange stuff going on with pexpect and the buffers, and I can't figure out what it is so any help would be appreciated. I should mention I never get teh timeout, the dive always responds. Also the total number of "root:#"in the log file exceedes the total number of lines sent.
If I go through and remove all ctl codes, the output looks cleaner but the problem still persists, its as if pextect cant handle ctl coodes in its buffer correctly. Any help is appreciated
UPDATE Minimum verifiable example
Ok I have been able to recreate PART of the problem on an isolated ubuntu environment sshing into itself.
first I need to create 4 commands that can be run on a host target, so put the follwing for files in ~/ I did this in ubuntu
~/check.py
#!/usr/bin/python3
import time
import io
#abcd^H^H^H^H^MABC
#mybytes = b'\x61\x62\x63\x64\x08\x08\x08\x0D\x41\x42\x43'
#abcdACB
mybytes = b'\x61\x62\x63\x64\x41\x42\x43'
f = open('test.txt', 'wb')
#time.sleep(1)
f.write(mybytes)
print(mybytes.decode('ascii'))
f.close()
~/check2.py
#!/usr/bin/python3
import time
import io
#0123^H^H^H^H^MABC
mybytes = b'\x30\x31\x32\x33\x08\x0D\x0D\x08\x08\x08\x08\x0D\x41\x42\x43'
f = open('test2.txt', 'wb')
#time.sleep(0.1)
f.write(mybytes)
print(mybytes.decode('ascii'))
f.close()
~/check3.py
#!/usr/bin/python3
import time
import io
#789:^H^H^H^H^DABC
mybytes = b'\x37\x38\x39\x3A\x08\x08\x08\x08\x08\x08\x08\x0D\x0D\x41\x42\x43'
f = open('test3.txt', 'wb')
#time.sleep(0.1)
f.write(mybytes)
print(mybytes.decode('ascii'))
f.close()
And lastly check4.py Sorry it took a wierd combination for the problem to show back up
#!/usr/bin/python3
import time
import io
#abcd^H^H^H^HABC
mybytes = b'\x61\x62\x63\x64\x08\x08\x08\x0D\x41\x42\x43'
f = open('test.txt', 'wb')
time.sleep(4)
f.write(mybytes)
print(mybytes.decode('ascii'))
f.close()
Noticing that the last one has a bigger sleep, this is to encounter texpect timeout. Though on my actual testing this doesn't occue, I have commands that take over 6 minutes to return any text so this might be part of it. Ok and the final file to run everything. It might look ugly but I did a massive trim so I could post it here:
#! /usr/bin/python3
#import paramiko
import time
import sys
import xml.etree.ElementTree as ET
import xml
import os.path
import traceback
import re
import datetime
import pexpect
import os
import os.path
ssh = None
respFile = None
#Error Codes:
DEBUG = True
NO_ERROR=0
EXCEPTION_THROWS=1
RETURN_TEXT_NEVER_FOUND = 2
LOG_CONSOLE = 1
LOG_FILE = 2
LOG_BOTH = 3
def log(out, dummy=None):
print(str(log))
def connect(user, addr, passwd):
global ssh
fout = open('session.log', 'wb')
#options = '-q -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oPubkeyAuthentication=no'
options = ' -oUserKnownHostsFile=/dev/null '
#options = ''
#REPLACE WITH YOU LOCAL USER NAME
#user = 'user'
#REPLACE WITH YOUR LOCAL PASSWORD
#passwd = '123TesT321'
#addr = '127.0.0.1'
ssh_cmd = 'ssh %s#%s %s' % (user, addr, options)
public = None
private = None
retval = 0
try:
ssh = pexpect.spawn(ssh_cmd, timeout=60)
ssh.logfile = fout
#the most common prompts
loginsuccess = [re.escape("#"), re.escape("$")]
lgsuc = ['(?i)(password)', re.escape("connecting (yes/no)? ")]
for item in loginsuccess:
lgsuc.append(item)
retval = ssh.expect(lgsuc)
except pexpect.TIMEOUT as exc:
log("Server never connected to SSH tunnel")
return 0
print('where here ret val = ' + str(retval))
try:
if(retval > 1):
return 1
elif(retval == 1):
hostkey = ssh.before.decode("utf-8")
ssh.sendline("yes")
log("Warning! new host key was added to the database: " + hostkey.split("\n")[1])
lgsuc = ['password: ']
for item in loginsuccess:
lgsuc.append(item)
retval = ssh.expect(lgsuc)
if(retval > 0):
return 1
else:
if(public is not None):
log("Warning public key authentication failed trying password if available...")
else:
if public is not None:
log("Warning public key authentication failed trying password if available...")
if(passwd is None):
log("No password and certificate authentication failed...")
return 0
ssh.sendline(passwd)
login = ['password: ' ]
for item in loginsuccess:
login.append(item)
retval = ssh.expect(login)
except pexpect.TIMEOUT as exc:
log("Server Never responded with expected login prompt: "+lgstr)
return 0
#return 0
if retval > 0:
retval = 1
if retval == 0:
log("Failed to connect to IP:"+addr +" User:"+user+" Password:"+passwd)
return retval
def disconnect():
log("Disconnecting...")
global ssh
if ssh is not None:
ssh.close()
else:
log("Something wierd happened with the SSH client while closing the session. Shouldn't really matter", False)
def RunCommand(cmd, waitTXT, timeout = 5):
global ssh
Error = 0
if DEBUG:
print('Debugging: cmd: '+ cmd+'. timeout: '+str(timeout) +'. len of txt tags: '+ str(len(waitTXT)))
if(type(waitTXT) is str):
waitTXT = [re.excape(waitTXT)]
elif(not hasattr(waitTXT ,'__iter__')):
waitTXT = [re.escape(str(waitTXT))]
else:
cnter = 0
for TXT in waitTXT:
waitTXT[cnter] = re.escape(str(TXT))
cnter +=1
#start = time.time()
#print("type3: "+str(type(ssh)))
#time.sleep(1)
#this is regex to match anything. Essentially clears the buffer so you don't get an invalid match from before
ssh.expect(['(?s).*'])
ssh.sendline(cmd)
print("Debugging: sent: "+cmd)
#GoOn = True
rettxt = ""
try:
foundind = ssh.expect(waitTXT, timeout)
allbytes = ssh.before
newbytes = bytearray()
for onebyte in allbytes:
if onebyte > 31:
newbytes.append(onebyte)
allbytes = bytes(newbytes)
rettxt = repr(allbytes.decode("us-ascii") + waitTXT[foundind])
#rettxt = ssh.before + waitTXT[foundind]
if DEBUG:
print("Debugging: We found "+rettxt)
except pexpect.TIMEOUT as exc:
if DEBUG:
txtret = ""
for strtxt in waitTXT:
txtret += strtxt +", "
print("ERROR Debugging: we timed out waiting for text:"+txtret)
pass
return (rettxt, Error)
def CloseAndExit():
disconnect()
global respFile
if respFile is not None and '_io.TextIOWrapper' in str(type(respFile)):
if not respFile.closed:
respFile.close()
def main(argv):
try:
cmds = ['~/check.py', '~/check2.py', '~/check3.py', '~/check2.py', '~/check3.py','~/check.py', '~/check2.py', '~/check3.py', '~/check2.py', '~/check3.py', '~/check4.py', '~/check3.py','~/check.py', '~/check2.py',]
##CHANGE THESE TO MTACH YOUR SSH HOST
ret = connect('user', '127.0.0.1', 'abcd1234')
for cmd in cmds:
cmdtxt = str(cmd)
#rett = RunCommand(ssh, "ls", "root", 0, 5)
strlen = (170 - (len(cmdtxt)))/2
dashval = ''
starval = ''
tcnt = 0
while(tcnt < strlen):
dashval +='-'
starval +='*'
tcnt +=1
if DEBUG:
print(dashval+cmdtxt+dashval)
#checkval = ['ABC']
#REPLACE THE FOLLOWING LINE WITH YOUR TARGET PROMPT
checkval = ['user-virtual-machine:~$']
rett = RunCommand(cmdtxt, checkval, 2)
if DEBUG:
print(starval+cmdtxt+starval)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
print(traceback.format_exc())
CloseAndExit()
#disconnect()
#respFile.close()
main(sys.argv)
Make sure that all for checks and the main python script are executble in permission via sudo chmod 774 or similar. In the main function call set your username ipaddress and password to where your target is that has the check.py and make sure they are in your ~/ directory.
Once you run this you can look at the session.log and at least on mind there is some wierd stuff going on with the buffer:
~/check4.py^M
~/check3.py
~/check3.py^M
abcd^H^H^H^MABC^M
^[]0;user#user-virtual-machine: ~^Guser#user-virtual-machine:~$ ~/check.py
~/check3.py^M
789:^H^H^H^H^H^H^H^M^MABC^M
^[]0;user#user-virtual-machine: ~^Guser#user-virtual-machine:~$ ~/check.py~/check2.py
And unfortunately its not as corrupt as my actual prbolem but I have several hundred commands I an embedded custom linux kernal that I obviously can't recreate for everyone. But anyway any help is greatly appreciated. Hopefully these examples work you you, I am just on ubuntu 16.04 lts. Also make sure to replace 'user-virtual-machine:~$' with whatever your target login prompt looks like