I am using below provided code to retrive some data with the socket. Issue with this code, it prints all the results till it breaks eventhough I only care for the received line right before it breaks, the second last in other words. So, I need some help to understand how that can be achieved.
import socket
import time
socket.setdefaulttimeout(10)
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(("192.168.50.102", 2102))
curIndex = "0"
while True:
sending_data = 'get,trx,'+curIndex
#print sending_data
client.send(sending_data)
data = client.recv(128)
print data.encode('UTF-8')
if data == "trx,notfound": break
spdata = data.split(",")
#print spdata[2] + 'kg' #Prints weight + kg
if len(spdata) >= 3:
curIndex = spdata[1]
time.sleep(0.5)
client.close()
Actual output
trx,2,1.250,0.000,19-07-11 14:08:01
trx,3,0.500,0.000,19-07-11 14:19:24
trx,4,0.500,0.000,19-07-11 15:04:37
trx,5,0.250,0.000,19-07-11 15:05:31
trx,6,0.177,0.000,19-07-11 21:06:59
trx,7,0.108,0.000,19-07-12 14:54:00
trx,8,0.106,0.000,19-07-16 17:51:06
trx,9,0.106,0.000,19-07-16 17:54:24
trx,10,0.106,0.000,19-07-18 14:31:49
trx,11,0.171,0.000,19-07-18 14:51:31
trx,notfound
Desired output
trx,11,0.171,0.000,19-07-18 14:51:31
Do not print it : save it.
Replace :
print data.encode('UTF-8')
if data == "trx,notfound": break
By
if data == "trx,notfound": break
last_data = data.encode('UTF-8')
Then, at the end (like after client.close()) you can print last_data
try
data_b = client.recv(1024)
while data_b:
data_b = client_socket.recv(1024)
data += data_b
instead of
data = client.recv(128)
example
import socket
def main():
port = 'Your port'
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind(("", int(port)))
server_socket.listen(5)
print('TCPServer Waiting for client on port '+port+'\n')
while True:
print('Listening ...', end='\r')
client, address = server_socket.accept()
print("Connection from ", address[0])
data = None
while True:
data_b = client.recv(1024)
data = data_b
if data_b:
print("Receiving a file...")
while data_b:
data_b = client_socket.recv(1024)
data += data_b
else:
break
print(data.decode())
client.close()
if __name__ == '__main__':
main()
Related
I am trying to send an encrypted file over TCP sockets in python, I don't want to have to encrypt the message, save it in %TEMP% and then send it (it could fill up hard drive space).
I am following this code I found online at: https://gist.github.com/giefko/2fa22e01ff98e72a5be2
Here is my server code:
from random import choice
import socket, os, threading, json
from cryptography.fernet import Fernet
chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPWRSTUVWXYZ1234567890!##$%^&*()"
#read the key or generate
key = b""
if os.path.exists("client.key"):
with open("client.key", "rb") as f:
key = f.read()
else:
with open("client.key", "wb") as f:
key = Fernet.generate_key()
f.write(key)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
port = 34467
host = "0.0.0.0"
s.bind((host, port))
print(f"LISTENING ON {host}:{port}")
s.listen(100)
def new_salt():
salt = ""
for x in range(15):
salt += choice(chars)
return salt
def handle_client(conn, addr):
encryption = False
def send_raw(content_type, Bytes_, salt=new_salt()):
seperator = "<|SEPERATE|>"
to_send = content_type + seperator + Bytes_.decode() + seperator + salt
to_send = to_send.encode()
if encryption:
to_send = Fernet(key).encrypt(to_send)
conn.send(to_send)
def recv_raw(BufferSize):
seperator = "<|SEPERATE|>".encode()
data = b""
while True:
data = conn.recv(BufferSize)
if data: break
if encryption:
data = Fernet(key).decrypt(data)
splitted = data.decode().split(seperator.decode())
content_type = splitted[0]
Bytes_ = splitted[1].encode()
salt = splitted[2]
return {"content_type": content_type, "bytes": Bytes_}
print("NEW CLIENT AT IP: " + str(addr[0]))
print("EXTANGING KEY")
send_raw("KEY", key)
client_key = recv_raw(1024)["bytes"]
if key == client_key:
print("KEY EXTANGE VERIFIED")
else:
print("UNABLE TO VERIFY, CLIENT MAY EXPERIENCE ISSUES")
print(key)
print(client_key)
encryption = True
print("GRAPPING SYSTEM INFO...")
sys_info_request = recv_raw(1024)
print("RECIVED, DECODING...")
sys_info = json.loads(sys_info_request["bytes"].decode())
print("BASIC INFO:")
print("Platoform: " + sys_info["platform"])
print("Architecture: " + str(sys_info["architecture"]))
print("Username: " + sys_info["username"])
if os.path.exists("autorun.txt"):
with open("autorun.txt", "r") as f:
print("FOUND AUTORUN, EXECUTING COMMANDS")
for line in f.readlines():
print("> " + line)
send_raw("command", line.encode())
output = recv_raw(1024)
print(output["bytes"].decode())
current_dir = sys_info["current_dir"]
while True:
try:
cmd = input(current_dir + "> " + sys_info["username"] + " $ ")
if cmd == "abort":
send_raw("abort", "".encode())
conn.close()
print("SAFE")
break
if cmd == "send_file":
# CODE GOES HERE
send_raw("command", cmd.encode())
output = recv_raw(1024)["bytes"].decode()
print(output)
except:
print("UNEXCPECTED ERROR")
while True:
conn, addr = s.accept()
threading.Thread(target=handle_client, args=(conn,addr,)).start()
I haven't found anything online that will work in my senario.
Okay, so you want to open a file, encrypt it and send it and avoid writing a tempfile to the hard disc, right? This works (taken from the example server code you linked):
while True:
conn, addr = s.accept() # Establish connection with client.
print('Got connection from', addr)
data = conn.recv(1024)
print('Server received', repr(data))
from cryptography.fernet import Fernet
key = Fernet.generate_key()
ff = Fernet(key)
filename='crs.py' #In the same folder or path is this file running must the file you want to tranfser to be
f = open(filename,'rb')
l = f.read(1024)
while (l):
enc = ff.encrypt(l)
conn.send(enc)
print('Sent ',repr(enc))
l = f.read(1024)
f.close()
print('Done sending')
conn.send(b'Thank you for connecting')
conn.close()
So, I am just opening the file, reading it 1024 bytes a time, encrypting it and then sending it along .. Does that answer your question?
When I start the program to share files to client it received it but when i request for another file download it failed with this error.
Now i keep getting this error from the client
socket1.send(bytes('0', 'UTF-8'))
BrokenPipeError: [Errno 32] Broken pipe
line 46 client.py
I tried breaking out of the server's filedownload loop but still not working.
server
#! /usr/bin/env python3
import socket
import sys
import os
import hashlib
import time
HOST = 127.0.0.1
PORT = 5000
c = 0 #used to count cycles
bufsize = 4096
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
print('Server Created')
except OSError as e:
print('Failed to create socket. Error Code : ' + str(msg[0]) + ' Message ' + msg[1])
sys.exit()
try:
s.bind((HOST, PORT))
except OSError as e:
print('Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1])
sys.exit()
print('Socket bind complete')
s.listen(1)
print('Server now listening')
while 1:
conn, addr = s.accept()
print('Connected with ' + addr[0] + ':' + str(addr[1]))
reqCommand = conn.recv(1024).decode("utf-8", errors='ignore')
print('Client> %s' % reqCommand)
string = reqCommand.split(' ', 1)
if reqCommand == 'quit':
break
elif reqCommand == 'lls':
toSend = ""
path = os.getcwd()
dirs = os.listdir(path)
for f in dirs:
toSend = toSend + f + ' '
conn.send(toSend.encode('utf-8'))
# print path
else:
string = reqCommand.split(' ', 1) # in case of 'put' and 'get' method
if len(string) > 1:
reqFile = string[1]
if string[0] == 'FileDownload':
with open(reqFile, 'rb') as file_to_send1:
# get the entire filesize, which sets the read sector to EOF
file_size = len(file_to_send1.read())
# reset the read file sector to the start of the file
file_to_send1.seek(0)
# take filesize and write it to a temp file
with open('temp01',mode='w', encoding='UTF-8') as file_to_send2:
file_to_send2.write(str(file_size))
# pass the file size over to client in a small info chunk
with open('temp01', 'rb') as file_to_send3:
conn.send(file_to_send3.read(1024))
#send the total file size off the client
while (c*bufsize) < file_size:
send_data = file_to_send1.read(bufsize)
conn.send(send_data)
c += 1
# get bool (0 is bad | 1 is good) from client
chunk_write_flag = int(conn.recv(1024))
while chunk_write_flag != 1: #while whole data was not sent..retry until successful
conn.send(send_data)
#get status from client after a retry
chunk_write_flag = int(conn.recv(1024))
# used on the last chunk of the file xfer
# if the file.read() is less than buffer size do last tasks
if (file_size - (c*bufsize)) < bufsize:
send_data = file_to_send1.read(bufsize)
conn.send(send_data)
file_to_send1.close()
break
#for data in file_to_send:
#conn.sendall(data)
print('Send Successful')
conn.close()
s.close()
client
#! /usr/bin/env python3
import socket
import sys
import os
import hashlib
import time
HOST = 127.0.0.1
PORT = 5000
c = 0
bufsize = 4096
def get(commandName):
socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName.encode("utf-8"))
string = commandName.split(' ', 1)
inputFile = string[1]
c = 0
# before starting to write new file, get file size
file_size = int(socket1.recv(1024)) # from file_to_send3
# print (file_size)
# set byte buffer size
bufsize = 4096
# start writing at the beginning and use following variable to track
write_sectors = 0
# this only opens the file, the while loop controls when to close
with open(inputFile, 'wb+') as file_to_write:
# while written bytes to out is less than file_size
while write_sectors < file_size:
# write the BUFSIZE while the write_sector is less than file_size
file_to_write.write(socket1.recv(bufsize))
c += 1
with open(inputFile, 'rb') as verify:
write_check = (len(verify.read()) / c)
verify.seek(0) # read cycle moves seek location, reset it
while write_check != bufsize:
# where the original write started, to send back to server
if c > 1: file_to_write.seek((c-1) * bufsize)
if c == 1: file_to_write.seek(0)
# send to server that the write was not successful
socket1.send(bytes('0', 'UTF-8'))
file_to_write.write(socket1.recv(bufsize))
write_check = int(len(verify.read()) /c )
# if last packet, smaller than bufsize
socket1.send(bytes('1', 'UTF-8')) #send SUCCESS back to server
if (file_size - (write_check * c)) < bufsize:
#file_to_write.write(socket1.recv(bufsize))
verify.close()
#file_to_write.close()
file_size = 0
write_sectors += bufsize # successful write, move 'while' check
# add the written sectors by the bufsize.
# example if filesize in bytes is 4096 you need to track how much
# was written to figure out where the EOF is
file_to_write.write(socket1.recv(bufsize)) # write the last chunk missed by while loop
#data = socket1.recv(4096).decode("utf-8", errors="ignore")
#if not data: break
#break
# print data
#file_to_write.write(bytes(data.encode()))
#file_to_write.close()
print('Download Successful')
socket1.close()
return
def serverList(commandName):
socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName.encode('utf-8'))
fileStr = socket1.recv(1024)
fileList = fileStr.decode('utf-8').split(' ')
for f in fileList:
print(f)
socket1.close()
return
msg = input('Enter your name: ')
while 1:
print("\n")
print('"FileDownload [filename]" to download the file from the server ')
print('"lls" to list all files in the server')
sys.stdout.write('%s> ' % msg)
inputCommand = sys.stdin.readline().strip()
if inputCommand == 'lls':
serverList('lls')
else:
string = inputCommand.split(' ', 1)
if string[0] == 'FileDownload':
Please can anyone help me,i don't know to fix it. I'll appreciate any help.
Thanks
Server.py
class BufferedReceiver():
def __init__(self, sock):
self.sock = sock
self.buffer = ""
self.bufferPos = 0
def _fetch(self):
while self.bufferPos >= len(self.buffer):
self.buffer = self.sock.recv(1024)
# print(self.buffer)
self.bufferPos = 0
def take(self, amount):
result = bytearray()
while(len(result) < amount):
# Fetch new data if necessary
self._fetch()
result.append(self.buffer[self.bufferPos])
self.bufferPos += 1
return bytes(result)
def take_until(self, ch):
result = bytearray()
while True:
# Fetch new data if necessary
self._fetch()
nextByte = self.buffer[self.bufferPos]
self.bufferPos += 1
result.append(nextByte)
if bytes([nextByte]) == ch:
break
return bytes(result)
Then, I simplified your server send routine after the else::
string = reqCommand.split(' ', 1) # in case of 'put' and 'get' method
if len(string) > 1:
reqFile = string[1]
if string[0] == 'FileDownload':
with open(reqFile, 'rb') as file_to_send1:
# get the entire filesize, which sets the read sector to EOF
file_size = len(file_to_send1.read())
# reset the read file sector to the start of the file
file_to_send1.seek(0)
# pass the file size over to client in a small info chunk
print('Filesize:', file_size)
conn.send((str(file_size)+'\n').encode())
#send the total file size off the client
c = 0
while (c*bufsize) < file_size:
send_data = file_to_send1.read(bufsize)
conn.send(send_data)
c += 1
print('Send Successful')
Client.py
def get(commandName):
socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName.encode("utf-8"))
receiver = BufferedReceiver(socket1)
string = commandName.split(' ', 1)
inputFile = string[1]
# before starting to write new file, get file size
file_size = int(receiver.take_until(b'\n').decode().strip()) # from file_to_send3
print ('Filesize:', file_size)
# set byte buffer size
bufsize = 4096
# this only opens the file, the while loop controls when to close
with open(inputFile, 'wb+') as file_to_write:
# while written bytes to out is less than file_size
c = 0
while True:
# Compute how much bytes we have left to receive
bytes_left = file_size - bufsize * c
# If we are almost done, do the final iteration
if bytes_left <= bufsize:
file_to_write.write(receiver.take(bytes_left))
break
# Otherwise, just continue receiving
file_to_write.write(receiver.take(bufsize))
c += 1
#TODO open file again, verify.
# Generate MD5 on server while sending. Then, send MD5 to client.
# Open finished file in client again and compare MD5s
print('Download Successful')
socket1.close()
return
I am trying to implement a peer to peer file transfer protocol and i came across this code but when I ported it to python 3.6 I got this error "type error a bytes-like object is required not 'str'". Please can somebody help me out i am new to this.
server.py
! /usr/bin/python3.6
import subprocess
import socket
import sys
import os
import hashlib
HOST = 'localhost'
PORT = 8000
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print('Server Created')
except OSError as e:
print('Failed to create socket. Error Code : ' + str(msg[0]) + ' Message ' + msg[1])
sys.exit()
try:
s.bind((HOST, PORT))
except OSError as e:
print('Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1])
sys.exit()
print('Socket bind complete')
s.listen(1)
print('Server now listening')
while (1):
conn, addr = s.accept()
print('Connected with ' + addr[0] + ':' + str(addr[1]))
reqCommand = conn.recv(1024)
print('Client> %s' %(reqCommand))
string = reqCommand.split(' ')
if (reqCommand == 'quit'):
break
elif (reqCommand == 'lls'):
toSend=""
path = os.getcwd()
dirs=os.listdir(path)
for f in dirs:
toSend=toSend+f+' '
conn.send(toSend)
#print path
elif (string[1]== 'shortlist'):
path = os.getcwd()
command = 'find '+path+ ' -type f -newermt '+string[2]+' '+string[3]+ ' ! -newermt '+string[4]+' '+string[5]
var = commands.getstatusoutput(command)
var1 = var[1]
var=var1.split('\n')
rslt = ""
for i in var:
comm = "ls -l "+i+" | awk '{print $9, $5, $6, $7, $8}'"
tup=commands.getstatusoutput(comm)
tup1=tup[1]
str=tup1.split(' ')
str1=str[0]
str2=str1.split('/')
rslt=rslt+str2[-1]+' '+str[1]+' '+str[2]+' '+str[3]+' '+str[4]+'\n'
conn.send(rslt)
elif (string[1]=='longlist'):
path = os.getcwd()
var= commands.getstatusoutput("ls -l "+path+" | awk '{print $9, $5, $6, $7, $8}'")
var1 = ""
var1= var1+''+var[1]
conn.send(var1)
elif (string[0] == 'FileHash'):
if(string[1]== 'verify'):
BLOCKSIZE = 65536
hasher = hashlib.sha1()
with open(string[2], 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
conn.send(hasher.hexdigest())
print('Hash Successful')
elif (string[1] == 'checkall'):
BLOCKSIZE = 65536
hasher = hashlib.sha1()
path = os.getcwd()
dirs=os.listdir(path)
for f in dirs:
conn.send(f)
with open(f, 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
conn.send(hasher.hexdigest())
print('Hash Successful')
else:
string = reqCommand.split(' ') #in case of 'put' and 'get' method
if(len(string) > 1):
reqFile = string[1]
if (string[0] == 'FileUpload'):
file_to_write = open(reqFile,'wb')
si = string[2:]
for p in si:
p = p + " "
print("User" + p)
file_to_write.write(p)
while True:
data = conn.recv(1024)
print("User" + data)
if not data:
break
file_to_write.write(data)
file_to_write.close()
print('Receive Successful')
elif (string[0] == 'FileDownload'):
with open(reqFile, 'rb') as file_to_send:
for data in file_to_send:
conn.sendall(data)
print('Send Successful')
conn.close()
s.close()
This is the error from the server.
Server Created
Socket bind complete
Server now listening
Connected with 127.0.0.1:37760
Client> b''
Traceback (most recent call last):
File "./server.py", line 36, in <module>
string = reqCommand.split(' ')
TypeError: a bytes-like object is required, not 'str'
This is the client side of my program but having the same problem.
client.py
#! /usr/bin/python3.6
import socket
import sys
import os
import hashlib
HOST = 'localhost' #server name goes in here
PORT = 8000
def put(commandName):
socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName)
string = commandName.split(' ', 1)
string = commandName.split(' ')
inputFile = string[1]
with open(inputFile, 'rb') as file_to_send:
for data in file_to_send:
socket1.sendall(data)
print("Client users " + data)
socket1.send(data)
print('Upload Successful')
socket1.close()
return
def get(commandName):
socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName(data. 'utf-8'))
string = commandName.split(' ')
inputFile = string[1]
with open(inputFile, 'wb') as file_to_write:
while True:
data = socket1.recv(1024)
if not data:
break
# print data
file_to_write.write(data)
file_to_write.close()
print('Download Successful')
socket1.close()
return
def FileHash(commandName):
string = commandName.split(' ')
if string[1] == 'verify':
verify(commandName)
elif string[1] == 'checkall':
checkall(commandName)
def verify(commandName):
socket1=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName)
hashValServer=socket1.recv(1024)
string = commandName.split(' ')
BLOCKSIZE = 65536
hasher = hashlib.sha1()
with open(string[2], 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
hashValClient = hasher.hexdigest()
print('hashValServer= %s', (hashValServer))
print('hashValClient= %s', (hashValClient))
if hashValClient == hashValServer:
print('No updates')
else:
print('Update Available')
socket1.close()
return
def checkall(commandName):
socket1=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName)
string = commandName.split(' ')
BLOCKSIZE = 65536
hasher = hashlib.sha1()
# f=socket1.recv(1024)
while True:
f=socket1.recv(1024)
with open(f, 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
hashValClient = hasher.hexdigest()
hashValServer=socket1.recv(1024)
print ('Filename = %s', f)
print('hashValServer= %s', (hashValServer))
print('hashValClient= %s', (hashValClient))
if hashValClient == hashValServer:
print('No updates')
else:
print('Update Available')
if not f:
break
socket1.close()
return
def quit():
socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName)
socket1.close()
return
def IndexGet(commandName):
socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
string = commandName.split(' ')
if string[1] == 'shortlist':
socket1.send(commandName)
strng=socket1.recv(1024)
strng=strng.split('\n')
for f in strng:
print(f)
elif (string[1]=='longlist'):
socket1.send(commandName)
path=socket1.recv(1024)
rslt=path.split('\n')
for f in rslt[1:]:
print(f)
socket1.close()
return
def serverList(commandName):
socket1=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName)
fileStr=socket1.recv(1024)
fileList=fileStr.split(' ')
for f in fileList[:-1]:
print(f)
socket1.close()
return
msg = input('Enter your name: ')
while(1):
print("\n")
print("****************")
print('Instruction')
print('"FileUpload [filename]" to send the file the server ')
print('"FileDownload [filename]" to download the file from the server ')
print('"ls" to list all files in this directory')
print('"lls" to list all files in the server')
print('"IndexGet shortlist <starttimestamp> <endtimestamp>" to list the files modified in mentioned timestamp.')
print('"IndexGet longlist" similar to shortlist but with complete file listing')
print('"FileHash verify <filename>" checksum of the modification of the mentioned file.')
print('"quit" to exit')
print("\n")
sys.stdout.write ('%s> ' %msg)
inputCommand = sys.stdin.readline().strip()
if (inputCommand == 'quit'):
quit()
break
elif (inputCommand == 'ls'):
path = os.getcwd()
dirs = os.listdir(path)
for f in dirs:
print(f)
elif (inputCommand == 'lls'):
serverList('lls')
else:
string = inputCommand.split(' ', 1)
if string[0] == 'FileDownload':
get(inputCommand)
elif string[0] == 'FileUpload':
put(inputCommand)
elif string[0] =='IndexGet':
IndexGet(inputCommand)
elif string[0] == 'FileHash':
FileHash(inputCommand)
I expexted it to transfer the file without any error, Please i am new to this can anybody help me out.
This the error from the client
FileUpload closer.mp3
g> Traceback (most recent call last):
File "./client.py", line 193, in <module>
put(inputCommand)
File "./client.py", line 16, in put
socket1.send(commandName)
TypeError: a bytes-like object is required, not 'str'
You're trying to send a string even though a byte array is needed.
in 'socket1.send(commandName)' write socket1.send(commandName.encode('utf-8') instead
same thing serverside.
You should really rethink coding a cryptocurrency if you can't figure out the difference between bytes and strings. Start with an easier project.
this code I use below is a ZMQ sub to a publisher that is giving me data. It uses the counter to tell me when its 30 and 59 seconds to run my write to CSV every 30 seconds or so.
Problem: I am now timing all of the processes in my thread. the lines where message and message2 = socket.recv_string() is taking anywhere from half a second to 20 seconds to receive string. Thus causing the thread to miss the 30 and 59 second intervals I set. This was not happening yesterday. The other timers for the if statements are taking .00001 or 0.0 seconds. So that part isnt the problem
Im wondering what could effect this. Could it be the processing power of my computer? Or is the receive string based on how long it waits for the publisher to actually send something?
I'm not running in a dev or production environment and its on a shared server with something like 15 other people and its virtual. A zero client. I've never had this problem before and on another script i have set up for another ZMQ pub/sub I'm receiving messages in .01 or .001 seconds all the way to 3 seconds. Which is more manageable but the norm was .01.
Any tips or help would be amazing. Thanks in advance
import zmq
import pandas as pd
import time
import threading
df_fills = pd.DataFrame()
df_signal = pd.DataFrame()
second_v = [30,59]
s = 0
m = 0
h = 0
d = 0
def counter():
global h,s,m,d
while True:
s += 1
#print("Second:{}".format(s))
if s >=60:
m +=1
s = 0
if m >= 60:
h += 1
m = 0
if h >= 24:
d += 1
h = 0
#print(s)
time.sleep(1)
class zmq_thread(threading.Thread):
def __init__(self,name):
threading.Thread.__init__(self)
self.name = name
def run(self):
global df_fills, second_v,s
print('zmq started')
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect(SERVER)
socket.setsockopt_string(zmq.SUBSCRIBE,'F')
print('socket connected')
tickers = [a bunch of tickers]
while True:
try:
start2 = time.time()
if s == 30:
print('break')
if df_fills.empty == True:
print('running fill thread again')
z = zmq_thread('Start_ZMQ')
#time.sleep(.7)
z.run()
else:
start = time.time()
print('writing fills')
filename = "a CSV"
with open(filename, 'a') as f:
df_fills.to_csv(f, encoding = 'utf-8', index = False, header = False)
f.close()
print('wrote fills')
end = time.time()
print(end-start)
df_fills = df_fills.iloc[0:0]
z = zmq_thread('Start_ZMQ')
z.run()
return df_fills
end2 = time.time()
print(end2-start2)
start3 = time.time()
message = socket.recv_string()
message2 = socket.recv_string()
end3 = time.time()
print(end3-start3, 'message timing')
print(s)
start1 = time.time()
if message == 'F':
# message2_split = message2.split("'")
message2_split = message2.split(";")
message3_split = [e[3:] for e in message2_split]
message4 = pd.Series(message3_split)
if message4[0] in tickers:
df_fills = df_fills.append(message4, ignore_index=True)
print('fill')
end1 = time.time()
print(end1-start1)
except KeyboardInterrupt:
break
counter = threading.Thread(target = counter)
zmq_loop = zmq_thread('Start_ZMQ')
#%%
counter.start()
zmq_loop.start()
I didn't realize that ZMQ typical recv_string is by default blocking. So I did this
message = socket.recv_string(flags = zmq.NOBLOCK)
message2 = socket.recv_string(flags = zmq.NOBLOCK)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
pass
else:
if message == 'ABA_BB':
message2_split = message2.split(";")
message3_split = [e[3:] for e in message2_split]
message4 = pd.Series(message3_split)
#print(message4)
if message4[2] == '300':
df_signal = df_signal.append(message4, ignore_index=True)
print('Signal Appended')
I have been fighting with a threaded send of an string image over python sockets for a while now and have had no luck on this issue.
code for the client side is:
import socket
from PIL import ImageGrab #windows only screenshot
from threading import Thread
import win32api, win32con
import re
import win32com.client
import getpass
import time
import select
shell = win32com.client.Dispatch("WScript.Shell")
host = raw_input("SERVER:")
dm = win32api.EnumDisplaySettings(None, 0)
dm.PelsHeight = 800
dm.PelsWidth = 600
win32api.ChangeDisplaySettings(dm, 0)
port = 9000
def picture():
while 1:
image = ImageGrab.grab().resize((800,600)) #send screen as string
data = image.tostring()
sendme = (data)
try:
s.sendall(sendme)
print ("sent")
except socket.error as e:
print e
except Exception as e:
print e
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
pict = Thread(target=picture)
pict.start()
while 1:
socket_list = [s]
# Get the list sockets which are readable
read_sockets, write_sockets, error_sockets = select.select(socket_list , [], [])
for sock in read_sockets:
if sock == s:
data = sock.recv(1024)
print data
if "LEFTC" in data:
data = data.replace("LEFTC","")
x = re.findall(r'X(.*?)Y',data)
y = re.findall(r'Y(.*?)EOC',data)
x = str(x)
y = str(y)
#REPLACE CODE TO BE REWRITTEN
x = x.replace("[","").replace("]","").replace("'","").replace(" ","")
y = y.replace("[","").replace("]","").replace("'","").replace(" ","")
print(str(x) + ' X\n')
print(str(y) + ' Y\n')
try:
win32api.SetCursorPos((int(x),int(y))) #click time
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN,int(x),int(y),0,0)
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP,int(x),int(y),0,0)
except Exception as e:
print e
elif "RIGHTC" in data:
data = data.replace("RIGHTC","")
x = re.findall(r'X(.*?)Y',data)
y = re.findall(r'Y(.*?)EOC',data)
x = str(x)
y = str(y)
#REPLACE FUNCTION MAREKD FOR REWRITE
x = x.replace("[","").replace("]","").replace("'","").replace(" ","")
y = y.replace("[","").replace("]","").replace("'","").replace(" ","")
print(str(x) + ' X\n')
print(str(y) + ' Y\n')
try: #click
win32api.SetCursorPos((int(x),int(y)))
win32api.mouse_event(win32con.MOUSEEVENTF_RIGHTDOWN,int(x),int(y),0,0)
win32api.mouse_event(win32con.MOUSEEVENTF_RIGHTUP,int(x),int(y),0,0)
except Exception as e:
print e
else:
#This does not work correctly: only BACKSPACE and the else are working.
if "CAPS" in data:
shell.SendKeys('{CAPSLOCK}')
elif "CAPSOFF" in data:
shell.SendKeys('{CAPSLOCK}')
elif "BACKSPACE" in data:
shell.SendKeys('{BACKSPACE}')
elif "SHIFT" in data:
shell.SendKeys('+' + data)
else:
shell.SendKeys(data)
time.sleep(0.1)
server code is:
import socket
import pygame
from pygame.locals import *
from threading import Thread
x = y = 0
host = ""
#port defined here
port = 9000
#This list is used to make the library more pythonic and compact. This also leads to less source code.
keylist = [pygame.K_a,pygame.K_b,pygame.K_c,pygame.K_d,pygame.K_e,pygame.K_f,pygame.K_g,pygame.K_h,pygame.K_i,pygame.K_j,pygame.K_k,pygame.K_l,pygame.K_m,pygame.K_n,pygame.K_o,pygame.K_p,pygame.K_q,pygame.K_r,pygame.K_s,pygame.K_t,pygame.K_u,pygame.K_v,pygame.K_w,pygame.K_x,pygame.K_y,pygame.K_z]
key = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','1','2','3','4','5','6','7','8','9','0']
i/o function
def ioinput(sock):
while 1:
evt = pygame.event.poll() #has to be in the same while loop as the evt called or wont work.
if evt.type == pygame.MOUSEBUTTONDOWN and evt.button == 1: # one for left
x, y = evt.pos
command = ("LEFTC" + " " + "X" + str(x) + "Y" + str(y) + "EOC")
sock.sendall(command)
elif evt.type == pygame.MOUSEBUTTONDOWN and evt.button == 3: # 3 for right 2 is middle which support comes for later.
x, y = evt.pos
command = ("RIGHTC" + " " + "X" + str(x) + "Y" + str(y) + "EOC")
sock.sendall(command)
elif evt.type == pygame.KEYDOWN:
keyname = pygame.key.name(evt.key)
if evt.key == pygame.K_BACKSPACE:
command = ("BACKSPACE")
sock.sendall(command)
elif evt.key in keylist:
if keyname in key:
command = (keyname)
sock.sendall(command)
def mainloop():
message = []
while 1:
try:
while True:
try:
conn, addr = server.accept()
except socket.error:
break
screen = pygame.display.set_mode((800,600))
clickctrl = Thread(target=ioinput, args=(conn,))
clickctrl.start()
while 1:
d = conn.recv(1024*1024*1)
if not d:
break
else:
message.append(d)
data = ''.join(message)
image = pygame.image.frombuffer(data,(800,600),"RGB")
screen.blit(image,(0,0))
pygame.display.flip()
except Exception as e:
continue
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.setblocking(False)
server.bind((host, port))
server.listen(55000)
print "Listening on %s" % ("%s:%s" % server.getsockname())
Main event loop.
mainloop()
The picture thread will run 3 to six times then die however the keyboard and mouse input layer continues to operate. I suspect that the GIL is getting in my way. Am i correct or am I missing something really simple here? This program is supposed to be a simplistic reverse remote desktop appication.
I found the problem after speaking with a good friend. turns out that my server side while loop was setup so that it would break.
i fixed this by changing:
while 1:
d = conn.recv(1024*1024*1)
if not d:
break
else:
message.append(d)
data = ''.join(message)
image = pygame.image.frombuffer(data,(800,600),"RGB")
screen.blit(image,(0,0))
pygame.display.flip()
to :
while 1:
d = conn.recv(1024*1024*1)
message.append(d)
try:
print("attempting to parse..")
data = ''.join(message)
image = pygame.image.frombuffer(data,(800,600),"RGB")
screen.blit(image,(0,0))
pygame.display.flip()
print("recieved pic")
except Exception as e:
print e
continue
Also, client side on the picture thread i added a time.sleep (1) after the exception handling, otherwise the image does not come though correctly.