I am using CMD2 to create a server. I have broken my code down to the smallest bit of code that still produces the problem.
import socket
import _thread
from cmd2 import *
def grab_data(conn):
while True:
try:
data = conn.recv(1024)
print(data)
except:
print("disconnected.")
break
def grab_client(sock):
while True:
conn, addr = sock.accept()
print("New connection")
_thread.start_new_thread(grab_data, (conn,))
def start_conn(ip, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((ip, int(port)))
sock.listen(10)
print("Socket listening")
_thread.start_new_thread(grab_client, (sock,))
class CLI(Cmd):
def __init__(self):
Cmd.__init__(self)
self.prompt = "Prompt> "
def do_listen(self, arg):
start_conn('0.0.0.0', '4446')
def emptyline(self):
pass
def do_quit(self, arg):
return True
cli = CLI()
cli.cmdloop("Welcome to the server.")
The issue I run into is when a client connects. It does not reprint the prompt. It hangs at a empty line with just the cursor. I am not sure how to get the prompt to print back.
You're blocking trying to read 1024 bytes, so it's got to wait until that entire buffer is filled. To do proper line-based buffering is a bit tricky, but a simple (albeit non-performant) implementation is to read a character at a time and check if it's a newline:
line = ""
while True:
data = conn.recv(1)
line += data
if data in "\n\r":
break
(This is not great code, but let's see if that solves the problem and we can improve it.)
Related
Into:
I am working in a TCP server that receives events over TCP. For this task, I decided to use asyncio Protocol libraries (yeah, maybe I should have used Streams), the reception of events works fine.
Problem:
I need to be able to connect to the clients, so I create another "server" used to look up all my connected clients, and after finding the correct one, I use the Protocol class transport object to send a message and try to grab the response by reading a buffer variable that always has the last received message.
My problem is, after sending the message, I don't know how to wait for the response, so I always get the previous message from the buffer.
I will try to simplify the code to illustrate (please, keep in mind that this is an example, not my real code):
import asyncio
import time
CONN = set()
class ServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
self.transport = transport
CONN.add(self)
def data_received(self, data):
self.buffer = data
# DO OTHER STUFF
print(data)
def connection_lost(self, exc=None):
CONN.remove(self)
class ConsoleProtocol(asyncio.Protocol):
def connection_made(self, transport):
self.transport = transport
# Get first value just to ilustrate
self.client = next(iter(CONN))
def data_received(self, data):
# Forward the message to the client
self.client.transport.write(data)
# wait a fraction of a second
time.sleep(0.2)
# foward the response of the client
self.transport.write(self.client.buffer)
def main():
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(
loop.create_server(protocol_factory=ServerProtocol,
host='0.0.0.0',
port=6789))
loop.run_until_complete(
loop.create_server(protocol_factory=ConsoleProtocol,
host='0.0.0.0',
port=9876))
try:
loop.run_forever()
except Exception as e:
print(e)
finally:
loop.close()
if __name__ == '__main__':
main()
This is not only my first experience writing a TCP server, but is also my first experience working with parallelism. So it took me days to realize that my sleep not only would not work, but I was locking the server while it "sleeps".
Any help is welcome.
time.sleep(0.2) is blocking, should not used in async programming, which will block the whole execution, if your program runing with 100 clients, the last client will be delayed for 0.2*99 seconds, which is not what you want.
the right way is trying to let program wait 0.2s but not blocking, then other concurrent clients would not be delayed,we can use thread.
import asyncio
import time
import threading
CONN = set()
class ServerProtocol(asyncio.Protocol):
def dealy_thread(self):
time.sleep(0.2)
def connection_made(self, transport):
self.transport = transport
CONN.add(self)
def data_received(self, data):
self.buffer = data
# DO OTHER STUFF
print(data)
def connection_lost(self, exc=None):
CONN.remove(self)
class ConsoleProtocol(asyncio.Protocol):
def connection_made(self, transport):
self.transport = transport
# Get first value just to ilustrate
self.client = next(iter(CONN))
def data_received(self, data):
# Forward the message to the client
self.client.transport.write(data)
# wait a fraction of a second
thread = threading.Thread(target=self.delay_thread, args=())
thread.daemon = True
thread.start()
# foward the response of the client
self.transport.write(self.client.buffer)
I am trying to write some non-blocking FIFO code with kqueue on my BSD machine. Here's the small server code: server.py
import os
import selectors
sel = selectors.KqueueSelector()
TMP_PATH="/tmp/myfifo"
def fifo_read(fd, mask):
data = os.read(fd, 8)
print("fd:{} gives:{} \n", fd, data)
sel.unregister(fd)
print("unregistered")
def fifo_accept(listen_fd, mask):
print("accepted {}".format(listen_fd))
fd = os.dup(listen_fd)
print("duped to {}".format(fd))
sel.register(fd, selectors.EVENT_READ, fifo_read)
if __name__ == "__main__":
try:
os.unlink(TMP_PATH)
except:
pass
os.mkfifo(TMP_PATH)
listen_fd = os.open(TMP_PATH, os.O_RDONLY, mode=0o600)
sel.register(listen_fd, selectors.EVENT_READ, fifo_accept)
while True:
events = sel.select()
for key, mask in events:
cb = key.data
cb(key.fileobj, mask)
sel.close()
Now, when I run a client.py as:
import os
TMP_PATH="/tmp/myfifo"
fd = os.open(TMP_PATH, os.O_WRONLY, mode=0o600)
res = os.write(fd, b"1234567")
print("sent {}".format(res))
When I run the client, I get:
sent 7
But on server, it runs to inifinite loop. Now I understand why the infinite loop is happening. I actually tried mimicking the socket way of using selectors in this Python Docs example.
Here's what I have tried:
I did try the code without duplicating the fd, but it's still in infinite loop.
I tried calling sel.unregister on the original listen_fd, but in this case, running the client the second time doesn't work (which is expected).
Can anyone please let me know if I'm missing something?
So I found one solution to this problem. With sockets, we get a new socket object on accept. So we need to emulate that behaviour by calling unregister on the original fileobj, open again and call register on that.
Fixed code:
import os
import selectors
sel = selectors.KqueueSelector()
try:
os.unlink("./myfifo")
except:
pass
os.mkfifo("./myfifo", 0o600)
def cb(fp):
sel.unregister(fp)
print(f"got {fp.read()}")
fp.close()
fp2 = open("./myfifo", "rb")
sel.register(fp2, selectors.EVENT_READ, cb)
if __name__ == "__main__":
orig_fp = open("./myfifo", "rb")
print("open done")
ev = sel.register(orig_fp, selectors.EVENT_READ, cb)
print(f"registration done for {ev}")
while True:
events = sel.select()
print(events)
for key, mask in events:
key.data(key.fileobj)
I am trying to build a program for my IT course. The point of the program is to have a client app to send commands to the server. It seemd to work pretty well until today where, after a few calls, when I receive a response from the server it is not up to date.
eg : I send a few commands that all work fine. But then send another command and receive the response from the previous one.
I checked the command sent by the client and it is the one I type and in the server part, when I receive a command from the client it is the one actually sent by the client (not the previous one)
Here is the Shell classes (in the server and client) that I use to send and receive messages aswell as an example on how I use it.
Server :
class Shell:
command = ""
next_command = True
def __init__(self, malware_os):
self._os = malware_os
self._response = ""
def receive(self):
self.command = distant_socket.recv(4096).decode("utf-8")
def execute_command(self):
if self.command[:2] == "cd":
os.chdir(self.command[3:])
if self._os == "Windows":
self.result = Popen("cd", shell=True, stdout=PIPE)
else:
self.result = Popen("pwd", shell=True, stdout=PIPE)
else:
self.result = Popen(self.command, shell=True, stdout=PIPE)
self._response = self.result.communicate()
def send(self):
self._response = self._response[0]
self._response = self._response.decode("utf-8", errors="ignore")
self._response = self._response + " "
self._response = self._response.encode("utf-8")
distant_socket.send(self._response)
self._response = None
Use in server :
shell.receive()
shell.execute_command()
shell.send()
Client :
class Shell:
def __init__(self):
self._history = []
self._command = ""
def send(self):
self._history.append(self._command)
s.send(self._command.encode("utf-8"))
def receive(self):
content = s.recv(4096).decode("utf-8", errors="ignore")
if content[2:] == "cd":
malware_os.chdir(self._command[3:].decode("utf-8", errors="ignore"))
print(content)
def history(self):
print("The history of your commands is:")
print("----------------------")
for element in self._history:
print(element)
def get_command(self):
return self._command
def set_command(self, command):
self._command = command
Use in client :
shell.set_command(getinfo.get_users())
shell.send()
shell.receive()
Thank you in advance for your help,
Cordially,
Sasquatch
Since you said the response is not up to date, I'm guessing you used TCP (you didn't post the socket creation). Like the comment mentioned, there are 2 things that you aren't doing right:
Protocol: TCP gives you a stream, which is divided as the OS sees fit into packets. When transferring data over the network, the receiving end must know when it has a complete transmission. The easiest way to do that would be to send the length of the transmission, in a fixed format (say 4 bytes, big endian), before the transmission itself. Also, use sendall. For example:
import struct
def send_message(sock, message_str):
message_bytes = message_str.encode("utf-8")
size_prefix = struct.pack("!I", len(message_bytes)) # I means 4 bytes integer in big endian
sock.sendall(size_prefix)
sock.sendall(message_bytes)
Since TCP is a stream socket, the receiving end might return from recv before the entire message was received. You need to call it in a loop, checking the return value at every iteration to correctly handle disconnects. Something such as:
def recv_message_str(sock):
#first, get the message size, assuming you used the send above
size_buffer = b""
while len(size_buffer) != 4:
recv_ret = sock.recv(4 - len(size_buffer))
if len(recv_ret) == 0:
# The other side disconnected, do something (raise an exception or something)
raise Exception("socket disconnected")
size_buffer += recv_ret
size = struct.unpack("!I", size_buffer)[0]
# Loop again, for the message string
message_buffer = b""
while len(message_buffer) != size:
recv_ret = sock.recv(size - len(message_buffer))
if len(recv_ret) == 0:
# The other side disconnected, do something (raise an exception or something)
raise Exception("socket disconnected")
message_buffer += recv_ret
return message_buffer.decode("utf-8", errors="ignore")
The code below used to work in spyder to communicate over serial with an arduino. In the console window of spyder, I would see lines of data being printed out:
78.7,77.9,100,80
78.7,77.9,100,80
78.7,77.9,100,80
78.7,77.9,100,80 ...
Data is from two temperature probes, a flowmeter, and the thermostat set temp.
I upgraded my Kubuntu 18.04 system to all things python3. Now, the code runs, but the spyder3 console window shows no visible characters, but scrolls blank lines. The rest of my python code for parsing and plotting this data doesn't work.
I've spent all day trying to fix this with no luck. I'm guessing it's a simple fix for someone with more experience than me.
The only difference between the old working code and the code below is that the print statements have parentheses added to remove the syntax error.
python
""" This code was originally copied from:
Listen to serial, return most recent numeric values
Lots of help from here:
http://stackoverflow.com/questions/1093598/pyserial-how-to-read-last-line-sent-from-serial-device
"""
from threading import Thread
import time
import serial
last_received = ''
def receiving(ser):
global last_received
buffer = ''
while True:
buffer = buffer + ser.read(ser.inWaiting())
if '\n' in buffer:
lines = buffer.split('\n') # Guaranteed to have at least 2 entries
last_received = lines[-2]
#If the Arduino sends lots of empty lines, you'll lose the
#last filled line, so you could make the above statement conditional
#like so: if lines[-2]: last_received = lines[-2]
buffer = lines[-1]
class SerialData(object):
def __init__(self, init=50):
try:
self.ser = serial.Serial(
port='/dev/ttyACM0',
baudrate=9600,
bytesize=serial.EIGHTBITS,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
timeout=0.1,
xonxoff=0,
rtscts=0,
interCharTimeout=None
)
except serial.serialutil.SerialException:
#no serial connection
self.ser = None
else:
Thread(target=receiving, args=(self.ser,)).start()
def next(self):
if not self.ser:
return '81.3,78.1,10.0,60.0,0' #100 #return anything so we can test when Arduino isn't connected
#return a float value or try a few times until we get one
for i in range(40):
raw_line = last_received
try:
# return float(raw_line.strip())
return str(raw_line.strip())
except ValueError:
print('bogus data',raw_line)
time.sleep(.005)
return 0.
def __del__(self):
if self.ser:
self.ser.close()
def write(self,val):
self.ser.write(val)
if __name__=='__main__':
s = SerialData()
for i in range(500):
time.sleep(.015)
print( s.next())
One of the most significant differences between Python 2.x and 3.x is the way text strings are encoded. For Python 3.x everything is Unicode, as compared to ASCII for 2.x, so you just need to decode the raw bytes you read from the serial port:
buffer = buffer + ser.read(ser.inWaiting()).decode('utf-8')
EDIT: now you seem to have a different problem involving an exception. It looks like your port is open, to be sure you can change the way you handle the exception when you instantiate the port:
except serial.serialutil.SerialException as e:
print(e)
self.ser = None
Once you know the error you should be able to handle it. Quite likely your port was not properly closed in an earlier session.
Bingo! That last suggestion fixed the program and the python GUI interface I wrote for it is working with these corrections.
python
from threading import Thread
import time
import serial
last_received = ''
def receiving(ser):
global last_received
buffer = ''
while True:
buffer = buffer + ser.read(ser.inWaiting()).decode('utf-8')
if '\n' in buffer:
lines = buffer.split('\n') # Guaranteed to have at least 2 entries
last_received = lines[-2]
#If the Arduino sends lots of empty lines, you'll lose the
#last filled line, so you could make the above statement conditional
#like so: if lines[-2]: last_received = lines[-2]
buffer = lines[-1]
class SerialData(object):
def __init__(self, init=50):
try:
self.ser = serial.Serial(
port='/dev/ttyACM1',
baudrate=9600,
bytesize=serial.EIGHTBITS,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
timeout=0.1,
xonxoff=0,
rtscts=0,
interCharTimeout=None
)
except serial.serialutil.SerialException as e:
print(e)
#no serial connection
self.ser = None
else:
Thread(target=receiving, args=(self.ser,)).start()
def next(self):
# if not self.ser:
# return '81.3,78.1,10.0,60.0,0' #100 #return anything so we can test when Arduino isn't connected
#return a float value or try a few times until we get one
for i in range(40):
raw_line = last_received
try:
# return float(raw_line.strip())
return str(raw_line.strip())
except ValueError:
print('bogus data',raw_line)
time.sleep(.005)
return 0.
def __del__(self):
if self.ser:
self.ser.close()
def write(self,val):
self.ser.write(val)
if __name__=='__main__':
s = SerialData()
for i in range(500):
time.sleep(.015)
print (s.next())
I would like to write an application that could stop the server based on client's input. The server is multi-threaded and I do not understand how can I do this.
Basically, I described my problem here: Modify server's variable from client's thread (threading, python).
However, this is the Python solution, not the general solution I could implement in Java, C, C++, etc.
I need to close other clients, when one of them guesses the number, but the server should be still alive, ready for the new game.
Can I ask for some advices, explanations?
I tried this (still do not know how to port it to C or Java), but it lets the clients send the numbers even if one of them just guesses it. It seems to me that kill_em_all does not do it's job, it does not close all the connections and does not disconnect the other clients as it should. How to improve this?
#!/usr/bin/env python
from random import randint
import socket, select
from time import gmtime, strftime
import threading
import sys
class Handler(threading.Thread):
def __init__(self, connection, randomnumber, server):
threading.Thread.__init__(self)
self.connection = connection
self.randomnumber = randomnumber
self.server = server
def run(self):
while True:
try:
data = self.connection.recv(1024)
if data:
print(data)
try:
num = int(data)
if self.server.guess(num) :
print 'someone guessed!'
self.server.kill_em_all()
break
else :
msg = "Try again!"
self.connection.sendall(msg.encode())
except ValueError as e:
msg = "%s" % e
self.connection.sendall(msg.encode())
else:
msg = "error"
self.connection.send(msg.encode())
except socket.error:
break
self.connection.close()
def send(self, msg):
self.connection.sendall(msg)
def close(self):
self.connection.close()
class Server:
randnum = randint(1,100)
def __init__(self, ip, port):
self.ip = ip
self.port = port
self.address = (self.ip, self.port)
self.server_socket = None
def guess(self, no):
if self.randnum == no:
self.randnum = randint(1, 100)
print("New number is ", self.randnum )
result = True
else:
result = False
return result
def kill_em_all(self):
for c in self.clients:
c.send("BYE!")
c.close()
def run(self):
try:
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.ip, self.port))
self.server_socket.listen(10)
self.clients = []
print('Num is %s' % self.randnum)
while True:
connection, (ip, port) = self.server_socket.accept()
c = Handler(connection, self.randnum, self)
c.start()
self.clients.append(c)
except socket.error as e:
if self.server_socket:
self.server_socket.close()
sys.exit(1)
if __name__ == '__main__':
s = Server('127.0.0.1', 7777)
s.run()
Client code:
import socket
import sys
port = 7777
s = None
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = socket.gethostname()
s.connect(('127.0.0.1', port))
except socket.error, (value, message):
if s:
s.close()
print "Could not open socket: " + message
sys.exit(1)
while True:
data = raw_input('> ')
s.sendall(data)
data = s.recv(1024)
if data:
if data == "BYE!":
break
else:
print "Server sent: %s " % data
s.close()
Log in. Using whatever protocol you have, send the server a message telliing it to shut down. In the server, terminate your app when you get the shutdown message. That's it. It's not a problem with any OS I have used - any thread of a process can terminate that process.