I am trying to run two threads concurrently in order to: collect data and save this data periodically. I am using socket to collect the data and module asyncio in order to run in parallel this two functions.
This is the code:
# IMPORT LIBRARIES:
import socket
import asyncio
# DEFINITION FUNCTION:
async def restart_variable():
print("executes restart_variable")
while True:
await asyncio.sleep(20)
print("Should clean and save data")
print("For the moment only clean")
totaldata = ""
async def collect_data():
print("executes collect data")
totaldata = ""
while True:
data = sock.recv(1024)
data = str(data, "utf-8")
totaldata += data
print(1, len(totaldata))
print(2, data)
async def main():
try:
sock.connect((HOST, PORT))
print("Connected")
except Exception as e:
print("Cannot connect to the server:", e)
try:
asyncio.gather(restart_variable(), collect_data())
except KeyboardInterrupt:
print("Manually interrupted")
sock.close()
except Exception as e:
print(e)
sock.close()
HOST = "HOST"
PORT = 2304
# MAIN EXECUTION:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
asyncio.run(main())
However, only collect data is working. restart_variable function is only executed one time. I dont understand why while True has no effect.
Related
Currently I am working on a project which involves usage of Asynchronous functions, due to the usage of certain set of libraries. My code runs fine as long as I don't integrate a web-socket server implementing functionality in my code.
But, I wish to stream the output 'Result' continuously in a websocket stream. So, I tried integrating websocket from socketio library as an AsyncServer.
Firstly, in my code, I want to gather all my inputs, and keep displaying the possible Result in a terminal. Once my inputs are finalized, I wish my result to be streamed over Websocket.
Initially, I just tried using web.run_app() in an asynchronous task in the main thread. Refer code below with #Type-1 comments. (Make sure that the lines with comment #Type-2 should be commented out). But I get the following exception "This event loop is already running".
I thought maybe if I run web.run_app() in a separate thread, then this issue might not come up. So, I changed my implementation slightly. Refer code below with #Type-2 comments. (Make sure that the lines with comment #Type-1 should be commented out). Now, I get another issue "set_wakeup_fd only works in main thread of the main interpreter".
Can someone please help me solve this issue, and let me know how must I use web.run_app()?
Here is the code:
import os, sys
import asyncio
import platform
import threading
import socketio
import json
from aioconsole import ainput
from aiohttp import web
from array import *
Result = -1
Inputs_Required = True
Input_arr = array('i')
sio = socketio.AsyncServer()
app = web.Application()
sio.attach(app)
Host = "192.168.0.7"
Port = 8050
async def IOBlock():
global Input_arr
global Inputs_Required
while(True):
response = input("Enter new input? (y/n): ")
if('y' == response or 'Y' == response):
Input = input("Enter number to be computed: ")
Input_arr.append(int(Input))
break
elif('n' == response or 'N' == response):
Inputs_Required = False
break
else:
print("Invalid response.")
async def main():
global Results
global Inputs_Required
global Input_arr
WebSocketStarted = False
#WebSocketThread = threading.Thread(target = WebStreaming, daemon = True) #Type-2
try:
while True:
if(Inputs_Required == True):
Task_AddInput = asyncio.create_task(IOBlock())
await Task_AddInput
elif (WebSocketStarted == False):
WebSocketStarted = True
#WebSocketThread.start() #Type-2
WebTask = asyncio.create_task(WebStreaming()) #Type-1
await WebTask #Type-1
if(len(Input_arr) > 0):
Task_PrintResult = asyncio.create_task(EvaluateResult())
await Task_PrintResult
except Exception as x:
print(x)
finally:
await Cleanup()
async def WebStreaming(): #Type-1
#def WebStreaming(): #Type-2
print("Starting web-socket streaming of sensor data..")
Web_loop = asyncio.new_event_loop #Type-1 or 2
asyncio.set_event_loop(Web_loop) #Type-1 or 2
web.run_app(app, host=Host, port=Port)
async def EvaluateResult():
global Input_arr
global Result
Result = 0
for i in range (0, len(Input_arr)):
Result += Input_arr[i]
print(f"The sum of inputs fed so far = {Result}.")
await asyncio.sleep(5)
async def Cleanup():
global Input_arr
global Inputs_Required
global Result
print("Terminating program....")
Result = -1
Inputs_Required = True
for i in reversed(range(len(Input_arr))):
del Input_arr[i]
#sio.event
async def connect(sid, environ):
print("connect ", sid)
#sio.event
async def OnClientMessageReceive(sid, data):
global Result
print("Client_message : ", data)
while True:
msg = json.dumps(Result)
print(msg)
await sio.send('OnServerMessageReceive', msg)
#sio.event
def disconnect(sid):
print('disconnect ', sid)
if __name__ == "__main__":
asyncio.run(main())
I've developed a pattern to use for a commanding a python daemon through cmd module shell using an eventloop. However, its not ready yet because I can't figure out how to gracefully exit the two applications (I'm still learning asyncio and can't figure out the following problem). When the cmd module is commanded to exit I get:
(Cmd) exit
Shutting down client...
Traceback (most recent call last):
File "client_loop.py", line 10, in <module>
loop.run_until_complete(test.cmdloop())
File "...\asyncio\base_events.py", line 467, in run_until_complete
future = tasks.ensure_future(future, loop=self)
File "...\lib\asyncio\tasks.py", line 526, in ensure_future
raise TypeError('An asyncio.Future, a coroutine or an awaitable is '
TypeError: An asyncio.Future, a coroutine or an awaitable is required
I'm not good with asyncio yet, what am I doing wrong? Sorry for the long question but the files/errors make it long and hopefully makes it easier to debug.
Here are the supporting files:
shell.py
# implementation of the (Cmd) prompt with history functionality
# standard imports
import cmd as cmd
class Shell(cmd.Cmd):
def __init__(self, **kwargs):
cmd.Cmd.__init__(self, **kwargs)
self.eventloop = None
self.shutdown_client = None
self.tcp_echo_client = None
def set_eventloop(self, loop):
self.eventloop = loop
def set_funcs(self, tcp_echo_client, shutdown_client):
self.tcp_echo_client = tcp_echo_client
self.shutdown_client = shutdown_client
def do_exit(self,*args):
"""
Exits the shell gracefully
:param args:
:return:
"""
print('Shutting down client...')
self.shutdown_client(self.eventloop)
return True
def default(self, line):
try:
self.eventloop.run_until_complete(self.tcp_echo_client(line, self.eventloop))
except SystemExit:
pass
server.py
# server logic to parse arguments coming over the TCP socket and echo it back
# standard imports
import asyncio
async def handle_echo(reader, writer):
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
print("Send: %r" % message)
writer.write(data)
await writer.drain()
print("Close the client socket")
writer.close()
loop = asyncio.get_event_loop()
coro = asyncio.start_server(handle_echo, '127.0.0.1', 8888, loop=loop)
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
# Close the server
for task in asyncio.Task.all_tasks():
loop.run_until_complete(task)
server.close()
loop.run_until_complete(server.wait_closed())
loop.stop()
loop.close()
exit(0)
client.py
# client functions to send message over TCP and process response
# standard imports
import asyncio
# user imports
import shell
async def tcp_echo_client(message, loop):
reader, writer = await asyncio.open_connection('127.0.0.1', 8888,
loop=loop)
print('Send: %r' % message)
writer.write(message.encode())
data = await reader.read(100)
print('Received: %r' % data.decode())
print('Close the socket')
writer.close()
def shutdown_client(loop):
loop.stop()
# Find all running tasks:
pending = asyncio.Task.all_tasks()
# Run loop until tasks done:
loop.run_until_complete(asyncio.gather(*pending))
loop = asyncio.get_event_loop()
test = shell.Shell()
test.set_eventloop(loop)
test.set_funcs(tcp_echo_client, shutdown_client)
loop.run_until_complete(test.cmdloop())
loop.close()
The problem lies in the fact that cmd doesn't require asyncio to process input from the user and send a message to through the TCP/IP socket. Simply removing asyncio from the client side solved the problem. It still offers the shell implementation and the client-server pattern. Here is the new code:
server.py
# server logic to parse arguments coming over the TCP socket and echo it back
# standard imports
import asyncio
async def handle_echo(reader, writer):
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
print("Send: %r" % message)
writer.write(data)
await writer.drain()
print("Close the client socket")
writer.close()
loop = asyncio.get_event_loop()
coro = asyncio.start_server(handle_echo, '127.0.0.1', 8888, loop=loop)
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
# Close the server
for task in asyncio.Task.all_tasks():
loop.run_until_complete(task)
server.close()
loop.run_until_complete(server.wait_closed())
loop.stop()
loop.close()
exit(0)
client_shell.py
# implementation of a shell prompt (using Cmd module) to send message over TCP and process response
# standard imports
import socket
import cmd as cmd
class Shell(cmd.Cmd):
def __init__(self, **kwargs):
cmd.Cmd.__init__(self, **kwargs)
def do_exit(self,*args):
"""
Exits the shell gracefully
:param args:
:return:
"""
print('Shutting down client...')
return True
def default(self, line):
try:
self._tcp_echo_client(line.encode())
except SystemExit:
pass
def _tcp_echo_client(self, message):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
print('Send: %r' % message)
s.connect(('127.0.0.1', 8888))
s.sendall(message)
data = s.recv(1000)
print('Received: %r' % data.decode())
print('Close the socket')
if __name__ == '__main__':
Shell().cmdloop()
I'm playing around with socket programming. In the following code snippet I'm trying to connect to client and if his input contains "hack" it will remove it and run shell command and sends back the output.
server side:
import socket
class SP:
def server(self):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('127.0.0.1', 9999))
s.listen(1)
while True:
try:
c, addr = s.accept()
print('Got connection from ', addr)
while True:
data = c.recv(1024)
if data:
if 'hack' in data.decode('utf-8'):
import subprocess
data = data.decode('utf-8')
data = data.strip('hack').lstrip().rstrip()
output = subprocess.call(data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
s.send(str(output).encode('utf-8'))
else:
d = data.decode('utf-8')
print('Got data: '+str(d))
c.send(str('ACK: '+str(d)+' ...').encode('utf-8'))
else:
print('No more data from client: '+str(addr))
break
finally:
s.close()
except Exception as e:
print('Caught Exception: '+str(e))
s.close()
obj = SP()
obj.server()
client-side:
import socket
class CS:
def client(self):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('127.0.0.1', 9999))
while True:
data = input('Enter data to be sent to server: \n')
if not data:
break
else:
s.send(data.encode('utf-8'))
reply = s.recv(1024).decode('utf-8')
print(str(reply))
else:
s.close()
except Exception as e:
print('Caught Exception: '+ str(e))
s.close()
obj = CS()
obj.client()
How can I resolve this the error ? Caught Exception: [Errno 32] Broken pipe doesn't tell me much.
update:
import socket
class SP:
def server(self):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('127.0.0.1', 9999))
s.listen(1)
while True:
try:
c, addr = s.accept()
print('Got connection from ', addr)
while True:
data = c.recv(1024)
if data:
if 'hack' in data.decode('utf-8'):
import subprocess
data = data.decode('utf-8')
data = data.strip('hack').lstrip().rstrip()
print(data)
#output = subprocess.call(data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
s.send(data.encode('utf-8'))
else:
d = data.decode('utf-8')
print('Got data: '+str(d))
c.send(str('ACK: '+str(d)+' ...').encode('utf-8'))
else:
print('No more data from client: '+str(addr))
break
finally:
s.close()
except Exception as e:
print('Caught Exception: '+str(e))
s.close()
obj = SP()
obj.server()
Even when I comment out the line where I call subprocess.call I still get "Broken Pipe" so the error isn't originating from the subprocess call.
You're using server's socket s instead of client's socket c to send the data to the client:
s.send(data.encode('utf-8'))
how about changing it to:
c.send(data.encode('utf-8'))
I'm trying a example to create a simple socket server, I need to receive multiline data from clients so this is my code for the socket server:
import socket
import sys
host = 'localhost'
port = 5006
c = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
c.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
c.bind((host, port))
c.listen(1)
while True:
try:
print("1")
csock, caddr = c.accept()
print("2")
with csock.makefile('rw', 2**16) as cfile:
print("3")
print(cfile.readline()) // When I use readline, it goes ok, but only get the first line, I need every line from data sent by client
print("4")
cfile.close()
print("5")
print("5.5")
csock.sendall("OK".encode('UTF-8'))
print("6")
csock.close()
print("7")
except KeyboardInterrupt:
print("Keyboard exit")
if 'csock' in locals():
csock.close()
sys.exit()
except Exception as e:
print(e)
Now when I use readlines(), my program just stuck and "does nothing" after print("3")
I tried with read() too, but still stuck, keep waiting both client and server
This is the code I'm using for client:
import socket
def query(host, port):
msg = \
'MSH|^~\&|REC APP|REC FAC|SEND APP|SEND FAC|20110708163513||QBP^Q22^QBP_Q21|111069|D|2.5|||||ITA||EN\r' \
'QPD|IHE PDQ Query|111069|#PID.5.2^SMITH||||\r' \
'RCP|I|'
# establish the connection
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((host, port))
# send the message
sock.sendall(msg.encode('UTF-8'))
# receive the answer
received = sock.recv(1024*1024)
return received
finally:
sock.close()
if __name__ == '__main__':
res = query('localhost', 5006)
print("Received response: ")
print(repr(res))
When I use readlines() and stop client after executing, server prints full message sent, I'm confused
I would like to write an application that could stop the server based on client's input. The server is multi-threaded and I do not understand how can I do this.
Basically, I described my problem here: Modify server's variable from client's thread (threading, python).
However, this is the Python solution, not the general solution I could implement in Java, C, C++, etc.
I need to close other clients, when one of them guesses the number, but the server should be still alive, ready for the new game.
Can I ask for some advices, explanations?
I tried this (still do not know how to port it to C or Java), but it lets the clients send the numbers even if one of them just guesses it. It seems to me that kill_em_all does not do it's job, it does not close all the connections and does not disconnect the other clients as it should. How to improve this?
#!/usr/bin/env python
from random import randint
import socket, select
from time import gmtime, strftime
import threading
import sys
class Handler(threading.Thread):
def __init__(self, connection, randomnumber, server):
threading.Thread.__init__(self)
self.connection = connection
self.randomnumber = randomnumber
self.server = server
def run(self):
while True:
try:
data = self.connection.recv(1024)
if data:
print(data)
try:
num = int(data)
if self.server.guess(num) :
print 'someone guessed!'
self.server.kill_em_all()
break
else :
msg = "Try again!"
self.connection.sendall(msg.encode())
except ValueError as e:
msg = "%s" % e
self.connection.sendall(msg.encode())
else:
msg = "error"
self.connection.send(msg.encode())
except socket.error:
break
self.connection.close()
def send(self, msg):
self.connection.sendall(msg)
def close(self):
self.connection.close()
class Server:
randnum = randint(1,100)
def __init__(self, ip, port):
self.ip = ip
self.port = port
self.address = (self.ip, self.port)
self.server_socket = None
def guess(self, no):
if self.randnum == no:
self.randnum = randint(1, 100)
print("New number is ", self.randnum )
result = True
else:
result = False
return result
def kill_em_all(self):
for c in self.clients:
c.send("BYE!")
c.close()
def run(self):
try:
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.ip, self.port))
self.server_socket.listen(10)
self.clients = []
print('Num is %s' % self.randnum)
while True:
connection, (ip, port) = self.server_socket.accept()
c = Handler(connection, self.randnum, self)
c.start()
self.clients.append(c)
except socket.error as e:
if self.server_socket:
self.server_socket.close()
sys.exit(1)
if __name__ == '__main__':
s = Server('127.0.0.1', 7777)
s.run()
Client code:
import socket
import sys
port = 7777
s = None
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = socket.gethostname()
s.connect(('127.0.0.1', port))
except socket.error, (value, message):
if s:
s.close()
print "Could not open socket: " + message
sys.exit(1)
while True:
data = raw_input('> ')
s.sendall(data)
data = s.recv(1024)
if data:
if data == "BYE!":
break
else:
print "Server sent: %s " % data
s.close()
Log in. Using whatever protocol you have, send the server a message telliing it to shut down. In the server, terminate your app when you get the shutdown message. That's it. It's not a problem with any OS I have used - any thread of a process can terminate that process.