Python Selector with FIFO running to infinite loop - python-3.x

I am trying to write some non-blocking FIFO code with kqueue on my BSD machine. Here's the small server code: server.py
import os
import selectors
sel = selectors.KqueueSelector()
TMP_PATH="/tmp/myfifo"
def fifo_read(fd, mask):
data = os.read(fd, 8)
print("fd:{} gives:{} \n", fd, data)
sel.unregister(fd)
print("unregistered")
def fifo_accept(listen_fd, mask):
print("accepted {}".format(listen_fd))
fd = os.dup(listen_fd)
print("duped to {}".format(fd))
sel.register(fd, selectors.EVENT_READ, fifo_read)
if __name__ == "__main__":
try:
os.unlink(TMP_PATH)
except:
pass
os.mkfifo(TMP_PATH)
listen_fd = os.open(TMP_PATH, os.O_RDONLY, mode=0o600)
sel.register(listen_fd, selectors.EVENT_READ, fifo_accept)
while True:
events = sel.select()
for key, mask in events:
cb = key.data
cb(key.fileobj, mask)
sel.close()
Now, when I run a client.py as:
import os
TMP_PATH="/tmp/myfifo"
fd = os.open(TMP_PATH, os.O_WRONLY, mode=0o600)
res = os.write(fd, b"1234567")
print("sent {}".format(res))
When I run the client, I get:
sent 7
But on server, it runs to inifinite loop. Now I understand why the infinite loop is happening. I actually tried mimicking the socket way of using selectors in this Python Docs example.
Here's what I have tried:
I did try the code without duplicating the fd, but it's still in infinite loop.
I tried calling sel.unregister on the original listen_fd, but in this case, running the client the second time doesn't work (which is expected).
Can anyone please let me know if I'm missing something?

So I found one solution to this problem. With sockets, we get a new socket object on accept. So we need to emulate that behaviour by calling unregister on the original fileobj, open again and call register on that.
Fixed code:
import os
import selectors
sel = selectors.KqueueSelector()
try:
os.unlink("./myfifo")
except:
pass
os.mkfifo("./myfifo", 0o600)
def cb(fp):
sel.unregister(fp)
print(f"got {fp.read()}")
fp.close()
fp2 = open("./myfifo", "rb")
sel.register(fp2, selectors.EVENT_READ, cb)
if __name__ == "__main__":
orig_fp = open("./myfifo", "rb")
print("open done")
ev = sel.register(orig_fp, selectors.EVENT_READ, cb)
print(f"registration done for {ev}")
while True:
events = sel.select()
print(events)
for key, mask in events:
key.data(key.fileobj)

Related

Running web.run_app() along with another async program : Python

Currently I am working on a project which involves usage of Asynchronous functions, due to the usage of certain set of libraries. My code runs fine as long as I don't integrate a web-socket server implementing functionality in my code.
But, I wish to stream the output 'Result' continuously in a websocket stream. So, I tried integrating websocket from socketio library as an AsyncServer.
Firstly, in my code, I want to gather all my inputs, and keep displaying the possible Result in a terminal. Once my inputs are finalized, I wish my result to be streamed over Websocket.
Initially, I just tried using web.run_app() in an asynchronous task in the main thread. Refer code below with #Type-1 comments. (Make sure that the lines with comment #Type-2 should be commented out). But I get the following exception "This event loop is already running".
I thought maybe if I run web.run_app() in a separate thread, then this issue might not come up. So, I changed my implementation slightly. Refer code below with #Type-2 comments. (Make sure that the lines with comment #Type-1 should be commented out). Now, I get another issue "set_wakeup_fd only works in main thread of the main interpreter".
Can someone please help me solve this issue, and let me know how must I use web.run_app()?
Here is the code:
import os, sys
import asyncio
import platform
import threading
import socketio
import json
from aioconsole import ainput
from aiohttp import web
from array import *
Result = -1
Inputs_Required = True
Input_arr = array('i')
sio = socketio.AsyncServer()
app = web.Application()
sio.attach(app)
Host = "192.168.0.7"
Port = 8050
async def IOBlock():
global Input_arr
global Inputs_Required
while(True):
response = input("Enter new input? (y/n): ")
if('y' == response or 'Y' == response):
Input = input("Enter number to be computed: ")
Input_arr.append(int(Input))
break
elif('n' == response or 'N' == response):
Inputs_Required = False
break
else:
print("Invalid response.")
async def main():
global Results
global Inputs_Required
global Input_arr
WebSocketStarted = False
#WebSocketThread = threading.Thread(target = WebStreaming, daemon = True) #Type-2
try:
while True:
if(Inputs_Required == True):
Task_AddInput = asyncio.create_task(IOBlock())
await Task_AddInput
elif (WebSocketStarted == False):
WebSocketStarted = True
#WebSocketThread.start() #Type-2
WebTask = asyncio.create_task(WebStreaming()) #Type-1
await WebTask #Type-1
if(len(Input_arr) > 0):
Task_PrintResult = asyncio.create_task(EvaluateResult())
await Task_PrintResult
except Exception as x:
print(x)
finally:
await Cleanup()
async def WebStreaming(): #Type-1
#def WebStreaming(): #Type-2
print("Starting web-socket streaming of sensor data..")
Web_loop = asyncio.new_event_loop #Type-1 or 2
asyncio.set_event_loop(Web_loop) #Type-1 or 2
web.run_app(app, host=Host, port=Port)
async def EvaluateResult():
global Input_arr
global Result
Result = 0
for i in range (0, len(Input_arr)):
Result += Input_arr[i]
print(f"The sum of inputs fed so far = {Result}.")
await asyncio.sleep(5)
async def Cleanup():
global Input_arr
global Inputs_Required
global Result
print("Terminating program....")
Result = -1
Inputs_Required = True
for i in reversed(range(len(Input_arr))):
del Input_arr[i]
#sio.event
async def connect(sid, environ):
print("connect ", sid)
#sio.event
async def OnClientMessageReceive(sid, data):
global Result
print("Client_message : ", data)
while True:
msg = json.dumps(Result)
print(msg)
await sio.send('OnServerMessageReceive', msg)
#sio.event
def disconnect(sid):
print('disconnect ', sid)
if __name__ == "__main__":
asyncio.run(main())

How to execute a ZeroMQ PUSH/PULL archetype in an asynchronous way?

I want to initiate a PULL in a port and want to receive from other ports to my PULL port. In the case of a PULL port, it listens asynchronously and when it receives a message, it just prints the message in the console. So for that I have written a method inside a Push-class, which will send the message to the PULL port.
My code is as follows :
import random
import zmq
import time
import sys
import string
import asyncio
import zmq.asyncio
class Push():
def __init__(self, port, addr='localhost'):
self.port = port
self.addr = addr
self.ctx = zmq.Context()
self.scoket = self.ctx.socket(zmq.PUSH)
self.scoket.connect(f'tcp://{self.addr}:{selfa.port}')
def send(self):
chars = string.ascii_uppercase + string.ascii_lowercase
message = ''.join(random.choice(chars) for _ in range(4))
self.scoket.send(bytes(message, 'utf-8'))
print(f'sending: {message}')
class Pull():
def __init__(self, port, addr='*'):
self.port = port
self.addr = addr
self.ctx = zmq.Context()
self.socket = self.ctx.socket(zmq.PULL)
self.socket.bind(f'tcp://{self.addr}:{self.port}')
async def listen(self, listener):
while True:
string = await self.socket.recv()
listener(string)
if __name__ == '__main__':
push = Push('55501')
async def send():
while True:
await asyncio.sleep(5)
print('Sending...')
push.send()
pull = Pull('55501')
try:
asyncio.run(
pull.listen(print),
send(),
)
except KeyboardInterrupt:
print('exiting...')
exit()
The above code is not running. The code stops at the listen method.
#ADAPTED FROM PYMATA EXPRESS EXAMPLE CONCURRENTTAKS
#https://github.com/MrYsLab/pymata-express/
import asyncio
import zmq
import json
import zmq.asyncio as zmq_asyncio
from pymata_express.pymata_express import PymataExpress
class ConcurrentTasks:
def __init__(self, board):
self.loop = board.get_event_loop()
self.board = board
self.ctxsync = zmq.Context()
self.context = zmq.asyncio.Context()
self.rep = self.context.socket(zmq.REP)
self.rep.bind("tcp://*:5558")
self.trigger_pin = 53
self.echo_pin = 51
loop.run_until_complete(self.async_init_and_run())
### START: NEW CODE THAT RESOLVED THE ISSUE
async def pingsonar(self):
value = await self.board.sonar_read(self.trigger_pin)
return value
async def readsonar(self):
while True:
rep_recv = await self.rep.recv()
value = await asyncio.wait([self.pingsonar()])
valuesonar = list(value[0])[0].result()
json_data = json.dumps(valuesonar)
await self.rep.send(json_data.encode())
await asyncio.sleep(1 / 1000) #maybe this line isn't necessary
### END : NEW CODE THAT RESOLVED THE ISSUE
async def async_init_and_run(self):
await self.board.set_pin_mode_sonar(self.trigger_pin, self.echo_pin)
readsonar = asyncio.create_task(self.readsonar())
await readsonar
# OTHER CREATED_TASK GO HERE, (removed them in the MVE, but they work fine)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
my_board = PymataExpress()
try:
ConcurrentTasks(my_board)
except (KeyboardInterrupt, RuntimeError):
loop.run_until_complete(my_board.shutdown())
print('goodbye')
finally:
loop.close()
The above code is not running.
The code is running,yet there are mistakes ( in concept ) and typos in the source code.
The code as-is inhibits the Push-class from ever become .connect()-ed, thus the Pull-counterparty .bind()-s, yet has nobody to talk to.
SOLUTION
1 )Repair this typo ( + better, explicitly detect and handle all potential error-states )
self.scoket.connect(f'tcp://{self.addr}:{selfa.port}') # this will NEVER FLY
#--------------------------------------------^
self.scoket.connect(f'tcp://{self.addr}:{self.port}') # this will ... ( + detect Error-state(s)
2 )Correct the concept - mixing low-hanging fruits ( several asynchronous frameworks used at once, one working against all the others ) is a sign of shallow understanding of the responsibility of professional engineering in reliable and robust-enough distributed-computing ( flaws in the concept of control, like during landing Apollo-11 on Moon or, on the contrary, the Chernobyl-kind of mis-management system + mentality, are clean, inspiring and warning enough examples how bad practices ( if not prohibited + detected + corrected + penalised ) can & will harm again and again and again ).
The Best Next Step
If you are keen to get to the professional-level, start with Pieter Hintjens' book "Code Connected, Volume 1" - worth time, worth efforts, worth understanding their discussed concepts.

Sharing asyncio.Queue with another thread or process

I've recently converted my old template matching program to asyncio and I have a situation where one of my coroutines relies on a blocking method (processing_frame).
I want to run that method in a seperate thread or process whenever the coroutine that calls that method (analyze_frame) gets an item from the shared asyncio.Queue()
I'm not sure if that's possible or worth it performance wise since I have very little experience with threading and multiprocessing
import cv2
import datetime
import argparse
import os
import asyncio
# Making CLI
if not os.path.exists("frames"):
os.makedirs("frames")
t0 = datetime.datetime.now()
ap = argparse.ArgumentParser()
ap.add_argument("-v", "--video", required=True,
help="path to our file")
args = vars(ap.parse_args())
threshold = .2
death_count = 0
was_found = False
template = cv2.imread('youdied.png')
vidcap = cv2.VideoCapture(args["video"])
loop = asyncio.get_event_loop()
frames_to_analyze = asyncio.Queue()
def main():
length = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
tasks = []
for _ in range(int(length / 50)):
tasks.append(loop.create_task(read_frame(50, frames_to_analyze)))
tasks.append(loop.create_task(analyze_frame(threshold, template, frames_to_analyze)))
final_task = asyncio.gather(*tasks)
loop.run_until_complete(final_task)
dt = datetime.datetime.now() - t0
print("App exiting, total time: {:,.2f} sec.".format(dt.total_seconds()))
print(f"Deaths registered: {death_count}")
async def read_frame(frames, frames_to_analyze):
global vidcap
for _ in range(frames-1):
vidcap.grab()
else:
current_frame = vidcap.read()[1]
print("Read 50 frames")
await frames_to_analyze.put(current_frame)
async def analyze_frame(threshold, template, frames_to_analyze):
global vidcap
global was_found
global death_count
frame = await frames_to_analyze.get()
is_found = processing_frame(frame)
if was_found and not is_found:
death_count += 1
await writing_to_file(death_count, frame)
was_found = is_found
def processing_frame(frame):
res = cv2.matchTemplate(frame, template, cv2.TM_CCOEFF_NORMED)
max_val = cv2.minMaxLoc(res)[1]
is_found = max_val >= threshold
print(is_found)
return is_found
async def writing_to_file(death_count, frame):
cv2.imwrite(f"frames/frame{death_count}.jpg", frame)
if __name__ == '__main__':
main()
I've tried using unsync but without much success
I would get something along the lines of
with self._rlock:
PermissionError: [WinError 5] Access is denied
If processing_frame is a blocking function, you should call it with await loop.run_in_executor(None, processing_frame, frame). That will submit the function to a thread pool and allow the event loop to proceed with doing other things until the call function completes.
The same goes for calls such as cv2.imwrite. As written, writing_to_file is not truly asynchronous, despite being defined with async def. This is because it doesn't await anything, so once its execution starts, it will proceed to the end without ever suspending. In that case one could as well make it a normal function in the first place, to make it obvious what's going on.

Python CMD2 Interrupt printing promp

I am using CMD2 to create a server. I have broken my code down to the smallest bit of code that still produces the problem.
import socket
import _thread
from cmd2 import *
def grab_data(conn):
while True:
try:
data = conn.recv(1024)
print(data)
except:
print("disconnected.")
break
def grab_client(sock):
while True:
conn, addr = sock.accept()
print("New connection")
_thread.start_new_thread(grab_data, (conn,))
def start_conn(ip, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((ip, int(port)))
sock.listen(10)
print("Socket listening")
_thread.start_new_thread(grab_client, (sock,))
class CLI(Cmd):
def __init__(self):
Cmd.__init__(self)
self.prompt = "Prompt> "
def do_listen(self, arg):
start_conn('0.0.0.0', '4446')
def emptyline(self):
pass
def do_quit(self, arg):
return True
cli = CLI()
cli.cmdloop("Welcome to the server.")
The issue I run into is when a client connects. It does not reprint the prompt. It hangs at a empty line with just the cursor. I am not sure how to get the prompt to print back.
You're blocking trying to read 1024 bytes, so it's got to wait until that entire buffer is filled. To do proper line-based buffering is a bit tricky, but a simple (albeit non-performant) implementation is to read a character at a time and check if it's a newline:
line = ""
while True:
data = conn.recv(1)
line += data
if data in "\n\r":
break
(This is not great code, but let's see if that solves the problem and we can improve it.)

Non-blocking input in python 3 [duplicate]

I'm working on a bot for a competition that receives its input through sys.stdin and uses Python's print() for output. I have the following:
import sys
def main():
while True:
line = sys.stdin.readline()
parts = line.split()
if len(parts) > 0:
# do stuff
The problem is that the input comes in through a stream and using the above, blocks me from printing anything back until the stream is closed. What can I do to make this work?
By turning blocking off you can only read a character at a time. So, there is no way to get readline() to work in a non-blocking context. I assume you just want to read key presses to control the robot.
I have had no luck using select.select() on Linux and created a way with tweaking termios settings. So, this is Linux specific but works for me:
import atexit, termios
import sys, os
import time
old_settings=None
def init_any_key():
global old_settings
old_settings = termios.tcgetattr(sys.stdin)
new_settings = termios.tcgetattr(sys.stdin)
new_settings[3] = new_settings[3] & ~(termios.ECHO | termios.ICANON) # lflags
new_settings[6][termios.VMIN] = 0 # cc
new_settings[6][termios.VTIME] = 0 # cc
termios.tcsetattr(sys.stdin, termios.TCSADRAIN, new_settings)
#atexit.register
def term_any_key():
global old_settings
if old_settings:
termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
def any_key():
ch_set = []
ch = os.read(sys.stdin.fileno(), 1)
while ch is not None and len(ch) > 0:
ch_set.append( ord(ch[0]) )
ch = os.read(sys.stdin.fileno(), 1)
return ch_set
init_any_key()
while True:
key = any_key()
if key is not None:
print(key)
else:
time.sleep(0.1)
A better Windows or cross-platform answer is here: Non-blocking console input?
You can use selectors for handle I/O multiplexing:
https://docs.python.org/3/library/selectors.html
Try this out:
#! /usr/bin/python3
import sys
import fcntl
import os
import selectors
# set sys.stdin non-blocking
orig_fl = fcntl.fcntl(sys.stdin, fcntl.F_GETFL)
fcntl.fcntl(sys.stdin, fcntl.F_SETFL, orig_fl | os.O_NONBLOCK)
# function to be called when enter is pressed
def got_keyboard_data(stdin):
print('Keyboard input: {}'.format(stdin.read()))
# register event
m_selector = selectors.DefaultSelector()
m_selector.register(sys.stdin, selectors.EVENT_READ, got_keyboard_data)
while True:
sys.stdout.write('Type something and hit enter: ')
sys.stdout.flush()
for k, mask in m_selector.select():
callback = k.data
callback(k.fileobj)
The above code will hold on the line
for k, mask in m_selector.select():
until a registered event occurs, returning a selector_key instance (k) and a mask of monitored events.
In the above example we registered only one event (Enter key press):
m_selector.register(sys.stdin, selectors.EVENT_READ, got_keyboard_data)
The selector key instance is defined as follows:
abstractmethod register(fileobj, events, data=None)
Therefore, the register method sets k.data as our callback function got_keyboard_data, and calls it when the Enter key is pressed:
callback = k.data
callback(k.fileobj)
A more complete example (and hopefully more useful) would be to multiplex stdin data from user with incoming connections from network:
import selectors
import socket
import sys
import os
import fcntl
m_selector = selectors.DefaultSelector()
# set sys.stdin non-blocking
def set_input_nonblocking():
orig_fl = fcntl.fcntl(sys.stdin, fcntl.F_GETFL)
fcntl.fcntl(sys.stdin, fcntl.F_SETFL, orig_fl | os.O_NONBLOCK)
def create_socket(port, max_conn):
server_addr = ('localhost', port)
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.setblocking(False)
server.bind(server_addr)
server.listen(max_conn)
return server
def read(conn, mask):
global GO_ON
client_address = conn.getpeername()
data = conn.recv(1024)
print('Got {} from {}'.format(data, client_address))
if not data:
GO_ON = False
def accept(sock, mask):
new_conn, addr = sock.accept()
new_conn.setblocking(False)
print('Accepting connection from {}'.format(addr))
m_selector.register(new_conn, selectors.EVENT_READ, read)
def quit():
global GO_ON
print('Exiting...')
GO_ON = False
def from_keyboard(arg1, arg2):
line = arg1.read()
if line == 'quit\n':
quit()
else:
print('User input: {}'.format(line))
GO_ON = True
set_input_nonblocking()
# listen to port 10000, at most 10 connections
server = create_socket(10000, 10)
m_selector.register(server, selectors.EVENT_READ, accept)
m_selector.register(sys.stdin, selectors.EVENT_READ, from_keyboard)
while GO_ON:
sys.stdout.write('>>> ')
sys.stdout.flush()
for k, mask in m_selector.select():
callback = k.data
callback(k.fileobj, mask)
# unregister events
m_selector.unregister(sys.stdin)
# close connection
server.shutdown()
server.close()
# close select
m_selector.close()
You can test using two terminals.
first terminal:
$ python3 test.py
>>> bla
open another terminal and run:
$ nc localhost 10000
hey!
back to the first
>>> qwerqwer
Result (seen on the main terminal):
$ python3 test.py
>>> bla
User input: bla
>>> Accepting connection from ('127.0.0.1', 39598)
>>> Got b'hey!\n' from ('127.0.0.1', 39598)
>>> qwerqwer
User input: qwerqwer
>>>
#-----------------------------------------------------------------------
# Get a character from the keyboard. If Block is True wait for input,
# else return any available character or throw an exception if none is
# available. Ctrl+C isn't handled and continues to generate the usual
# SIGINT signal, but special keys like the arrows return the expected
# escape sequences.
#
# This requires:
#
# import sys, select
#
# This was tested using python 2.7 on Mac OS X. It will work on any
# Linux system, but will likely fail on Windows due to select/stdin
# limitations.
#-----------------------------------------------------------------------
def get_char(block = True):
if block or select.select([sys.stdin], [], [], 0) == ([sys.stdin], [], []):
return sys.stdin.read(1)
raise error('NoChar')
This is a posix solution, similar to the answer by swdev.
As they stated, you have to play with termios.VMIN and termios.VTIME to catch more than one char without requiring user to press Enter. Trying to only use raw mode will be a problem as special keys like arrows can mess next keypress.
Here we use tty.setcbreak() or tty.setraw() as a shortcut, but they have short internals.
import termios
import tty
import sys
import select
def get_enter_key():
fd = sys.stdin.fileno()
orig_fl = termios.tcgetattr(fd)
try:
tty.setcbreak(fd) # use tty.setraw() instead to catch ^C also
mode = termios.tcgetattr(fd)
CC = 6
mode[CC][termios.VMIN] = 0
mode[CC][termios.VTIME] = 0
termios.tcsetattr(fd, termios.TCSAFLUSH, mode)
keypress, _, _ = select.select([fd], [], [])
if keypress:
return sys.stdin.read(4095)
finally:
termios.tcsetattr(fd, termios.TCSANOW, orig_fl)
try:
while True:
print(get_enter_key())
except KeyboardInterrupt:
print('exiting')
sys.exit()
note that there are two potential timeouts you could add here:
one is adding last parameter to select.select()
another is playing with VMIN and VTIME
Might I suggest nobreak? If'n you are willing to use curses.
https://docs.python.org/3/library/curses.html#curses.window.nodelay
You should be able to get read of a stream with either
sys.stdin.read(1)
to read utf-8 decoded chars or:
sys.stdin.buffer.read(1)
to read raw chars.
I would do this if I wanted to get raw data from the stdin and do something with it in a timely manner, without reading a newline or filling up the internal buffer first. This is suitable for running programs remotely via ssh where tty is not available, see:
ssh me#host '/usr/bin/python -c "import sys; print(sys.stdin.isatty())"'
There are some other things to think about to make programs work as expected in this scenario. You need to flush the output when you're done to avoid buffering delays, and it could be easy to assume a program hasn't read the input, when you've simply not flushed the output.
stdout.write("my data")
stdout.flush()
But usually it's not the input reading that's the problem but that the terminal (or program) supplying the input stream is not handing it over when you expect, or perhaps it's not reading your output when you expect. If you have a tty to start with (see ssh check above) you can put it into raw mode with the tty module.
import sys
import termios
import tty
old = termios.tcgetattr(sys.stdin)
tty.setraw(sys.stdin)
c = None
try:
c = sys.stdin.read(1)[0]
finally:
termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old)
print(c)
... if using Mac/Linux. If using Windows you could use msvcrt.getch().
Use a generator - thankfully sys.stdin is already a generator!
A generator enables you to work on an infinite stream. Always when you call it, it returns the next element. In order to build a generator you need the yield keyword.
for line in sys.stdin:
print line
if a_certain_situation_happens:
break
Do not forget to place a break statement into the loop if a certain, wished situation happens.
You can find more information about generators on:
http://www.dabeaz.com/generators/index.html
http://linuxgazette.net/100/pramode.html

Resources