I am following the David Beazley Python Concurrency From the Ground Up: LIVE! - PyCon 2015 talk where I aim to understand basics of Python concurrency.
I would like to know why Python select is failing which is used for polling the operating system (OS) and check if there is some task that needs to be done.
from socket import *
from select import select
from collections import deque
tasks = deque()
recv_wait = {}
send_wait = {}
def fib(n):
if n <= 2:
return 1
else:
return fib(n-1)+fib(n-2)
def run():
while any([tasks, recv_wait, send_wait]):
while not tasks:
can_recv, can_send, _ = select(recv_wait, send_wait, [])
for s in can_recv:
tasks.append(recv_wait.pop(s))
for s in can_send:
tasks.append(send_wait.pop(s))
task = tasks.popleft()
try:
why, what = next(task)
if why == 'recv':
recv_wait[what] = task
elif why == 'send':
send_wait[what] = task
else:
raise RuntimeError("Arg!!")
except StopIteration:
print("task done")
def fib_server(address):
sock = socket(AF_INET, SOCK_STREAM)
sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
sock.bind(address)
sock.listen(5)
while True:
yield 'recv', sock
client, addr = sock.accept()
print("Connection", addr)
tasks.append(fib_handler(client))
def fib_handler(client):
while True:
yield 'recv', client
req = client.recv(100)
if not req:
break
n = int(req)
result = fib(n)
result = fib(n)
resp = str(result).encode('ascii') + b'\n'
yield 'send',resp
client.send(resp)
print("Closed")
tasks.append(fib_server(('', 25000)))
run()
# Separate terminal window
nc localhost 25000
12
# Running Python server
➜ python3 -i aserver.py
Connection ('127.0.0.1', 61098)
Traceback (most recent call last):
File "aserver.py", line 63, in <module>
run()
File "aserver.py", line 20, in run
can_recv, can_send, _ = select(recv_wait, send_wait,[])
TypeError: argument must be an int, or have a fileno() method.
def fib_handler(client):
while True:
yield 'recv', client
req = client.recv(100)
if not req:
break
n = int(req)
result = fib(n)
result = fib(n) # duplicate
resp = str(result).encode('ascii')+ b'\n'
yield 'send',resp # should be yielding fd (client)
client.send(resp)
print("Closed")
Related
I am developing a python program for reading ports. My script has a print for every open port checked. But I would like to remove this print and put it inside a class. For when the programmer wants to see print he calls the class.
I can create common classes to get user input from a main file (main.py) and run inside the script, but I can't control print from the main file
def ping(target,scale):
def portscan(port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
con = s.connect((target,port))
time.sleep(5)
port_print = 'Port :',port,"Is Open!."
time.sleep(5)
#python = sys.executable
#os.execl(python, python, * sys.argv)
print('Terminated')
con.close()
except:
#result = None
#return result
pass
r = 1
scal = int(scale)
for x in range(1,scal):
t = threading.Thread(target=portscan,kwargs={'port':r})
r += 1
t.start()
As you can see I created the variable port_print, and I would like that when the user called in the main file, there would be the print.
Use a Queueto get around return limitations in threads:
from queue import Queue
def ping(target,scale, queue):
def portscan(port, queue):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
con = s.connect((target,port))
time.sleep(5)
port_print = 'Port :',port,"Is Open!."
queue.put(port_print)
time.sleep(5)
#python = sys.executable
#os.execl(python, python, * sys.argv)
print('Terminated')
con.close()
except:
#result = None
#return result
pass
r = 1
scal = int(scale)
for x in range(1,scal):
t = threading.Thread(target=portscan,kwargs={'port':r, queue=queue})
r += 1
t.start()
def main():
my_queue = Queue()
target = 'some target'
scale = 10
ping(target, scale, my_queue)
random_port_print = my_queue.get()
print(random_port_print)
Not tested but prly pretty close to correct.
I am using pool.imap_unordered to apply a function over different txt files saved locally.
Is it possible to capture the exception and pass?
If my code runs into an exception, it blocks the entire loop.
pool = Pool(processes=15)
results = {}
files = glob.glob('{}/10K_files/*.txt'.format(path_input))
for key, output in tqdm(pool.imap_unordered(process_file, files),total=len(files)):
results[key] = output
I've tried something like this:
pool = Pool(processes=15)
results = {}
files = glob.glob('{}/10K_files/*.txt'.format(path_input))
try:
for key, output in tqdm(pool.imap_unordered(process_file, files), total=len(files)):
results[key] = output
except:
print("error")
but then I want to resume the loop from where I started.
Thanks!
You could catch the exception in process_file and return it. Then test for whether the return value is an exception. Here is an example:
import os
import traceback
import multiprocessing as mp
def main():
work_items = [i for i in range(20)]
pool = mp.Pool()
for result in pool.imap_unordered(process_file_exc, work_items):
if isinstance(result, Exception):
print("Got exception: {}".format(result))
else:
print("Got OK result: {}".format(result))
def process_file_exc(work_item):
try:
return process_file(work_item)
except Exception as ex:
return Exception("Err on item {}".format(work_item)
+ os.linesep + traceback.format_exc())
def process_file(work_item):
if work_item == 9:
# this will raise ZeroDivisionError exception
return work_item / 0
return "{} * 2 == {}".format(work_item, work_item * 2)
if __name__ == '__main__':
main()
I have a class function which unbuffers stdout and stderr, like so:
class Unbuffered:
def __init__(self, stream):
self.stream = stream
def write(self, data):
data = data.strip()
if data.startswith("INFO: "):
data = data[6:]
if '[' in data:
progress = re.compile(r"\[(\d+)/(\d+)\]")
data = progress.match(data)
total = data.group(2)
current = data.group(1)
data = '{0}/{1}'.format(current, total)
if data.startswith("ERROR: "):
data = data[7:]
self.stream.write(data + '\n')
self.stream.flush()
def __getattr__(self, attr):
return getattr(self.stream, attr)
The output is from a function run in ProcessPoolExecutor when inbound from websocket arrives.
I want the output printed in console as well as sent to my websocket client. I tried asyncing Unbuffered and passing websocket instance to it but no luck.
UPDATE: The essentials of run() and my websocket handler() look something like this:
def run(url, path):
logging.addLevelName(25, "INFO")
fmt = logging.Formatter('%(levelname)s: %(message)s')
#----
output.progress_stream = Unbuffered(sys.stderr)
stream = Unbuffered(sys.stdout)
#----
level = logging.INFO
hdlr = logging.StreamHandler(stream)
hdlr.setFormatter(fmt)
log.addHandler(hdlr)
log.setLevel(level)
get_media(url, opt)
async def handler(websocket, path):
while True:
inbound = json.loads(await websocket.recv())
if inbound is None:
break
url = inbound['url']
if 'path' in inbound:
path = inbound['path'].rstrip(os.path.sep) + os.path.sep
else:
path = os.path.expanduser("~") + os.path.sep
# blah more code
while inbound != None:
await asyncio.sleep(.001)
await loop.run_in_executor(None, run, url, path)
run(), handler() and Unbuffered are separate from each other.
Rewriting get_media() to use asyncio instead of running it in a different thread would be the best. Otherwise, there are some options to communicate between a regular thread and coroutines, for example, using a socketpair:
import asyncio
import socket
import threading
import time
import random
# threads stuff
def producer(n, writer):
for i in range(10):
# print("sending", i)
writer.send("message #{}.{}\n".format(n, i).encode())
time.sleep(random.uniform(0.1, 1))
def go(writer):
threads = [threading.Thread(target=producer, args=(i + 1, writer,))
for i in range(5)]
for t in threads:
t.start()
for t in threads:
t.join()
writer.send("bye\n".encode())
# asyncio coroutines
async def clock():
for i in range(11):
print("The time is", i)
await asyncio.sleep(1)
async def main(reader):
buffer = ""
while True:
buffer += (await loop.sock_recv(reader, 10000)).decode()
# print(len(buffer))
while "\n" in buffer:
msg, _nl, buffer = buffer.partition("\n")
print("Got", msg)
if msg == "bye":
return
reader, writer = socket.socketpair()
reader.setblocking(False)
threading.Thread(target=go, args=(writer,)).start()
# time.sleep(1.5) # socket is buffering
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait([clock(), main(reader)]))
loop.close()
You can also try this 3rd-party thread+asyncio compatible queue: janus
I created server + client codes on python.
The goal is as itemized below:
Client sends a command to the server.
Server receives a command, and runs a function to return some
information depending on what command was received.
The server also adds a string containing the ip-addresses of
connected clients to the information to be returned.
Server sends it back to client
Client receives the data and uses it however it chooses
The server is started as python server.py. But for the client, you need to first import it, after importing the client, you call it like so
nw = Client()
data = ''
try:
nw.connect()
nw.send_msg('Hello')
data = nw.recv_msg()
nw.close()
except:
data = "An error Occurred"
The above client code works well with the server; however, when I run the code like so
item = "somethingthereandhere"
nw = Client()
data = "CMDGENERATE" + item
try:
nw.connect()
nw.send_msg(data)
data = nw.recv_msg()
except:
data = "An Error Occured"
I get an error on the server side. The traceback looks like so
Accepted connection from ('127.0.0.1', 5381)
in recv_msg
in recvall
in recvall
Data received
in doing something
in lorem_gen
in send_msg
reply sent
in recv_msg
in recvall
Traceback (most recent call last):
File "Slocket.py", line 92, in <module>
new_client.connect()
File "Slocket.py", line 25, in connect
data = self.recv_msg(client)
File "Slocket.py", line 43, in recv_msg
raw_msglen = self.recvall(client, 4)
File "Slocket.py", line 53, in recvall
packet = client.recv(n - len(data))
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
All the information printed on the server console are tracker-statements to know where the interpreter is at any point.
Below is the server code
#server.py
import socket
import struct
import time
class Server:
def __init__(self, sock = socket.socket(
socket.AF_INET, socket.SOCK_STREAM)):
self.clients = ''
if sock is None:
self.sock = socket.socket(
socket.AF_INET, socket.SOCK_STREAM)
else:
self.sock = sock
def connect(self, host="127.0.0.1", port=5007):
self.sock.bind((host, port))
self.sock.listen(20)
print ("Listening on port: "+str(port))
while True:
client, sock_addr = self.sock.accept()
print ("Accepted connection from", sock_addr)
self.clients += "=========" + str(sock_addr)
while True:
data = self.recv_msg(client)
if data is None: break
print("Data received")
reply = self.do_something(data)
self.send_msg(client, reply)
print('reply sent')
client.close()
print("Connection Closed")
def send_msg(self, client, msg):
print("in send_msg")
msg = struct.pack('>I', len(msg)) + msg
client.sendall(msg)
def recv_msg(self,client):
print("in recv_msg")
raw_msglen = self.recvall(client, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
return self.recvall(client, msglen)
def recvall(self, client, n):
print("in recvall")
data = ''
while len(data) < n:
packet = client.recv(n - len(data))
if not packet:
return None
data += packet.decode()
return data.encode()
def do_something(self, data=""):
print("in doing something")
#change string to byte
b = self.clients.encode()
if 'CMDGENERATE'.encode() in data:
data = lorem_gen(50)
if 'CMDUPPERCASE'.encode() in data:
data = data.upper()
# add separator
b += "====DD===".encode() + data
if type(b) is str:
b = b.encode()
return b
def lorem_gen(n = 20):
print("in lorem_gen")
n = int(n)
rd = "No file found or an error occured"
a = " "
try:
srcfile = open('data/lorem.txt', 'r')
rd = srcfile.read()
rd = rd.split(' ')
rd = rd[0:n]
a = ' '.join(rd)
rd = a
except:
pass
return rd.encode()
if __name__ == "__main__":
new_client = Server()
new_client.connect()
Below is the client code:
#client.py
import socket
import struct
import time
class Client:
def __init__(self, sock=None):
if sock is None:
self.sock = socket.socket(
socket.AF_INET, socket.SOCK_STREAM)
else:
self.sock = sock
def connect(self, host="127.0.0.1", port=5007):
self.sock.connect((host, port))
def send_msg(self, msg):
msg = struct.pack('>I', len(msg)) + msg.encode()
self.sock.sendall(msg)
print("send_time", time.time())
def recv_msg(self):
raw_msglen = self.recvall(4)
if not raw_msglen:
return None
print("raw_msglen is %s " % raw_msglen)
msglen = struct.unpack('>I', raw_msglen)[0]
print("msglen is %s " % msglen)
return self.recvall(msglen)
def recvall(self, n):
data = ''
while len(data) < n:
packet = self.sock.recv(n - len(data))
if not packet:
return None
data += packet.decode()
print("receive_time", time.time())
return data.encode()
def close(self):
print("close_time", time.time())
return self.sock.close()
Thanks in advance for your help.
I try to make Async ping process using subprocess.Popen , I try to understand how i implement it in this case
aList = []
async def sn(frm, to):
i = 0
for i in list(range(frm, to)):
aList.append(i)
cmd = "ping -n 1 " + '10.0.0.'
coroutines = [subprocess.Popen(cmd + str(i), stdout=subprocess.PIPE) for i in aList]
results = await asyncio.gather(*coroutines)
print(results)
loop = asyncio.get_event_loop()
loop.run_until_complete(sn(frm, to))
loop.close()
You can find simpler code for pinging host without async-await. But if necessary you can try the following working example to ping with async-await
import platform
import subprocess
import aiohttp
import asyncio
async def getPingedHost(host, netTimeout=3):
""" Description: Function to ping a host and get string of outcome or False
Import: from shared.getPingedHost import getPingedHost
Testing: python -m shared.getPingedHost
"""
args = ['ping']
platformOs = platform.system().lower()
if platformOs == 'windows':
args.extend(['-n', '1'])
args.extend(['-w', str(netTimeout * 1000)])
elif platformOs in ('linux', 'darwin'):
args.extend(['-c', '1'])
args.extend(['-W', str(netTimeout)])
else:
raise NotImplemented('Unsupported OS: {}'.format(platformOs))
args.append(host)
output = ''
try:
outputList = []
if platformOs == 'windows':
output = subprocess.run(args, check=True, universal_newlines=True,
stdout=subprocess.PIPE, # Capture standard out
stderr=subprocess.STDOUT, # Capture standard error
).stdout
outputList = str(output).split('\n')
if output and 'TTL' not in output:
output = False
else:
subprocess.run(args, check=True)
output = outputList[2]
except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
output = False
return output
async def main():
async with aiohttp.ClientSession() as client:
output = await getPingedHost('google.com')
print(output)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
class rscan(object):
state = {'online': [], 'offline': []} # Dictionary with list
ips = [] # Should be filled by function after taking range
# Amount of pings at the time
thread_count = 8
# Lock object to prevent race conditions
lock = threading.Lock()
# Using Windows ping command
def ping(self, ip):
answer = subprocess.call(['ping','-n','1',ip],stdout = open('1.txt','w'))
return answer == 0 and ip
def pop_queue(self):
ip = None
self.lock.acquire() # lock !!!
if self.ips:
ip = self.ips.pop()
self.lock.release()
return ip
def noqueue(self):
while True:
ip = self.pop_queue()
if not ip:
return None
result = 'online' if self.ping(ip) else 'offline'
self.state[result].append(ip) ### check again
def start(self):
threads = []
for i in range(self.thread_count):
t = threading.Thread(target=self.noqueue)
t.start()
threads.append(t)
# Wait for all threads
[ t.join() for t in threads ]
return self.state
def rng(self, frm, to, ip3):
self.frm = frm
self.to = to
self.ip3 = ip3
for i in range(frm, to):
ip = ip3 + str(i)
self.ips.append(ip)
if __name__== '__main__':
scant = rscan()
scant.thread_count = 8
edited a bit class i have found also used threads instead of Async & await
Credit: http://blog.boa.nu/2012/10/python-threading-example-creating-pingerpy.html