I need some help with multiprocessing module of python. I am using Python3.6.6. My code structure is somewhat like this:
class ABC():
def __init__(self):
self.HOST = 'hostserver.com'
self.TCP_PORT = 0123
self.BUFFER_SIZE = 1024
self.SERVER_INFO = ""
self.SOCK = None
def connect_socket(self):
self.SOCK = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.SOCK.settimeout(1)
self.SOCK.connect((self.HOST, self.TCP_PORT))
self.SOCK.setblocking(True)
def recTask(self):
while True:
self.receive_data()
time.sleep(0.01)
def sendTask(self):
while True:
self.SOCK.sendall(bytes)
print("\n*Message sent*\n")
time.sleep(0.01)
if __name__ == '__main__':
Class_obj = ABC()
id = Class_obj.connect_socket()
ts = mp.Process(name='send_Process', target=Class_obj.sendTask())
ts.daemon = True
tr = mp.Process(name='rec_Process', target=Class_obj.recTask())
tr.daemon = True
tr.start()
ts.start()
ts.join()
tr.join()
Can I call the methods of the same class using objects within the process? I want the 2 functions to run independently of each other. Also, when I run this I just see the "Message sent" being printed. In receive_data() function I have print("Message received") but it never prints. When I comment the code
ts = mp.Process(name='send_Process', target=Class_obj.sendTask())
ts.daemon = True
ts.start()
I see the "Message Received" being printed. Is there something which I am missing?
Related
Currently I am working on a project which involves usage of Asynchronous functions, due to the usage of certain set of libraries. My code runs fine as long as I don't integrate a web-socket server implementing functionality in my code.
But, I wish to stream the output 'Result' continuously in a websocket stream. So, I tried integrating websocket from socketio library as an AsyncServer.
Firstly, in my code, I want to gather all my inputs, and keep displaying the possible Result in a terminal. Once my inputs are finalized, I wish my result to be streamed over Websocket.
Initially, I just tried using web.run_app() in an asynchronous task in the main thread. Refer code below with #Type-1 comments. (Make sure that the lines with comment #Type-2 should be commented out). But I get the following exception "This event loop is already running".
I thought maybe if I run web.run_app() in a separate thread, then this issue might not come up. So, I changed my implementation slightly. Refer code below with #Type-2 comments. (Make sure that the lines with comment #Type-1 should be commented out). Now, I get another issue "set_wakeup_fd only works in main thread of the main interpreter".
Can someone please help me solve this issue, and let me know how must I use web.run_app()?
Here is the code:
import os, sys
import asyncio
import platform
import threading
import socketio
import json
from aioconsole import ainput
from aiohttp import web
from array import *
Result = -1
Inputs_Required = True
Input_arr = array('i')
sio = socketio.AsyncServer()
app = web.Application()
sio.attach(app)
Host = "192.168.0.7"
Port = 8050
async def IOBlock():
global Input_arr
global Inputs_Required
while(True):
response = input("Enter new input? (y/n): ")
if('y' == response or 'Y' == response):
Input = input("Enter number to be computed: ")
Input_arr.append(int(Input))
break
elif('n' == response or 'N' == response):
Inputs_Required = False
break
else:
print("Invalid response.")
async def main():
global Results
global Inputs_Required
global Input_arr
WebSocketStarted = False
#WebSocketThread = threading.Thread(target = WebStreaming, daemon = True) #Type-2
try:
while True:
if(Inputs_Required == True):
Task_AddInput = asyncio.create_task(IOBlock())
await Task_AddInput
elif (WebSocketStarted == False):
WebSocketStarted = True
#WebSocketThread.start() #Type-2
WebTask = asyncio.create_task(WebStreaming()) #Type-1
await WebTask #Type-1
if(len(Input_arr) > 0):
Task_PrintResult = asyncio.create_task(EvaluateResult())
await Task_PrintResult
except Exception as x:
print(x)
finally:
await Cleanup()
async def WebStreaming(): #Type-1
#def WebStreaming(): #Type-2
print("Starting web-socket streaming of sensor data..")
Web_loop = asyncio.new_event_loop #Type-1 or 2
asyncio.set_event_loop(Web_loop) #Type-1 or 2
web.run_app(app, host=Host, port=Port)
async def EvaluateResult():
global Input_arr
global Result
Result = 0
for i in range (0, len(Input_arr)):
Result += Input_arr[i]
print(f"The sum of inputs fed so far = {Result}.")
await asyncio.sleep(5)
async def Cleanup():
global Input_arr
global Inputs_Required
global Result
print("Terminating program....")
Result = -1
Inputs_Required = True
for i in reversed(range(len(Input_arr))):
del Input_arr[i]
#sio.event
async def connect(sid, environ):
print("connect ", sid)
#sio.event
async def OnClientMessageReceive(sid, data):
global Result
print("Client_message : ", data)
while True:
msg = json.dumps(Result)
print(msg)
await sio.send('OnServerMessageReceive', msg)
#sio.event
def disconnect(sid):
print('disconnect ', sid)
if __name__ == "__main__":
asyncio.run(main())
I am developing a python program for reading ports. My script has a print for every open port checked. But I would like to remove this print and put it inside a class. For when the programmer wants to see print he calls the class.
I can create common classes to get user input from a main file (main.py) and run inside the script, but I can't control print from the main file
def ping(target,scale):
def portscan(port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
con = s.connect((target,port))
time.sleep(5)
port_print = 'Port :',port,"Is Open!."
time.sleep(5)
#python = sys.executable
#os.execl(python, python, * sys.argv)
print('Terminated')
con.close()
except:
#result = None
#return result
pass
r = 1
scal = int(scale)
for x in range(1,scal):
t = threading.Thread(target=portscan,kwargs={'port':r})
r += 1
t.start()
As you can see I created the variable port_print, and I would like that when the user called in the main file, there would be the print.
Use a Queueto get around return limitations in threads:
from queue import Queue
def ping(target,scale, queue):
def portscan(port, queue):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
con = s.connect((target,port))
time.sleep(5)
port_print = 'Port :',port,"Is Open!."
queue.put(port_print)
time.sleep(5)
#python = sys.executable
#os.execl(python, python, * sys.argv)
print('Terminated')
con.close()
except:
#result = None
#return result
pass
r = 1
scal = int(scale)
for x in range(1,scal):
t = threading.Thread(target=portscan,kwargs={'port':r, queue=queue})
r += 1
t.start()
def main():
my_queue = Queue()
target = 'some target'
scale = 10
ping(target, scale, my_queue)
random_port_print = my_queue.get()
print(random_port_print)
Not tested but prly pretty close to correct.
I'm trying to use ZeroMQ in Python (pyzmq) together with multiprocessing. As a minmal (not) working example I have a server- and a client-class which both inherit from multiprocessing.Process. The client as a child-process should send a message to the server-child-process which should print the message:
#mpzmq_class.py
from multiprocessing import Process
import zmq
class Server(Process):
def __init__(self):
super(Server, self).__init__()
self.ctx = zmq.Context()
self.socket = self.ctx.socket(zmq.PULL)
self.socket.connect("tcp://localhost:6068")
def run(self):
msg = self.socket.recv_string()
print(msg)
class Client(Process):
def __init__(self):
super(Client, self).__init__()
self.ctx = zmq.Context()
self.socket = self.ctx.socket(zmq.PUSH)
self.socket.bind("tcp://*:6068")
def run(self):
msg = "Hello World!"
self.socket.send_string(msg)
if __name__ == "__main__":
s = Server()
c = Client()
s.start()
c.start()
s.join()
c.join()
Now if I run this the server-process seems to hang at the receive-call msg = socket.receive_string(). In another (more complicated) case, it even hung at the socket.connect("...")-statement.
If I rewrite the script to use functions instead of classes/objects, it runs just fine:
# mpzmq_function.py
from multiprocessing import Process
import zmq
def server():
ctx = zmq.Context()
socket = ctx.socket(zmq.PULL)
socket.connect("tcp://localhost:6068")
msg = socket.recv_string()
print(msg)
def client():
ctx = zmq.Context()
socket = ctx.socket(zmq.PUSH)
socket.bind("tcp://*:6068")
msg = "Hello World!"
socket.send_string(msg)
if __name__ == "__main__":
s = Process(target=server)
c = Process(target=client)
s.start()
c.start()
s.join()
c.join()
Output:
paul#AP-X:~$ python3 mpzmq_function.py
Hello World!
Can anybody help me with this? I guess it's something I didn't understand concerning the usage of multiprocessing.
Thank you!
I run into the same issue.
I guess the problem is, that the run method has no access to the context object.
Maybe it has something to do with the C implementation and the fact, that processes do not have shared memory.
If instantiate the context in the run method, it works.
Here a working example:
#mpzmq_class.py
from multiprocessing import Process
import zmq
class Base(Process):
"""
Inherit from Process and
holds the zmq address.
"""
def __init__(self, address):
super().__init__()
self.address = address
class Server(Base):
def run(self):
ctx = zmq.Context()
socket = ctx.socket(zmq.PULL)
socket.connect(self.address)
msg = socket.recv_string()
print(msg)
class Client(Base):
def run(self):
ctx = zmq.Context()
socket = ctx.socket(zmq.PUSH)
socket.bind(self.address)
msg = "Hello World!"
socket.send_string(msg)
if __name__ == "__main__":
server_addr = "tcp://127.0.1:6068"
client_addr = "tcp://*:6068"
s = Server(server_addr)
c = Client(client_addr)
s.start()
c.start()
s.join()
c.join()
I added a base class to demonstrate that you can still access normal Python objects from the run method. If you put the context object into the init Method, it won't work.
I have a class function which unbuffers stdout and stderr, like so:
class Unbuffered:
def __init__(self, stream):
self.stream = stream
def write(self, data):
data = data.strip()
if data.startswith("INFO: "):
data = data[6:]
if '[' in data:
progress = re.compile(r"\[(\d+)/(\d+)\]")
data = progress.match(data)
total = data.group(2)
current = data.group(1)
data = '{0}/{1}'.format(current, total)
if data.startswith("ERROR: "):
data = data[7:]
self.stream.write(data + '\n')
self.stream.flush()
def __getattr__(self, attr):
return getattr(self.stream, attr)
The output is from a function run in ProcessPoolExecutor when inbound from websocket arrives.
I want the output printed in console as well as sent to my websocket client. I tried asyncing Unbuffered and passing websocket instance to it but no luck.
UPDATE: The essentials of run() and my websocket handler() look something like this:
def run(url, path):
logging.addLevelName(25, "INFO")
fmt = logging.Formatter('%(levelname)s: %(message)s')
#----
output.progress_stream = Unbuffered(sys.stderr)
stream = Unbuffered(sys.stdout)
#----
level = logging.INFO
hdlr = logging.StreamHandler(stream)
hdlr.setFormatter(fmt)
log.addHandler(hdlr)
log.setLevel(level)
get_media(url, opt)
async def handler(websocket, path):
while True:
inbound = json.loads(await websocket.recv())
if inbound is None:
break
url = inbound['url']
if 'path' in inbound:
path = inbound['path'].rstrip(os.path.sep) + os.path.sep
else:
path = os.path.expanduser("~") + os.path.sep
# blah more code
while inbound != None:
await asyncio.sleep(.001)
await loop.run_in_executor(None, run, url, path)
run(), handler() and Unbuffered are separate from each other.
Rewriting get_media() to use asyncio instead of running it in a different thread would be the best. Otherwise, there are some options to communicate between a regular thread and coroutines, for example, using a socketpair:
import asyncio
import socket
import threading
import time
import random
# threads stuff
def producer(n, writer):
for i in range(10):
# print("sending", i)
writer.send("message #{}.{}\n".format(n, i).encode())
time.sleep(random.uniform(0.1, 1))
def go(writer):
threads = [threading.Thread(target=producer, args=(i + 1, writer,))
for i in range(5)]
for t in threads:
t.start()
for t in threads:
t.join()
writer.send("bye\n".encode())
# asyncio coroutines
async def clock():
for i in range(11):
print("The time is", i)
await asyncio.sleep(1)
async def main(reader):
buffer = ""
while True:
buffer += (await loop.sock_recv(reader, 10000)).decode()
# print(len(buffer))
while "\n" in buffer:
msg, _nl, buffer = buffer.partition("\n")
print("Got", msg)
if msg == "bye":
return
reader, writer = socket.socketpair()
reader.setblocking(False)
threading.Thread(target=go, args=(writer,)).start()
# time.sleep(1.5) # socket is buffering
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait([clock(), main(reader)]))
loop.close()
You can also try this 3rd-party thread+asyncio compatible queue: janus
So I have spent at least a couple of days on this problem.
I would like to have 2 threads HTTP server each serving two different IP:ports.
I open a FF and navigate to either say: http://196.64.131.250:8001/ or http://196.64.131.250:8002 and it should do a GET.
Also I like my threads or program itself stop after a given duration from command line say 5 sec.
I have done everything I could, even tried SIGAlarm and variable "keepRunning" which gets changed by a third thread after duration , but my program does Not stop. What am I doing wrong please.
note that I have commented the daemon: i.e (ZhttpThread[1-2].daemon = True)
if I dont comment it my threads stop right away. I want my HTTP server threads continue servicing, and if the duration DUR expires , then the program stops or threads stop.
import SimpleHTTPServer, SocketServer, logging, cgi, sys, signal, threading, time
import subprocess as sub
keepRunning = True
origTime = int(time.time())
class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_GET(self):
logging.warning("======= GET STARTED =======")
getHdr = self.headers
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
getHdr = self.headers
print(', '.join((getHdr)))
#if ("accept-encoding" in getHdr):
if ("accept-encoding" in (', '.join((getHdr)))):
print ('Test Passed ---------------')
signal.alarm(1)
class threadWithTO(threading.Thread):
def __init__(self, thID, ip, port, timeout):
threading.Thread.__init__(self)
self.ip = ip
self.port = port
self.handler = ServerHandler
self.httpd = SocketServer.TCPServer((self.ip, self.port), self.handler)
def run(self):
print (self.httpd)
#self.httpd.serve_forever()
if (keepRunning == True):
self.httpd.handle_request()
else:
self._stop.set()
def Run(self):
self.start()
def timeHandler(signum, frame):
print('Times up', signum)
sys.exit()
def timeChkr( threadName, dur, t1, t2):
print (origTime)
print ('Begin Timer thread')
while True:
nowTime = int(time.time())
print (nowTime)
delta = nowTime - origTime
if (delta >= dur):
keepRunning = False
t1.stop()
t2.stop()
else:
time.sleep(1)
def main():
#signal.signal(signal.SIGALRM, timeHandler)
#signal.alarm(DUR)
origTime = int(time.time())
ZhttpThread1 = threadWithTO("thread1", I, PORT, DUR)
ZhttpThread2 = threadWithTO("thread2", I, (int(PORT)+1), DUR)
t = threading.Thread(target=timeChkr, args = ("thread3",DUR))
t.daemon = True
t.start()
#ZhttpThread1.daemon = True
#ZhttpThread2.daemon = True
ZhttpThread1.Run()
ZhttpThread2.Run()
ok i figured out the issue is with socket. My socket is open and even though I have tried socket.settimeout I still cant get the socket to close
Thanks to Andrew.. my son whom sparked the idea in my head... here is the solution..
class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_GET(self):
logging.warning("======= GET STARTED =======")
logging.warning(self.headers)
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
def do_POST(self):
logging.warning("======= POST STARTED =======")
logging.warning(self.headers)
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD':'POST',
'CONTENT_TYPE':self.headers['Content-Type'],
})
logging.warning("======= POST VALUES =======")
print form.list
'''
for item in form.list:
logging.warning(item) '''
logging.warning("\n")
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
class StoppableHTTPServer(BaseHTTPServer.HTTPServer):
def server_bind(self):
BaseHTTPServer.HTTPServer.server_bind(self)
self.socket.settimeout(1)
self.run = True
def get_request(self):
while self.run:
try:
sock, addr = self.socket.accept()
sock.settimeout(None)
return (sock, addr)
except socket.timeout:
pass
def stop(self):
self.run = False
def serve(self):
while self.run:
#print "1"
self.handle_request()
if __name__=="__main__":
if len(sys.argv) < 1:
I = ""
PORT = 8000
DUR = 10
else:
I = sys.argv[1]
PORT = int(sys.argv[2])
DUR = int(sys.argv[3])
#httpd1 = StoppableHTTPServer((I,PORT), SimpleHTTPServer.SimpleHTTPRequestHandler)
#httpd2 = StoppableHTTPServer((I,(int(PORT)+1)), SimpleHTTPServer.SimpleHTTPRequestHandler)
httpd1 = StoppableHTTPServer((I,PORT), ServerHandler)
httpd2 = StoppableHTTPServer((I,(int(PORT)+1)), ServerHandler)
thread.start_new_thread(httpd1.serve, ())
thread.start_new_thread(httpd2.serve, ())
#raw_input("Press <RETURN> to stop server\n")`enter code here`
print '0'
time.sleep(DUR)
print 'Times up Dude'
httpd1.stop()
httpd2.stop()