I'm looking to mock a set of REST APIs for some tests. The following main() function works fine (i.e. it returns {"some-data": 1234} as json to the browser when I GET localhost:8099). The issue is it blocks the main thread:
from gevent import monkey, sleep, pywsgi
monkey.patch_all()
import flask
from flask_restful import reqparse, abort, Api, Resource
import queue
import sys
import threading
STUFFS = {"some-data": 1234}
class Stuff(Resource):
def get(self):
return flask.jsonify(STUFFS)
class ControlThread(threading.Thread):
def __init__(self, http_server, stop_event):
threading.Thread.__init__(self)
self.stop_event = stop_event
self.http_server = http_server
self.running = False
def run(self):
try:
while not self.stop_event.is_set():
if not self.running:
self.http_server.start()
self.running = True
sleep(0.001)
except (KeyboardInterrupt, SystemExit):
pass
self.http_server.stop()
class StuffMock:
def __init__(self, port, name=None):
if name is None:
name = __name__
self.app = flask.Flask(name)
self.api = Api(self.app)
self.api.add_resource(Stuff, "/stuff/")
self.stop_event = threading.Event()
self.http_server = pywsgi.WSGIServer(('', port), self.app)
self.serving_thread = ControlThread(self.http_server,
self.stop_event)
self.serving_thread.daemon = True
def start(self):
self.serving_thread.start()
def stop(self):
self.stop_event.set()
self.serving_thread.join()
def main():
mocker = StuffMock(8099)
mocker.start()
try:
while True:
sleep(0.01)
except (KeyboardInterrupt, SystemExit):
mocker.stop()
sys.exit()
if __name__ == "__main__":
main()
Without the sleep() call in the while loop above, nothing resolves. Here is a more succinct usage to demonstrate:
import time
from stuff_mock import StuffMock
mocker = StuffMock(8099)
mocker.start()
while True:
user_text = input("let's do some work on the main thread: ")
# will only resolve the GET request after user input
# (i.e. when the main thread executes this sleep call)
time.sleep(0.1)
if user_text == "q":
break
mocker.stop()
The gevent threading module seems to work differently from the core one. Does anyone have any tips or ideas about what's going on under the hood?
Found that if I switch out threading for multiprocessing (and threading.Thread for multiprocessing.Process), everything works as expected, and I can spin up arbitrary numbers of mockers without blocking.
Related
I want to ask, right now I’m doing the python3 http web server. However it has the issue on “if-condition” when it stuck at “while-true”. When I want to use other “if condition”, the program stuck at “while-true” and cannot proceed other program.
from http.server import BaseHTTPRequestHandler, HTTPServer
import subprocess
Request = None
class RequestHandler_httpd(BaseHTTPRequestHandler):
def do_GET(self):
global Request
messagetosend = bytes('Hello Worldddd!',"utf")
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', len(messagetosend))
self.end_headers()
self.wfile.write(messagetosend)
Request = self.requestline
Request = Request[5 : int(len(Request)-9)]
print(Request)
if Request == 'onAuto':
def always_run():
subprocess.run("python3 satu.py ;", shell=True)
subprocess.run("python3 dua.py ;", shell=True)
while True:
always_run() #the program stuck here and other if cannot be used
if Request == 'onFM':
subprocess.run("python3 satu.py ;", shell=True)
if Request == 'onQR':
subprocess.run("python3 dua.py ;", shell=True)
if Request == 'offSYS':
subprocess.run("python3 OFF_SYSTEM.py ;", shell=True)
return
server_address_httpd = ('X.X.X.X',8080) #my private address
httpd = HTTPServer(server_address_httpd, RequestHandler_httpd)
print('Starting Server')
httpd.serve_forever()
As JonSG commented. Your
while True:
always_run()
is blocking the further execution of your code. So you have to run it in a separate thread:
import threading
class AlwaysThread(threading.Thread):
def __init__(self):
super(AlwaysThread, self).__init__()
self.stopThread = False
def run(self):
self.stopThread = False
while not self.stopThread:
always_run()
# where you previously have done the endless loop
t = AlwaysThread()
t.start()
# stop it with t.stopThread = True
I would also use a switch statement instead of the if cascade.
I was looking for a watchdog and I found this great library. I need to fit a DBSCAN model if a file is created in a folder. Joblib is used in the scikit-learn's DBSCAN implementation and joblib doesn't allow to use multiprocessing if the DBSCAN running code is not in the main thread. If I use watchdog, DBSCAN code can't run in the main thread. How can I solve this issue? Below you can find the watchdog script and a simple function to test it. When I run the main_watchdog.py and add a file in the folder where the watchdog is watching, it runs the simple_function.py in the Thread-1. In the mean time, main_watchdog.py runs in the MainThread.
PS: A solution could be starting a subprocess every time calling the simple_function.py but I am afraid that this may cause some issues if multiple files created in the watchdog folder. Imagine receiving 10 or 100 or 10000 files at once...
#main_watchdog.py
import time
import logging
import threading
from watchdog.observers import Observer
from watchdog.events import LoggingEventHandler
from a_function import simple_function
class Event(LoggingEventHandler):
def on_created(self, event):
simple_function(x)
def on_modified(self, event):
simple_function(x)
if __name__ == "__main__":
x = 1
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
# path = sys.argv[1] if len(sys.argv) > 1 else '.'
path = '/path/to/watch/the/folder'
event_handler = Event()
observer = Observer()
observer.schedule(event_handler, path, recursive=False)
observer.start()
try:
while True:
time.sleep(1)
print(threading.current_thread().name)
except KeyboardInterrupt:
observer.stop()
observer.join()
#a_function.py
import threading
def simple_function(x):
x += 1
print(threading.current_thread().name)
print(x)
If I am to understand this problem correctly, you need you business logic to run in the main thread and the observer to run in the background thread.
This can be easily solved by calling the observer thread in the backgound by using the threading library and then passing the value of those events to your function call by way of Queues which are a way of communication between threads.
#main_watchdog.py
import time
import logging
import threading
from watchdog.observers import Observer
from watchdog.events import LoggingEventHandler
from a_function import simple_function
import sys
from queue import Queue
q = Queue()
x = 1
class Event(LoggingEventHandler):
def on_created(self, event):
q.put(x)
def on_modified(self, event):
q.put(x)
def run_observer():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
path = sys.argv[1] if len(sys.argv) > 1 else '.'
# path = '/path/to/watch/the/folder'
event_handler = Event()
observer = Observer()
observer.schedule(event_handler, path, recursive=False)
observer.start()
while True:
time.sleep(1)
print(threading.currentThread().name)
if __name__ == "__main__":
background_thread = threading.Thread(target=run_observer, args=())
background_thread.daemon = True
background_thread.start()
print('Business logic')
while True:
val = q.get(True)
simple_function(val)
The other function can remain the same.
I have been trying to get my application to terminate gracefully for quite some time now, but so far none of the answers I have found worked.
The sample code below illustrates the structure of my application. It basically is a chain of threads that passes data to one another using Queues.
from abc import abstractmethod
from time import sleep
from threading import Thread, Event
from queue import Queue
import signal
import sys
class StoppableThread(Thread):
def __init__(self):
super().__init__()
self.stopper = Event()
self.queue = Queue()
#abstractmethod
def actual_job(self):
pass
def stop_running(self):
self.stopper.set()
def run(self):
while not self.stopper.is_set():
print(self.stopper.is_set())
self.actual_job()
self.queue.join()
class SomeObjectOne(StoppableThread):
def __init__(self, name, some_object_two):
super().__init__()
self.name = name
self.obj_two = some_object_two
def actual_job(self):
# print('{} is currently running'.format(self.name))
input_string = 'some string'
print('{} outputs {}'.format(self.name, input_string))
self.obj_two.queue.put(input_string)
sleep(2)
class SomeObjectTwo(StoppableThread):
def __init__(self, name, some_object_three):
super().__init__()
self.name = name
self.some_object_three = some_object_three
def actual_job(self):
# print('{} is currently running'.format(self.name))
some_string = self.queue.get()
inverted = some_string[::-1]
print('{} outputs {}'.format(self.name , inverted))
self.some_object_three.queue.put(inverted)
sleep(2)
class SomeObjectThree(StoppableThread):
def __init__(self, name):
super().__init__()
self.name = name
def actual_job(self):
print('{} is currently running'.format(self.name))
some_string = self.queue.get()
print('{} outputs {}'.format(self.name ,some_string[::-1]))
sleep(2)
class ServiceExit(Exception):
"""
Custom exception which is used to trigger the clean exit
of all running threads and the main program.
"""
pass
def service_shutdown(signum, frame):
print('Caught signal %d' % signum)
raise ServiceExit
signal.signal(signal.SIGTERM, service_shutdown)
signal.signal(signal.SIGINT, service_shutdown)
if __name__ == '__main__':
thread_three = SomeObjectThree('SomeObjectThree')
thread_two = SomeObjectTwo('SomeObjectTwo', thread_three)
thread_one = SomeObjectOne('SomeObjectOne', thread_two)
try:
thread_three.start()
thread_two.start()
thread_one.start()
# Keep the main thread running, otherwise signals are ignored.
while True:
sleep(0.5)
except ServiceExit:
print('Running service exit')
thread_three.stop_running()
thread_two.stop_running()
thread_one.stop_running()
thread_one.join()
thread_two.join()
thread_three.join()
sys.exit(0)
Now, if I run this code and ctrl-C to terminate, thread_one seems to join as expected, but the code gets stuck at thread_two.join().
Because thread_one is the only thread with a continuous empty queue, I expect it has something to do with the queue.
Any ideas?
In the run() method of StoppableThread you have this:
self.queue.join()
join() is a blocking method:
Blocks until all items in the queue have been gotten and processed.
The count of unfinished tasks goes up whenever an item is added to the
queue. The count goes down whenever a consumer thread calls
task_done() to indicate that the item was retrieved and all work on it
is complete. When the count of unfinished tasks drops to zero, join()
unblocks.
So in order for join() to return, it's not enough to get() an item in the other thread, you must also indicate that it's been processed with task_done():
from abc import abstractmethod
from time import sleep
from threading import Thread, Event
from queue import Queue
import signal
import sys
class StoppableThread(Thread):
def __init__(self):
super().__init__()
self.stopper = Event()
self.queue = Queue()
#abstractmethod
def actual_job(self):
pass
def stop_running(self):
self.stopper.set()
def run(self):
while not self.stopper.is_set():
print(self.stopper.is_set())
self.actual_job()
self.queue.join()
class SomeObjectOne(StoppableThread):
def __init__(self, name, some_object_two):
super().__init__()
self.name = name
self.obj_two = some_object_two
def actual_job(self):
# print('{} is currently running'.format(self.name))
input_string = 'some string'
print('{} outputs {}'.format(self.name, input_string))
self.obj_two.queue.put(input_string)
sleep(2)
class SomeObjectTwo(StoppableThread):
def __init__(self, name, some_object_three):
super().__init__()
self.name = name
self.some_object_three = some_object_three
def actual_job(self):
# print('{} is currently running'.format(self.name))
some_string = self.queue.get()
inverted = some_string[::-1]
print('{} outputs {}'.format(self.name , inverted))
self.queue.task_done()
self.some_object_three.queue.put(inverted)
sleep(2)
class SomeObjectThree(StoppableThread):
def __init__(self, name):
super().__init__()
self.name = name
def actual_job(self):
print('{} is currently running'.format(self.name))
some_string = self.queue.get()
print('{} outputs {}'.format(self.name ,some_string[::-1]))
self.queue.task_done()
sleep(2)
class ServiceExit(Exception):
"""
Custom exception which is used to trigger the clean exit
of all running threads and the main program.
"""
pass
def service_shutdown(signum, frame):
print('Caught signal %d' % signum)
raise ServiceExit
signal.signal(signal.SIGTERM, service_shutdown)
signal.signal(signal.SIGINT, service_shutdown)
if __name__ == '__main__':
thread_three = SomeObjectThree('SomeObjectThree')
thread_two = SomeObjectTwo('SomeObjectTwo', thread_three)
thread_one = SomeObjectOne('SomeObjectOne', thread_two)
try:
thread_three.start()
thread_two.start()
thread_one.start()
# Keep the main thread running, otherwise signals are ignored.
while True:
sleep(0.5)
except ServiceExit:
print('Running service exit')
thread_three.stop_running()
thread_two.stop_running()
thread_one.stop_running()
thread_one.join()
thread_two.join()
thread_three.join()
I'm using PyQt5 and Python3, I use 3 QThread classes to run something and after they are done I need to execute a 4th QThread class. But the execution of the 4th need to take place after all of the QThread classes finish work, or only 2 or only 1. It must not run while the first 3 are working.
I looked on the internet but I couldn't find a solution. My code looks like this:
class MyWindow(QtWidgets.QMainWindow):
def __init__(self):
QtWidgets.QMainWindow.__init__(self)
file_path = os.path.abspath('builder_gui.ui')
uic.loadUi(file_path, self)
self.obj1 = TasksThread1(self.comboBox.currentText(),self.comboBox_6.currentText())
self.obj2 = TasksThread2(self.comboBox_2.currentText(),self.comboBox_5.currentText())
self.obj3 = TasksThread3(self.comboBox_3.currentText(),self.comboBox_4.currentText())
self.obj4 = TasksThread4()
self.menubar.setNativeMenuBar(False)
self.progressVal = 1
self.cwd = os.getcwd()
self.obj1.newValueProgress.connect(self.increment_progress)
self.obj1.message.connect(self.status_bar)
self.obj2.newValueProgress.connect(self.increment_progress)
self.obj2.message.connect(self.status_bar)
self.obj3.newValueProgress.connect(self.increment_progress)
self.obj3.message.connect(self.status_bar)
self.obj4.newValueProgress.connect(self.increment_progress)
self.obj4.message.connect(self.status_bar)
self.obj4.doneSignal.connect(self.calculate_done_limit)
self.pushButton.pressed.connect(self.execute_build_script)
def calculate_done_limit(self):
limitCalc = 100 - int(self.progressBar.value())
self.increment_progress(limitCalc)
def run_gits_all(self):
if self.crowdTwistCheck.isChecked():
self.obj1.start()
else:
pass
if self.ThemeCheck.isChecked():
self.obj2.start()
else:
pass
if self.mainAwsCheck.isChecked():
self.obj3.start()
else:
pass
def execute_build_script(self):
self.progressBar.setValue(1)
self.progressVal = 1
self.run_gits_all()
def execute_last_part(self):
self.obj4.start()
def status_bar(self, value_in):
read1 = self.textBrowser.toPlainText()
self.textBrowser.setText(read1 + "\n" + value_in)
def increment_progress(self,valueIn):
self.progressVal += valueIn
self.progressBar.setValue(self.progressVal)
if __name__ == '__main__':
import sys
app = QtWidgets.QApplication(sys.argv)
window = MyWindow()
window.show()
sys.exit(app.exec_())
My first 3 QThreads are like this:
class TasksThread1(QThread):
newValueProgress = QtCore.pyqtSignal(int)
message = QtCore.pyqtSignal(str)
doneSignal = QtCore.pyqtSignal()
def __init__(self, branch, git):
QThread.__init__(self)
self.branch = branch
self.git = git
def remove_folder(self):
do_something_1
def CrowdTwistRepo(self):
do_something_2
def run(self):
self.remove_folder()
self.CrowdTwistRepo()
My last QThread looks like this:
class TasksThread4(QThread):
newValueProgress = QtCore.pyqtSignal(int)
message = QtCore.pyqtSignal(str)
doneSignal = QtCore.pyqtSignal()
def __init__(self):
QThread.__init__(self)
def gulp_sass_function(self):
do_something_1
def gulp_uglify_function(self):
do_something_2
def zipping_function(self):
do_something_3
def run(self):
self.gulp_sass_function()
self.gulp_uglify_function()
self.zipping_function()
If I run the code, all of the QThreads start and I want my 4th QThread to start only after the first 3 have done working. I used QThreads to improve the GUI experience, the GUI froze alot.
thanks,
When your first 3 threads are done, send a signal. Then connect this signal to a function that will start the last thread.
I'm using tornado with threads.
In short, each time the websocket handler receives a requests, it start to execute a task, which might take a few minutes.
However, once a client is connected, no other client can be connected, until the first one disconnects.
Any ideas?
I've attached a minimal example that uses time.sleep to simulate long running tasks.
import tornado.web
import tornado.websocket
import tornado.httpserver
import tornado.ioloop
import time
import json
import threading
class TaskHandler(tornado.websocket.WebSocketHandler):
def open(self):
pass
def check_origin(self, origin):
return True
def on_message(self, message):
try:
print 'received: ', message
self.write_message(json.dumps({'status': 'running'}))
def worker_A(kwargs):
time.sleep(100)
pass
def worker_B(kwargs):
time.sleep(100)
pass
threads = []
for target in [worker_A, worker_B]:
t = threading.Thread(target = target, args = ({'xxx': 'yyy'}, ))
t.daemon = True
t.start()
threads.append(t)
for t in threads:
t.join()
except Exception, e:
print 'TaskHandler: exception: ', e
pass
self.write_message(json.dumps({'status': 'done'}))
def on_close(self):
pass
class Server(tornado.web.Application):
def __init__(self):
handlers = [
('/task', TaskHandler),
]
tornado.web.Application.__init__(self, handlers)
if __name__ == '__main__':
server = tornado.httpserver.HTTPServer(Server())
server.listen(8765, address = '127.0.0.1')
tornado.ioloop.IOLoop.instance().start()
You block the whole Tornado event loop for 100 seconds in t.join. Unless you have a yield statement or schedule a callback and exit a function, then your function is not asynchronous. Notice how your function "on_message" begins two threads and then calls t.join on each -- how can Tornado's event loop accomplish any other work while your function is waiting for t.join?
Instead, use a ThreadPoolExecutor something like this:
thread_pool = ThreadPoolExecutor(4)
class TaskHandler(tornado.websocket.WebSocketHandler):
# Make this an asynchronous coroutine
#gen.coroutine
def on_message_coroutine(self, message):
print 'received: ', message
self.write_message(json.dumps({'status': 'running'}))
def worker_A(kwargs):
time.sleep(100)
pass
def worker_B(kwargs):
time.sleep(100)
pass
futures = []
for target in [worker_A, worker_B]:
f = thread_pool.submit(target, {'xxx': 'yyy'})
futures.append(future)
# Now the event loop can do other things
yield futures
def on_message(self, message):
IOLoop.current().spawn_callback(self.on_message_coroutine,
message)