Python multiprocess spawn process stuck if imported - python-3.x

Question

I am trying to do OCR(Optical Character recognition) on multiple images present in a folder. I am using python multiprocessing module to do it parallely. I am spawning new processes.
When I am running code directly using python command it works fine but when I import Recogniser class and call process_file, it keeps on spawning new process from calling class.
I tried freeze_support but that didn't help.
Right now to make it work, I am doing subprocess call which is not at all right
ImageRecogniser code:
from tesserocr import PyTessBaseAPI
import os
from multiprocessing import get_context, Value
import sys
import time
from datetime import datetime
import json
from PIL import Image
from itertools import islice
queue_counter = Value('i', 0)
class ImageRecogniser:
def __init__(self, input_folder_name):
self.input_folder_name = input_folder_name
self.total_files = 0
def get_images(self, extension=(".png",)):
files_in_dir = os.listdir(self.input_folder_name)
image_list = [os.path.join(self.input_folder_name, file_) for file_ in files_in_dir
if os.path.splitext(file_)[-1] in extension]
self.total_files = len(image_list)
return image_list
def process_images(self, images):
string_list = []
api = PyTessBaseAPI()
for count, image in enumerate(images):
img = Image.open(image)
api.SetImage(img)
output_text = api.GetUTF8Text()
temp_filename = os.path.splitext(os.path.split(image)[-1])[0]
page_number = temp_filename.split("_")[-1]
print("Processed page : {}".format(page_number))
output_dict = {"page_number": page_number, "output": output_text}
string_list.append(output_dict)
api.End()
return string_list
def process_file(self, parallel=False):
print("Getting images from pdf : {}".format(datetime.now()))
image_list = self.get_images()
total_no_of_pages = len(image_list)
print("Initiating extraction : {}".format(datetime.now()))
begin_time = datetime.now()
if parallel:
available_cpus = len(os.sched_getaffinity(0))
pool_workers = available_cpus // 1
if total_no_of_pages > pool_workers:
size_of_queue = total_no_of_pages // pool_workers
split_queue = [size_of_queue] * pool_workers
sum_of_split_queue = sum(split_queue)
if sum_of_split_queue != total_no_of_pages:
pages_left = total_no_of_pages - sum_of_split_queue
for i in range(pages_left):
split_queue[i] = split_queue[i] + 1
else:
split_queue = [0] * pool_workers
for i in range(total_no_of_pages):
split_queue[i] = 1
# page_ids_list_iterator = iter(page_ids_list)
print('Split queue : {}'.format(split_queue))
to_be_processed = []
start = 1
for chunksize in split_queue:
end = chunksize + start
to_be_processed.append(image_list[start:end])
start = end
with get_context('spawn').Pool(processes=pool_workers) as pool:
result = pool.map_async(self.process_images, to_be_processed)
output = result.get()
output = [actual_value for queue_list in output for actual_value in queue_list]
#json_output = json.dumps(output)
else:
output = self.process_images(image_list)
json_file_name = self.input_folder_name+ ".json"
with open(json_file_name, "w") as fp:
json.dump(output, fp)
if __name__ == '__main__':
source = sys.argv[1]
t = ImageRecogniser(source)
try:
current = os.environ["OMP_THREAD_LIMIT"]
except:
current = None
os.environ["OMP_THREAD_LIMIT"] = "1"
t.process_file(parallel=True)
if current:
os.environ["OMP_THREAD_LIMIT"] = str(current)
worker.py
import os
import logging
import yaml
from pdf_converter.pdf_to_image import PDFtoImage
from pdf_converter.settings import DATA_FOLDER_PATH, LOGGER_FILE_PATH
from recogniser.image_recogniser import ImageRecogniser
import subprocess
with open(LOGGER_FILE_PATH, 'r') as f:
config = yaml.safe_load(f.read())
logging.config.dictConfig(config)
logger = logging.getLogger(__name__)
logger.info("STARTING WORKER")
file_name = "4506-T.pdf"
file_path = os.path.join(DATA_FOLDER_PATH, file_name)
logger.info("STARTING PDFTOIMAGE CONVERSION")
pdf_image = PDFtoImage(file_path)
files = pdf_image.run()
logger.info("CREATED {} IMAGE FILES FROM {}".format(len(files),file_name))
image_files_path = os.path.join(DATA_FOLDER_PATH, os.path.splitext(file_name)[0])
# subprocess.call(['python', 'recogniser/image_recogniser_cmd.py', image_files_path])
img_rec = ImageRecogniser(image_files_path)
img_rec.process_file(parallel=True)
error
2020-07-27 19:57:49,244 - __mp_main__ - INFO - CREATED 53 IMAGE FILES FROM 4506-T.pdf
Getting images from pdf : /home/shashank-mq/project/doOCR/data/4506-T.pdf
Initiating extraction : 2020-07-27 19:57:49.245889
Split queue : [7, 7, 7, 7, 7, 6, 6, 6]
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "/usr/lib/python3.6/multiprocessing/spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "/usr/lib/python3.6/multiprocessing/spawn.py", line 114, in _main
prepare(preparation_data)
File "/usr/lib/python3.6/multiprocessing/spawn.py", line 225, in prepare
_fixup_main_from_path(data['init_main_from_path'])
File "/usr/lib/python3.6/multiprocessing/spawn.py", line 277, in _fixup_main_from_path
run_name="__mp_main__")
File "/usr/lib/python3.6/runpy.py", line 263, in run_path
pkg_name=pkg_name, script_name=fname)
File "/usr/lib/python3.6/runpy.py", line 96, in _run_module_code
mod_name, mod_spec, pkg_name, script_name)
File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/home/shashank-mq/project/doOCR/worker.py", line 27, in <module>
img_rec.process_file(parallel=True)
File "/home/shashank-mq/project/doOCR/recogniser/image_recogniser.py", line 78, in process_file
with get_context('spawn').Pool(processes=pool_workers) as pool:
File "/usr/lib/python3.6/multiprocessing/context.py", line 119, in Pool
context=self.get_context())
File "/usr/lib/python3.6/multiprocessing/pool.py", line 174, in __init__
self._repopulate_pool()
File "/usr/lib/python3.6/multiprocessing/pool.py", line 239, in _repopulate_pool
w.start()
File "/usr/lib/python3.6/multiprocessing/process.py", line 105, in start
self._popen = self._Popen(self)
File "/usr/lib/python3.6/multiprocessing/context.py", line 284, in _Popen
return Popen(process_obj)
File "/usr/lib/python3.6/multiprocessing/popen_spawn_posix.py", line 32, in __init__
super().__init__(process_obj)
File "/usr/lib/python3.6/multiprocessing/popen_fork.py", line 19, in __init__
self._launch(process_obj)
File "/usr/lib/python3.6/multiprocessing/popen_spawn_posix.py", line 42, in _launch
prep_data = spawn.get_preparation_data(process_obj._name)
File "/usr/lib/python3.6/multiprocessing/spawn.py", line 143, in get_preparation_data
_check_not_importing_main()
File "/usr/lib/python3.6/multiprocessing/spawn.py", line 136, in _check_not_importing_main
is not going to be frozen to produce an executable.''')
RuntimeError:
An attempt has been made to start a new process before the
current process has finished its bootstrapping phase.
This probably means that you are not using fork to start your
child processes and you have forgotten to use the proper idiom
in the main module:
if __name__ == '__main__':
freeze_support()
...
The "freeze_support()" line can be omitted if the program
is not going to be frozen to produce an executable.
Please suggest what I am missing and how to do it