Tensorflow: Too many open files - multithreading

Question

My code structure is as follows -
I have a main file which calls two function in a loop ->
1- function1()(exist in separate file) which reads the image files from a folder say "Input" do some transformation on images and save these transformed images to a folder say "Output"
2- function2() train a neural network to map images in "Input" folder to images in "Output" folder
Main function is present in mainfile.py , function1() exist in separate file and function2() in different file also (so three files in total)
def function1():
images = reader.image(FLAGS.BATCH_SIZE, FLAGS.HEIGHT, FLAGS.WIDTH, FLAGS.TRAIN_IMAGES_PATH, subtract_mean=True)
\\loss function stuff
with tf.Session(config=config) as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
start_time = time.time()
batches_count = 0
try:
while not coord.should_stop():
writer = tf.summary.FileWriter(FLAGS.summary_path, sess.graph)
sess.run([batch_assign_op,initial_assign_op])
batch_time = time.time()
for step in range(FLAGS.NUM_ITERATIONS):
_, loss_t = sess.run(
[train_op, total_loss])
summary_str = sess.run(summary)
writer.add_summary(summary_str, step)
image_t = sess.run(jpegs)
for i in range(image_t.shape[0]):
filename = '%s/%d.jpg' % (FLAGS.outputdir, (batches_count * FLAGS.BATCH_SIZE + i + 1))
with open(filename, 'wb') as f:
f.write(image_t[i])
f.close()
batches_count = batches_count + 1
after = process.memory_percent()
print("MEMORY CHANGE %.4f -> %.4f" % (before, after))
before = after
except tf.errors.OutOfRangeError:
print("final time elspased", (time.time() - start_time))
print('Done doing non paramteric part')
finally:
coord.request_stop()
coord.join(threads)
Next is the definition of the function 2
def function2():
tf.reset_default_graph()
run_id = FLAGS.MODEL_NAME if FLAGS.MODEL_NAME else str(uuid.uuid4())
model_path = '%s/%s' % (FLAGS.MODEL_DIR, run_id)
if not os.path.exists(model_path):
os.makedirs(model_path)
images, target_ig = reader.net_batch(FLAGS.BATCH_SIZE, FLAGS.HEIGHT, FLAGS.WIDTH, FLAGS.TRAIN_IMAGES_PATH,
FLAGS.TARGET_PATH, epochs=FLAGS.EPOCHS, subtract_mean=False,zero_one=False
,input_mean=FLAGS.input_mean,target_mean=FLAGS.output_mean)
ae_inputs = tf.placeholder(tf.float32, (None, FLAGS.HEIGHT, FLAGS.WIDTH, 3),
name='auto_input') # input to the network (MNIST images)
target = tf.placeholder(tf.float32, (None, FLAGS.HEIGHT, FLAGS.WIDTH, 3),
name='target')
ae_output = model.net(ae_inputs, training=True)
learning_rate = tf.placeholder(tf.float32, shape=[], name='learning_rate')
#ogit_output = tf.nn.sigmoid(ae_output)
# loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=target, logits=logit_output,name="p_loss"))
size = tf.size(ae_output)
loss = tf.nn.l2_loss((target - ae_output), name="p_loss")/tf.to_float(size)
global_step = tf.Variable(FLAGS.gs_val, name="p_global_step", trainable=False)
train_op = tf.train.AdamOptimizer(learning_rate, name="p_trainopt").minimize(loss, global_step=global_step)
# Statistics
average_pl = tf.placeholder(tf.float32,shape=[])
with tf.name_scope('losses'):
tf.summary.scalar('total loss', loss)
tf.summary.image('param generated', (ae_output))
tf.summary.image('original', (ae_inputs))
tf.summary.image('target', (target))
tf.summary.scalar("average_loss", average_pl)
summary = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
with tf.Session(config=config) as sess:
writer = tf.summary.FileWriter(FLAGS.summary_path, sess.graph)
saver = tf.train.Saver(tf.trainable_variables())
file = tf.train.latest_checkpoint(model_path)
sess.run(init_op)
if file:
print('Restoring model from {}'.format(file))
saver.restore(sess, file)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
start_time = time.time()
start_ti = time.time()
acc_loss = []
count =1
try:
while not coord.should_stop():
ig_b, tg_b =sess.run([images,target_ig])
if count % FLAGS.record == 0:
_, loss_t, step, summary_str = sess.run([train_op, loss, global_step, summary],
feed_dict={ae_inputs: ig_b,
target: tg_b,
learning_rate: FLAGS.LEARNING_RATE,
average_pl:(sum(acc_loss)/len(acc_loss))})
print(step,"--> loss: ", loss_t," ,average loss: ",(sum(acc_loss)/len(acc_loss)),
" ,elpased time: ", elapsed_time)
if FLAGS.writesum:
writer.add_summary(summary_str, step)
if count % FLAGS.chanelr == 0:
FLAGS.LEARNING_RATE = FLAGS.LEARNING_RATE / FLAGS.div
print('learning rate now: ', FLAGS.LEARNING_RATE)
acc_loss = []
else:
_, loss_t, step = sess.run([train_op, loss, global_step],
feed_dict={ae_inputs: ig_b,
target: tg_b,
learning_rate: FLAGS.LEARNING_RATE})
acc_loss.append(loss_t)
elapsed_time = time.time() - start_time
start_time = time.time()
count = count+1
except tf.errors.OutOfRangeError:
print(step, loss_t, elapsed_time)
saver.save(sess, model_path + '/style-model',global_step=step)
print("final time elspased", (time.time() - start_ti))
print('Done training -- epoch limit reached')
finally:
coord.request_stop()
coord.join(threads)
return step
And here is the function to read images in batches -
def image(batch, height, width, path, epochs=1, shuffle=False, subtract_mean = False):
# TODO: Find a proper way to do this
ll = [int(i.split('.')[0]) for i in listdir(path)]
ll.sort()
filenames = [join(path, '%d.jpg' %(f)) for f in ll if isfile(join(path, '%d.jpg' %(f)))]
png = filenames[0].lower().endswith('png') # If first file is a png, assume they all are
filename_queue = tf.train.string_input_producer(filenames,
shuffle=shuffle, num_epochs=epochs)
reader = tf.WholeFileReader()
_, img_bytes = reader.read(filename_queue)
image = tf.image.decode_png(img_bytes, channels=3)\
if png else tf.image.decode_jpeg(img_bytes, channels=3)
processed_image = preprocess(image, height,width, subtract_mean=subtract_mean)
return tf.train.batch([processed_image], batch)
def net_batch(batch, height, width, path_input,path_target, epochs=1, zero_one= True, shuffle=False,
subtract_mean = False,input_mean=None, target_mean=None):
#TODO: Find a proper way to do this
ll_in = [int(i.split('.')[0]) for i in listdir(path_input)]
ll_in.sort()
filenames_input = [join(path_input, '%d.jpg' % (f)) for f in ll_in if isfile(join(path_input, '%d.jpg' % (f)))]
#filenames_input = [join(path_input, f) for f in listdir(path_input) if isfile(join(path_input, f))]
png_input = filenames_input[0].lower().endswith('png') # If first file is a png, assume they all are
ll_out = [int(i.split('.')[0]) for i in listdir(path_target)]
ll_out.sort()
filenames_target = [join(path_target, '%d.jpg' % (f)) for f in ll_out if isfile(join(path_target, '%d.jpg' % (f)))]
# filenames_target = [join(path_target, f) for f in listdir(path_target) if isfile(join(path_target, f))]
png_output = filenames_target[0].lower().endswith('png') # If first file is a png, assume they all are
file_queue = tf.train.slice_input_producer([filenames_input,filenames_target],shuffle=True, num_epochs=epochs)
input_file = tf.read_file(file_queue[0])
input_image = tf.image.decode_png(input_file, channels=3)\
if png_input else tf.image.decode_jpeg(input_file, channels=3)
processed_image_input = preprocess(input_image, height,width, subtract_mean=subtract_mean,zero_one=zero_one,
net=True,meandata=input_mean)
target_file = tf.read_file(file_queue[1])
target_image = tf.image.decode_png(target_file, channels=3) \
if png_output else tf.image.decode_jpeg(target_file, channels=3)
processed_image_target = preprocess(target_image, height, width, subtract_mean=subtract_mean,zero_one=zero_one,
net=True,meandata=target_mean)
return tf.train.batch([processed_image_input,processed_image_target], batch)
Problem In between few run of the loop to call function1() and
function2() the whole code crashes giving following error -
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/apport_python_hook.py", line 63, in appo
rt_excepthook
File "<frozen importlib._bootstrap>", line 969, in _find_and_load
File "<frozen importlib._bootstrap>", line 958, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 673, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 661, in exec_module
File "<frozen importlib._bootstrap_external>", line 766, in get_code
File "<frozen importlib._bootstrap_external>", line 818, in get_data
OSError: [Errno 24] Too many open files: '/usr/lib/python3/dist-packages/apport
/__init__.py'
Original exception was:
Traceback (most recent call last):
File "Mainfile.py", line 157, in <module>
File "Mainfile.py", line 131, in main
File "/home/suryabhan/Desktop/New_NST_MAC/slowNST.py", line 246, in Nonpapram
etric
File "/home/suryabhan/Desktop/New_NST_MAC/slowNST.py", line 214, in create_st
yleimage
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/coord
inator.py", line 389, in join
File "/home/suryabhan/.local/lib/python3.5/site-packages/six.py", line 686, i
n reraise
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/queue
_runner_impl.py", line 238, in _run
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session
.py", line 1235, in _single_operation_run
File "/usr/lib/python3.5/contextlib.py", line 66, in __exit__
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/erro
rs_impl.py", line 466, in raise_exception_on_not_ok_status
tensorflow.python.framework.errors_impl.ResourceExhaustedError: Input/76834.jpg
[[Node: ReaderReadV2 = ReaderReadV2[_device="/job:localhost/replica:0/
task:0/cpu:0"](WholeFileReaderV2, input_producer)]]
[[Node: Assert_2/Assert/_72 = _Recv[client_terminated=false, recv_devi
ce="/job:localhost/replica:0/task:0/gpu:0", send_device="/job:localhost/replica
:0/task:0/cpu:0", send_device_incarnation=1, tensor_name="edge_157_Assert_2/Ass
ert", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"]()]
]
I have spend hours tries to fix the issue but nowhere close , my theory is the threads in function1() doesn't stop even after calling coord.request_stop() which keep accumulating open threads and finally code crashes. Even if this is the reason , I really don't know how to solve it.