Filling queue from python iterator - python-3.x

Question

I would like to create a queue which is filled from an iterator. In the following MWE however, always the same value is enqueued:
import tensorflow as tf
import numpy as np
# data
imgs = [np.random.randn(i,i) for i in [2,3,4,5]]
# iterate through data infinitly
def data_iterator():
while True:
for img in imgs:
yield img
it = data_iterator()
# create queue for data
q = tf.FIFOQueue(capacity=5, dtypes=[tf.float64])
# feed next element from iterator
enqueue_op = q.enqueue(list(next(it)))
# setup queue runner
numberOfThreads = 1
qr = tf.train.QueueRunner(q, [enqueue_op] * numberOfThreads)
tf.train.add_queue_runner(qr)
# dequeue
dequeue_op = q.dequeue()
dequeue_op = tf.Print(dequeue_op, data=[dequeue_op], message="dequeue()")
# We start the session as usual ...
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(10):
data = sess.run(dequeue_op)
print(data)
.
coord.request_stop()
coord.join(threads)
Do I necessarily have to use a feed_dict? If yes, how do I have to used it in combination with the QueueRunner?

Answer 1

When running
enqueue_op = q.enqueue(list(next(it)))
tensorflow will execute list(next(it)) exactly one time. Thereafter it will save this first list and add it to q every time you run enqueue_op. To avoid this you must use a placeholder. Feeding placeholders is not compatible with tf.train.QueueRunner. Instead use this:
import tensorflow as tf
import numpy as np
import threading
# data
imgs = [np.random.randn(i,i) for i in [2,3,4,5]]
# iterate through data infinitly
def data_iterator():
while True:
for img in imgs:
yield img
it = data_iterator()
# create queue for data
q = tf.FIFOQueue(capacity=5, dtypes=[tf.float64])
# feed next element from iterator
img_p = tf.placeholder(tf.float64, [None, None])
enqueue_op = q.enqueue(img_p)
dequeue_op = q.dequeue()
with tf.Session() as sess:
coord = tf.train.Coordinator()
def enqueue_thread():
with coord.stop_on_exception():
while not coord.should_stop():
sess.run(enqueue_op, feed_dict={img_p: list(next(it))})
numberOfThreads = 1
for i in range(numberOfThreads):
threading.Thread(target=enqueue_thread).start()
for i in range(3):
data = sess.run(dequeue_op)
print(data)