Function call stack: train_function -> train_function -> train_function - python-3.x

Question

My training data is a list of different length numpy arrays. For example,
x_train[0] = [[ 0.67836523 0.39823654 0.9661108 ... 0.19785157 0.1766675
0.6182588 ]
[-1.664766 -0.360997 0.096446 ... -0.635498 0.300886
-0.045028 ]
[-0.615297 -0.190688 -0.226994 ... 1.648792 -1.691676
-0.411259 ]
...
[-1.380328 -0.231574 -0.078576 ... 1.54852 -1.323094
1.493816 ]
[-2.35968 -4.016114 1.077576 ... -1.23973 -0.65608
1.095033 ]
[ 0.551824 0.115759 -0.163607 ... -0.285045 0.472944
-0.664072 ]]
Here is examples of x_train y_train dimension:
x_train[0].shape = (1136, 512) x_train[1].shape = (650, 512)...etc
y_train[0].shape = (1136, 19) y_train[1] = (650, 19)...etc
Here is my model:
model = Sequential()
model.add(GRU(128, return_sequences=True, input_shape=(None, 512)))
model.add(GRU(128, return_sequences=True))
model.add(TimeDistributed(Dense(19, activation='softmax')))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
plot_model(
model,
to_file='model.png',
show_shapes=True,
show_layer_names=False,
rankdir='LR')
And a data generator for creating batch of similar sequence length:
def train_generator(x, y):
while True:
index = np.random.randint(len(x))
Xb = np.expand_dims(x[index], axis=0)
yb = np.expand_dims(y[index], axis=0)
for i in range(len(x)):
if (0 <= len(x[i]) - len(x[index]) <= 200) & (i != index):
x_tmp = np.expand_dims(x[i][:len(x[index])], axis=0)
y_tmp = np.expand_dims(y[i][:len(y[index])], axis=0)
Xb = np.append(x_tmp, Xb, 0)
yb = np.append(y_tmp, yb, 0)
yield (Xb, yb)
Then fit:
model.fit_generator(train_generator(x_train, y_train), epochs=10, verbose=1)
Here is the error I'm getting:
Epoch 1/10
---------------------------------------------------------------------------
UnknownError Traceback (most recent call last)
<ipython-input-324-9e3ade034201> in <module>
----> 1 model.fit_generator(train_generator(x_train, y_train), epochs=10, verbose=1)
~\miniconda3\lib\site-packages\tensorflow\python\util\deprecation.py in new_func(*args, **kwargs)
322 'in a future version' if date is None else ('after %s' % date),
323 instructions)
--> 324 return func(*args, **kwargs)
325 return tf_decorator.make_decorator(
326 func, new_func, 'deprecated',
~\miniconda3\lib\site-packages\tensorflow\python\keras\engine\training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1827 use_multiprocessing=use_multiprocessing,
1828 shuffle=shuffle,
-> 1829 initial_epoch=initial_epoch)
1830
1831 #deprecation.deprecated(
~\miniconda3\lib\site-packages\tensorflow\python\keras\engine\training.py in _method_wrapper(self, *args, **kwargs)
106 def _method_wrapper(self, *args, **kwargs):
107 if not self._in_multi_worker_mode(): # pylint: disable=protected-access
--> 108 return method(self, *args, **kwargs)
109
110 # Running inside `run_distribute_coordinator` already.
~\miniconda3\lib\site-packages\tensorflow\python\keras\engine\training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
1096 batch_size=batch_size):
1097 callbacks.on_train_batch_begin(step)
-> 1098 tmp_logs = train_function(iterator)
1099 if data_handler.should_sync:
1100 context.async_wait()
~\miniconda3\lib\site-packages\tensorflow\python\eager\def_function.py in __call__(self, *args, **kwds)
778 else:
779 compiler = "nonXla"
--> 780 result = self._call(*args, **kwds)
781
782 new_tracing_count = self._get_tracing_count()
~\miniconda3\lib\site-packages\tensorflow\python\eager\def_function.py in _call(self, *args, **kwds)
838 # Lifting succeeded, so variables are initialized and we can run the
839 # stateless function.
--> 840 return self._stateless_fn(*args, **kwds)
841 else:
842 canon_args, canon_kwds = \
~\miniconda3\lib\site-packages\tensorflow\python\eager\function.py in __call__(self, *args, **kwargs)
2827 with self._lock:
2828 graph_function, args, kwargs = self._maybe_define_function(args, kwargs)
-> 2829 return graph_function._filtered_call(args, kwargs) # pylint: disable=protected-access
2830
2831 #property
~\miniconda3\lib\site-packages\tensorflow\python\eager\function.py in _filtered_call(self, args, kwargs, cancellation_manager)
1846 resource_variable_ops.BaseResourceVariable))],
1847 captured_inputs=self.captured_inputs,
-> 1848 cancellation_manager=cancellation_manager)
1849
1850 def _call_flat(self, args, captured_inputs, cancellation_manager=None):
~\miniconda3\lib\site-packages\tensorflow\python\eager\function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
1922 # No tape is watching; skip to running the function.
1923 return self._build_call_outputs(self._inference_function.call(
-> 1924 ctx, args, cancellation_manager=cancellation_manager))
1925 forward_backward = self._select_forward_and_backward_functions(
1926 args,
~\miniconda3\lib\site-packages\tensorflow\python\eager\function.py in call(self, ctx, args, cancellation_manager)
548 inputs=args,
549 attrs=attrs,
--> 550 ctx=ctx)
551 else:
552 outputs = execute.execute_with_cancellation(
~\miniconda3\lib\site-packages\tensorflow\python\eager\execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
58 ctx.ensure_initialized()
59 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 60 inputs, attrs, num_outputs)
61 except core._NotOkStatusException as e:
62 if name is not None:
UnknownError: Fail to find the dnn implementation.
[[{{node CudnnRNN}}]]
[[sequential_18/gru_36/PartitionedCall]] [Op:__inference_train_function_54574]
Function call stack:
train_function -> train_function -> train_function
I've tried googling the error myself, but didn't found any solution.
Any help would be massively appreciated :)