How can I implement VGG-net on a dataset of different shape? - keras

Question

I am trying to use a part of the VGG16 model for transfer learning using the Fashion MNIST dataset. The data is processed and the model is specified as per below:
data = keras.datasets.fashion_mnist
(train_img, train_labels), (test_img, test_labels) = data.load_data()
train_img.shape, train_labels.shape, test_img.shape, test_labels.shape
#((60000, 28, 28), (60000,), (10000, 28, 28), (10000,))
# transform to rgb as required by VGG
train_img=tf.image.grayscale_to_rgb(tf.expand_dims(train_img, axis=3))
test_img=tf.image.grayscale_to_rgb(tf.expand_dims(test_img, axis=3))
#resize to minimum size of (32x32
train_img=tf.image.resize_with_pad(train_img,32,32)
test_img=tf.image.resize_with_pad(train_img,32,32)
train_img = train_img / 255.
test_img = test_img / 255.
from keras.applications.vgg16 import preprocess_input
train_img = tf.expand_dims(train_img, axis=0)
test_img = tf.expand_dims(test_img, axis=0)
#preprocessing as required by VGG16
train_img=preprocess_input(train_img)
test_img=preprocess_input(test_img)
#using model without last layers
vgg16=tf.keras.applications.VGG16(include_top=False, weights='imagenet', input_shape=(32,32,3))
layer_dict = dict([(layer.name, layer) for layer in vgg16.layers])
#stop at block3_pool and get output
output = layer_dict['block3_pool'].output
x = keras.layers.Flatten()(output)
...add some fully connected layers here...
x = keras.layers.Dense(10, activation='softmax')(x)
final = keras.models.Model(inputs=vgg16.input, outputs=model)
for layer in final.layers[:7]:
layer.trainable = False
final.fit(train_img, train_labels, epochs=50, validation_split=0.2)
When I try to fit the model I get the following error:
UnboundLocalError Traceback (most recent call last)
<ipython-input-65-6a0b99b56337> in <module>()
1 early_stopping_cb=keras.callbacks.EarlyStopping(patience=3, verbose=1,restore_best_weights=True)
----> 2 vgg16_1.fit(train_img, train_labels, epochs=50, validation_split=0.2, callbacks=[early_stopping_cb])
1 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)
857 logs = tmp_logs # No error, now safe to assign to logs.
858 callbacks.on_train_batch_end(step, logs)
--> 859 epoch_logs = copy.copy(logs)
860
861 # Run validation.
UnboundLocalError: local variable 'logs' referenced before assignment
I thought this might be due to the training set shape being faulty, but then if I use train_img[0] instead, which has shape (60000,32,32,3), then I get the following error instead:
ValueError Traceback (most recent call last)
<ipython-input-66-2b893ccd9ac9> in <module>()
1 early_stopping_cb=keras.callbacks.EarlyStopping(patience=3, verbose=1,restore_best_weights=True)
----> 2 vgg16_1.fit(train_img[0], train_labels, epochs=50, validation_split=0.2, callbacks=[early_stopping_cb])
10 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
64 def _method_wrapper(self, *args, **kwargs):
65 if not self._in_multi_worker_mode(): # pylint: disable=protected-access
---> 66 return method(self, *args, **kwargs)
67
68 # Running inside `run_distribute_coordinator` already.
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)
849 batch_size=batch_size):
850 callbacks.on_train_batch_begin(step)
--> 851 tmp_logs = train_function(iterator)
852 # Catch OutOfRangeError for Datasets of unknown size.
853 # This blocks until the batch has finished executing.
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)
578 xla_context.Exit()
579 else:
--> 580 result = self._call(*args, **kwds)
581
582 if tracing_count == self._get_tracing_count():
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)
625 # This is the first call of __call__, so we have to initialize.
626 initializers = []
--> 627 self._initialize(args, kwds, add_initializers_to=initializers)
628 finally:
629 # At this point we know that the initialization is complete (or less
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)
504 self._concrete_stateful_fn = (
505 self._stateful_fn._get_concrete_function_internal_garbage_collected( # pylint: disable=protected-access
--> 506 *args, **kwds))
507
508 def invalid_creator_scope(*unused_args, **unused_kwds):
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py in _get_concrete_function_internal_garbage_collected(self, *args, **kwargs)
2444 args, kwargs = None, None
2445 with self._lock:
-> 2446 graph_function, _, _ = self._maybe_define_function(args, kwargs)
2447 return graph_function
2448
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)
2775
2776 self._function_cache.missed.add(call_context_key)
-> 2777 graph_function = self._create_graph_function(args, kwargs)
2778 self._function_cache.primary[cache_key] = graph_function
2779 return graph_function, args, kwargs
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)
2665 arg_names=arg_names,
2666 override_flat_arg_shapes=override_flat_arg_shapes,
-> 2667 capture_by_value=self._capture_by_value),
2668 self._function_attributes,
2669 # Tell the ConcreteFunction to clean up its graph once it goes out of
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)
979 _, original_func = tf_decorator.unwrap(python_func)
980
--> 981 func_outputs = python_func(*func_args, **func_kwargs)
982
983 # invariant: `func_outputs` contains only Tensors, CompositeTensors,
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, **kwds)
439 # __wrapped__ allows AutoGraph to swap in a converted function. We give
440 # the function a weak reference to itself to avoid a reference cycle.
--> 441 return weak_wrapped_fn().__wrapped__(*args, **kwds)
442 weak_wrapped_fn = weakref.ref(wrapped_fn)
443
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
966 except Exception as e: # pylint:disable=broad-except
967 if hasattr(e, "ag_error_metadata"):
--> 968 raise e.ag_error_metadata.to_exception(e)
969 else:
970 raise
ValueError: in user code:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:571 train_function *
outputs = self.distribute_strategy.run(
/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py:951 run **
return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py:2290 call_for_each_replica
return self._call_for_each_replica(fn, args, kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py:2649 _call_for_each_replica
return fn(*args, **kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:533 train_step **
y, y_pred, sample_weight, regularization_losses=self.losses)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/compile_utils.py:204 __call__
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/losses.py:143 __call__
losses = self.call(y_true, y_pred)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/losses.py:246 call
return self.fn(y_true, y_pred, **self._fn_kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/losses.py:1527 categorical_crossentropy
return K.categorical_crossentropy(y_true, y_pred, from_logits=from_logits)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py:4561 categorical_crossentropy
target.shape.assert_is_compatible_with(output.shape)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/tensor_shape.py:1117 assert_is_compatible_with
raise ValueError("Shapes %s and %s are incompatible" % (self, other))
ValueError: Shapes (32, 1) and (32, 10) are incompatible
Any clues where these errors come from and what I am doing wrong? It feels like I might have missed something obvious, but being a Keras novice I can't get my head around what it is. Help much appreciated.

Answer 1

You need to comment two lines on expanding dims as follows. What happens is that it updates the shape of train_img to (1,60000,32,32,3) and model.fit complains that you are using single image for training.
#train_img = tf.expand_dims(train_img, axis=0)
#test_img = tf.expand_dims(test_img, axis=0)
I updated your code and shared Here. You need to update the architecture to improve it for better accuracy. Follow transfer learning approach mentioned here and update your code for better accuacy. Thanks!

Answer 2

Seems the issue was that I had a dense output layer of size 10, while the labels have size 1. Solution was to use sparse categorical cross-entropy loss function instead of simple categorical.