Adding a layer stops learning Keras - keras

Question

Code
import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential,Model
from keras.layers import LeakyReLU,Dropout, Flatten, Dense,Input
from keras import applications
from keras.preprocessing import image
from keras import backend as K
from keras import regularizers
from keras.optimizers import adam
K.set_image_dim_ordering('tf')
input_tensor = Input(shape=(150,150,3))
img_width, img_height = 150,150
top_model_weights_path = 'bottleneck_fc_model.h5'
train_data_dir = 'Cats and Dogs Dataset/train'
validation_data_dir = 'Cats and Dogs Dataset/validation'
nb_train_samples = 20000
nb_validation_samples = 5000
epochs = 50
batch_size = 128
base_model=applications.inception_v3.InceptionV3(include_top=False, weights='imagenet', input_tensor=input_tensor, pooling=None)
i=0;
for layer in base_model.layers:
layer.trainable = False
i+=1
base_model.output
top_model=Sequential()
top_model.add(Flatten(input_shape=base_model.output_shape[1:]))
top_model.add(Dense(1024,activation="relu"))
top_model.add(Dropout(0.5))
top_model.add(Dense(10,activation="relu"))//Layer with issue
top_model.add(Dropout(0.8))//
top_model.add(Dense(2, activation='softmax'))
model = Model(inputs=base_model.input,outputs=top_model(base_model.output))
model.summary
datagen = ImageDataGenerator(rescale=1. / 255)
train_data = datagen.flow_from_directory(train_data_dir,target_size=(img_width, img_height),batch_size=batch_size,classes=[ 'cats','dogs'])#,class_mode="binary",shuffle=True)
validation_data = datagen.flow_from_directory(validation_data_dir,target_size=(img_width, img_height), batch_size=batch_size,classes=['cats','dogs'])#,class_mode="binary",shuffle=True)
adm=adam(lr=0.02)
model.compile(optimizer=adm,loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(train_data, steps_per_epoch=nb_train_samples//batch_size, epochs=epochs,validation_data=validation_data, shuffle=True,verbose=1)
I have implemented a Image Classifier on the cats and dogs Dataset(https://www.kaggle.com/c/dogs-vs-cats/data) using keras(transfer learned using the inception netowrk). The code runs without errors but the accuracy is stuck at 50% for the validation set and the training set from the first epoch and the loss isnt decreasing. I am using Atom with hydrogen.
The issue goes away when I remove the marked layer , I cant seem to understand why this is happening.
What I have tried to fix this
different batch sizes - 4,16,64,256
change optimizer - tried adam ,rmsprop , sgd with modified learning rates
tried different activations for the layer - relu,sigmoid and leakyrelu
changed the dropout - the issue vanishes when dropout is 0.9(i.e. make the
layer useless, this works for obvious reason but also points out there is something that i am missing )
changed the final activation to sigmoid
Can someone please tell me what i am missing because i cant think of any reason why adding a layer stops learning

Answer 1

import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential,Model
from keras.layers import LeakyReLU,Dropout, Flatten, Dense,Input
from keras import applications
from keras.preprocessing import image
from keras import backend as K
from keras import regularizers
from keras.optimizers import adam
K.set_image_dim_ordering('tf')
input_tensor = Input(shape=(150,150,3))
img_width, img_height = 150,150
top_model_weights_path = 'bottleneck_fc_model.h5'
train_data_dir = 'Cats and Dogs Dataset/train'
validation_data_dir = 'Cats and Dogs Dataset/validation'
nb_train_samples = 20000
nb_validation_samples = 5000
epochs = 50
batch_size = 64
base_model=applications.inception_v3.InceptionV3(include_top=False, weights='imagenet', input_tensor=input_tensor, pooling=None)
i=0;
for layer in base_model.layers:
layer.trainable = False
i+=1
base_model.output
top_model=Sequential()
top_model.add(Flatten(input_shape=base_model.output_shape[1:]))
top_model.add(Dense(512,activation="relu")) //decrease in units
top_model.add(Dropout(0.4)) // change the drop out
top_model.add(Dense(128,activation="relu")) //increase in units
top_model.add(Dropout(0.2)) // decrease in dropout
top_model.add(Dense(2, activation='softmax'))
model = Model(inputs=base_model.input,outputs=top_model(base_model.output))
model.summary
datagen = ImageDataGenerator(rescale=1. / 255)
train_data = datagen.flow_from_directory(train_data_dir,target_size=(img_width, img_height),batch_size=batch_size,classes=[ 'cats','dogs'])#,class_mode="binary",shuffle=True)
validation_data = datagen.flow_from_directory(validation_data_dir,target_size=(img_width, img_height), batch_size=batch_size,classes=['cats','dogs'])#,class_mode="binary",shuffle=True)
adm=adam(lr=0.02)
model.compile(optimizer=adm,loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(train_data, steps_per_epoch=nb_train_samples//batch_size, epochs=epochs,validation_data=validation_data, shuffle=True,verbose=1)
I have reduce the number of units in first dense layer while increase the number of units in 2nd dense layer .. and also deceasing the drop out rate .. run this code and let me know. one more thing more complex the network is higher the chance of over-fitting .. increase in dropout value might result in no learning for that layer. try to keep you network simple .