Multi-layer autoencoder using keras, specifying different optimizers - keras

Question

Currently I'm trying to implement a multi-layer autoencoder using Keras, working on the Mnist dataset (handwritten digits). My code is looking like this:
from keras.layers import Input, Dense, initializers
from keras.models import Model
import numpy as np
from Dataset import Dataset
import matplotlib.pyplot as plt
from keras import optimizers, losses
from keras import backend as K
import tensorflow as tf
from keras.callbacks import TensorBoard
from keras.layers import Dropout
from keras.models import Sequential
from keras import models
from keras import layers
import keras
from keras.optimizers import Adam
#global variables
d = Dataset()
num_features = d.X_train.shape[1]
low_dim = 32
def autoencoder(epochs):
w = initializers.RandomNormal(mean=0.0, stddev=0.05, seed=None)
model = Sequential()
#First autoencoder
model.add(Dense(400, activation='relu', kernel_initializer=w, input_dim=num_features, name='hidden'))
model.add(Dropout(0.2))
model.add(Dense(num_features, activation='sigmoid', input_dim = 400, name = 'output'))
#Second autoencoder
model.add(Dense(100, activation='relu', kernel_initializer=w, input_dim=num_features, name='hidden2'))
model.add(Dropout(0.2))
model.add(Dense(num_features, activation = 'sigmoid', input_dim = 100, name='output2'))
#Third autoencoder
model.add(Dense(50, activation='relu', kernel_initializer=w, input_dim=num_features, name='hidden3'))
model.add(Dropout(0.2))
model.add(Dense(num_features, activation='sigmoid', input_dim=10, name='output3'))
model.compile(optimizer=Adam(lr=0.01), loss='binary_crossentropy', metrics=['accuracy'])
history = model.fit(d.X_train, d.X_train,
epochs=epochs,
batch_size=64,
shuffle=True,
validation_data=(d.X_test, d.X_test))
model.test_on_batch(d.X_test, d.X_test)
print(history.history.keys())
plt.plot(history.history['acc'])
print(history.history['acc'])
plt.show()
return model
def finding_index():
elements, index = np.unique(d.Y_test, return_index = True)
return elements, index
def plotting():
ae = autoencoder(2)
elements, index = finding_index()
y_proba = ae.predict(d.X_test)
plt.figure(figsize=(20, 4))
#size = 20
for i in range(len(index)):
ax = plt.subplot(2, len(index), i + 1)
plt.imshow(d.X_test[index[i]].reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
ax = plt.subplot(2, len(index), i + 1 + len(index))
plt.imshow(y_proba[index[i]].reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
plotting()
I have two questions, is it supposed to be like this when you stack autoencoders or should I let one layer reduce dimensions to let's say 400 and then the next to a 100 and so on, or the way I have done it? The second one is, can you different optimizers (in my case Adam) for different layers? I would like to use SGD (stochastic gradient descent) for the last layer. Thanks in advance!

Answer 1

You should not do it the way you've done it, but the way you described it in the question. Also you should go down first and then up again (e.g 400, 100, 50, 25, 10, 25, 50, 100, 400) in granular steps.
For the second question is the answer that it depends. You could train the model with Adam first and then freeze all but the last layer to train this further with SGD. But you can't tell Keras to use different classifiers for different layers.