I'm working with word embeddings. I obtained word embeddings using 'BERT'.
I have a data like this
1992 regular unleaded 172 6 MANUAL all wheel drive 4 Luxury Midsize Sedan 21 16 3105 200
and as a label:
df['Make'] = df['Make'].replace(['Chrysler'],1)
I try to give embeddings as a LSTM inputs.
Using below code for BERT:
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
bert_model = BertModel.from_pretrained('bert-base-uncased')
For tokinizing this code:
def tokenize_text(df, max_seq):
return [
tokenizer.encode(text, add_special_tokens=True)[:max_seq] for text in df
]
def pad_text(tokenized_text, max_seq):
return np.array([el + [0] * (max_seq - len(el)) for el in tokenized_text])
def tokenize_and_pad_text(df, max_seq):
tokenized_text = tokenize_text(df, max_seq)
padded_text = pad_text(tokenized_text, max_seq)
return padded_text
and :
train_indices = tokenize_and_pad_text(df_train, max_seq)
for BERT embedding matrix:
def get_bert_embed_matrix():
bert = transformers.BertModel.from_pretrained('bert-base-uncased')
bert_embeddings = list(bert.children())[0]
bert_word_embeddings = list(bert_embeddings.children())[0]
mat = bert_word_embeddings.weight.data.numpy()
return mat
embedding_matrix = get_bert_embed_matrix()
and LSTM Model:
embedding_layer =Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1], weights=[embedding_matrix], input_length=max_seq_len, trainable=True)
model = Sequential()
model.add(embedding_layer)
model.add(LSTM(128, dropout=0.3, recurrent_dropout=0.3))
model.add(Dense(1, activation='softmax'))
model.compile(
optimizer=tf.keras.optimizers.Adam(1e-5),
loss="categorical_crossentropy",
metrics=["accuracy"],
)
model.summary()
and for model fit this code:
model.fit(train_indices, y_train, epochs=20, verbose=1)
I give a output like this:
Epoch 1/20
1/1 [==============================] - 3s 3s/step - loss: 0.0000e+00 - accuracy: 0.3704
Epoch 20/20
1/1 [==============================] - 0s 484ms/step - loss: 0.0000e+00 - accuracy: 0.3704
I don't know what I'm missing.
Firstly, what can we do about it?
Secondly, how can we implement Pytorch Model?
Thanks a lot.
Related
I’m new to pytorch and my problem may be a little naive
I’m training a pretrained VGG16 network on my dataset which it’s size is near 33000 images in 8 classes with labels [1,2,…,8] and my classes are imbalanced. my problem is that during training, validation and training accuracy is low and doesn’t increase, is there any problem in my code?
if not, what do you suggest to improve training?
'''
import torch
import time
import torch.nn as nn
import numpy as np
from sklearn.model_selection import train_test_split
from torch.optim import Adam
import cv2
import torchvision.models as models
from classify_dataset import Classification_dataset
from torchvision import transforms
transform = transforms.Compose([transforms.Resize((224,224)),
transforms.RandomHorizontalFlip(p=0.5),
transforms.RandomVerticalFlip(p=0.5),
transforms.RandomRotation(degrees=45),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
dataset = Classification_dataset(root_dir=r'//home/arisa/Desktop/Hamid/IQA/Hamid_Dataset',
csv_file=r'/home/arisa/Desktop/Hamid/IQA/new_label.csv',transform=transform)
target = dataset.labels - 1
train_indices, test_indices = train_test_split(np.arange(target.shape[0]), stratify=target)
test_dataset = torch.utils.data.Subset(dataset, indices=test_indices)
train_dataset = torch.utils.data.Subset(dataset, indices=train_indices)
class_sample_count = np.array([len(np.where(target[train_indices] == t)[0]) for t in np.unique(target)])
weight = 1. / class_sample_count
samples_weight = np.array([weight[t] for t in target[train_indices]])
samples_weight = torch.from_numpy(samples_weight)
samples_weight = samples_weight.double()
sampler = torch.utils.data.WeightedRandomSampler(samples_weight, len(samples_weight), replacement = True)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=64,
sampler=sampler)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=64,
shuffle=False)
for param in model.parameters():
param.requires_grad = False
num_ftrs = model.classifier[0].in_features
model.classifier = nn.Linear(num_ftrs,8)
optimizer = Adam(model.parameters(), lr = 0.0001 )
criterion = nn.CrossEntropyLoss()
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.01)
path = '/home/arisa/Desktop/Hamid/IQA/'
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print(device)
def train_model(model, train_loader,valid_loader, optimizer, criterion, scheduler=None, num_epochs=10 ):
min_valid_loss = np.inf
model.train()
start = time.time()
TrainLoss = []
model = model.to(device)
for epoch in range(num_epochs):
total = 0
correct = 0
train_loss = 0
#lr_scheduler.step()
print('Epoch {}/{}'.format(epoch+1, num_epochs))
print('-' * 10)
train_loss = 0.0
for x,y in train_loader:
x = x.to(device)
#print(y.shape)
y = y.view(y.shape[0],).to(device)
y = y.to(device)
y -= 1
out = model(x)
loss = criterion(out, y)
optimizer.zero_grad()
loss.backward()
TrainLoss.append(loss.item()* y.shape[0])
train_loss += loss.item() * y.shape[0]
_,predicted = torch.max(out.data,1)
total += y.size(0)
correct += (predicted == y).sum().item()
optimizer.step()
lr_scheduler.step()
accuracy = 100*correct/total
valid_loss = 0.0
val_loss = []
model.eval()
val_correct = 0
val_total = 0
with torch.no_grad():
for x_val, y_val in test_loader:
x_val = x_val.to(device)
y_val = y_val.view(y_val.shape[0],).to(device)
y_val -= 1
target = model(x_val)
loss = criterion(target, y_val)
valid_loss += loss.item() * y_val.shape[0]
_,predicted = torch.max(target.data,1)
val_total += y_val.size(0)
val_correct += (predicted == y_val).sum().item()
val_loss.append(loss.item()* y_val.shape[0])
val_acc = 100*val_correct / val_total
print(f'Epoch {epoch + 1} \t\t Training Loss: {train_loss / len(train_loader)} \t\t Validation Loss: {valid_loss / len(test_loader)} \t\t Train Acc:{accuracy} \t\t Validation Acc:{val_acc}')
if min_valid_loss > (valid_loss / len(test_loader)):
print(f'Validation Loss Decreased({min_valid_loss:.6f}--->{valid_loss / len(test_loader):.6f}) \t Saving The Model')
min_valid_loss = valid_loss / len(test_loader)
state = {'state_dict': model.state_dict(),'optimizer': optimizer.state_dict(),}
torch.save(state,'/home/arisa/Desktop/Hamid/IQA/checkpoint.t7')
end = time.time()
print('TRAIN TIME:')
print('%.2gs'%(end-start))
train_model(model=model, train_loader=train_loader, optimizer=optimizer, criterion=criterion, valid_loader= test_loader,num_epochs=500 )
Thanks in advance
here is the result of 15 epoch
Epoch 1/500
----------
Epoch 1 Training Loss: 205.63448420514916 Validation Loss: 233.89266112356475 Train Acc:39.36360386127994 Validation Acc:24.142040038131555
Epoch 2/500
----------
Epoch 2 Training Loss: 199.05699240435197 Validation Loss: 235.08799531243065 Train Acc:41.90998291820601 Validation Acc:24.27311725452812
Epoch 3/500
----------
Epoch 3 Training Loss: 199.15626737127448 Validation Loss: 236.00033430619672 Train Acc:41.1035633416756 Validation Acc:23.677311725452814
Epoch 4/500
----------
Epoch 4 Training Loss: 199.02581041173886 Validation Loss: 233.60767459869385 Train Acc:41.86628530568466 Validation Acc:24.606768350810295
Epoch 5/500
----------
Epoch 5 Training Loss: 198.61493769454472 Validation Loss: 233.7503859202067 Train Acc:41.53656695665991 Validation Acc:25.0
Epoch 6/500
----------
Epoch 6 Training Loss: 198.71323942956585 Validation Loss: 234.17176149830675 Train Acc:41.639852222619474 Validation Acc:25.369399428026693
Epoch 7/500
----------
Epoch 7 Training Loss: 199.9395153770592 Validation Loss: 234.1744423635078 Train Acc:40.98041552456998 Validation Acc:24.84509056244042
Epoch 8/500
----------
Epoch 8 Training Loss: 199.3533399020355 Validation Loss: 235.4645173188412 Train Acc:41.26643626107337 Validation Acc:24.165872259294567
Epoch 9/500
----------
Epoch 9 Training Loss: 199.6451746921249 Validation Loss: 233.33387595956975 Train Acc:40.96452548365312 Validation Acc:24.59485224022879
Epoch 10/500
----------
Epoch 10 Training Loss: 197.9305159737011 Validation Loss: 233.76405122063377 Train Acc:41.8782028363723 Validation Acc:24.6186844613918
Epoch 11/500
----------
Epoch 11 Training Loss: 199.33247244055502 Validation Loss: 234.41085289463854 Train Acc:41.59218209986891 Validation Acc:25.119161105815063
Epoch 12/500
----------
Epoch 12 Training Loss: 199.87399289874256 Validation Loss: 234.23621463775635 Train Acc:41.028085647320545 Validation Acc:24.49952335557674
Epoch 13/500
----------
Epoch 13 Training Loss: 198.85540591944292 Validation Loss: 234.33149099349976 Train Acc:41.206848607635166 Validation Acc:24.857006673021925
Epoch 14/500
----------
Epoch 14 Training Loss: 199.92641723337513 Validation Loss: 233.37722391070741 Train Acc:41.15520597465539 Validation Acc:24.988083889418494
Epoch 15/500
----------
Epoch 15 Training Loss: 197.82172771698328 Validation Loss: 234.4943131533536 Train Acc:41.69943987605768 Validation Acc:24.380362249761678
You freezed your model through
for param in model.parameters():
param.requires_grad = False
which basically says "do not calculate any gradient for any weight" which is equivalent of not updating weights - hence no optimization
my problem was in model.train(). This phrase should be inside the training loop. but in my case I put it outside the training loop and when it comes to model.eval(), model maintained in this mode
My LSTM model is like this, and I would like to get state_c
def _get_model(input_shape, latent_dim, num_classes):
inputs = Input(shape=input_shape)
lstm_lyr,state_h,state_c = LSTM(latent_dim,dropout=0.1,return_state = True)(inputs)
fc_lyr = Dense(num_classes)(lstm_lyr)
soft_lyr = Activation('relu')(fc_lyr)
model = Model(inputs, [soft_lyr,state_c])
model.compile(optimizer='adam', loss='mse', metrics=['accuracy'])
return model
model =_get_model((n_steps_in, n_features),latent_dim ,n_steps_out)
history = model.fit(X_train,Y_train)
But I canot extract the state_c from the history. How to return that?
I am unsure of what you mean by "How to get state_c", because your LSTM layer is already returning the state_c with the flag return_state=True. I assume you are trying to train the multi-output model in this case. Currently, you only have a single output but your model is compiled with multiple outputs.
Here is how you work with multi-output models.
from tensorflow.keras import layers, Model, utils
def _get_model(input_shape, latent_dim, num_classes):
inputs = layers.Input(shape=input_shape)
lstm_lyr,state_h,state_c = layers.LSTM(latent_dim,dropout=0.1,return_state = True)(inputs)
fc_lyr = layers.Dense(num_classes)(lstm_lyr)
soft_lyr = layers.Activation('relu')(fc_lyr)
model = Model(inputs, [soft_lyr,state_c]) #<------- One input, 2 outputs
model.compile(optimizer='adam', loss='mse')
return model
#Dummy data
X = np.random.random((100,15,5))
y1 = np.random.random((100,4))
y2 = np.random.random((100,7))
model =_get_model((15, 5), 7 , 4)
model.fit(X, [y1,y2], epochs=4) #<--------- #One input, 2 outputs
Epoch 1/4
4/4 [==============================] - 2s 6ms/step - loss: 0.6978 - activation_9_loss: 0.2388 - lstm_9_loss: 0.4591
Epoch 2/4
4/4 [==============================] - 0s 6ms/step - loss: 0.6615 - activation_9_loss: 0.2367 - lstm_9_loss: 0.4248
Epoch 3/4
4/4 [==============================] - 0s 7ms/step - loss: 0.6349 - activation_9_loss: 0.2392 - lstm_9_loss: 0.3957
Epoch 4/4
4/4 [==============================] - 0s 8ms/step - loss: 0.6053 - activation_9_loss: 0.2392 - lstm_9_loss: 0.3661
Is it possible to use LSTM together with an array of words that I've classified ?
For example I have a array with 1000 words:
'Green'
'Blue'
'Red'
'Yellow'
I classify the words to be Green = 0, Blue = 1 , Red = 2, Yellow = 3.
And I want to predict the 4th word. The words can come in different orders in the sequence. For example first sequence can be input = green, blue, red, target = yellow next sequence is input = blue,red,yellow, target = green and so on.
Maybe I shouldn't use LSTM for this, but I guess I should since I want to check the 3 earliers inputs and predict the 4th.
This is what I have so far, I'm more or less stuck with the reshape of my words list. and I can't really understand what input_shape I should have. I guess it's Timesteps = 3, and features = 4
# define documents
words = [0,1,2,3,2,3,1,0,0,1,2,3,2,0,3,1,1,2,3,0]
words_cat = to_categorical(words,4)
X_train = ?
y_train = ?
# define the model
model = Sequential()
model.add(LSTM(32, input_shape=(3,4)))
model.add(Dense(4, activation='softmax'))
# compile the model
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
# summarize the model
print(model.summary())
# fit the model
model.fit(X_train, y_train epochs=50, verbose=0)
Br
As already mentioned by the first comment, a LSTM network is maybe a little bit of an overkill in this case. But I assume you do this for pedagogic reasons.
Here's a working example:
# define documents
words = [0,1,2,3,2,3,1,0,0,1,2,3,2,0,3,1,1,2,3,0]
# create labels
labels = np.roll(words[:-3], -3)
X_train = np.array([words[i:(i+3)%len(words)] for i in range(len(words)-3)]).reshape(-1,1,3)
y_train = labels
# define the model
model = Sequential()
model.add(LSTM(32, input_shape=(None,3)))
model.add(Dense(4, activation='softmax'))
# compile the model
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
# summarize the model
print(model.summary())
# fit the model
model.fit(X_train, y_train, epochs=5, batch_size=1, verbose=1)
preds = model.predict(X_train).argmax(1)
print(preds)
print(y_train)
Output:
Epoch 1/5
17/17 [==============================] - 2s 88ms/step - loss: 1.3771 - accuracy: 0.1765
Epoch 2/5
17/17 [==============================] - 0s 9ms/step - loss: 1.3647 - accuracy: 0.3529
Epoch 3/5
17/17 [==============================] - 0s 6ms/step - loss: 1.3568 - accuracy: 0.2353
Epoch 4/5
17/17 [==============================] - 0s 8ms/step - loss: 1.3496 - accuracy: 0.2353
Epoch 5/5
17/17 [==============================] - 0s 7ms/step - loss: 1.3420 - accuracy: 0.4118
[1 2 1 2 0 0 0 1 1 2 1 0 2 1 1 1 2]
[3 2 3 1 0 0 1 2 3 2 0 3 1 1 0 1 2]
So I took the words provided by you and reshaped them. The first three entries are the series to train on and the fourth entry is the label.
If your sequence is random the model will have a hard time predicting the next value. Otherwise you might wanna train longer or provide more examples (however the number of combinations in this case is fairly limited).
Please tell me if it's possible to change scoring for sklearn.model_selection.cross_val_score ? It returns the negative value of the mean_absolute_percentage_error regardless of my choice in the estimator. For example if I select metrics=['mean_absolute_error'] it would be changed during ANN training but cross_val_score will still return the negative value of the mean_absolute_percentage.
My code is:
def create_network():
model = models.Sequential()
model.add(layers.Dense(64, activation='relu', input_shape=(X.shape[1],)))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(1))
model.compile(optimizer='rmsprop',
loss='mse',
metrics=['mean_absolute_error'])
return model
from keras.wrappers.scikit_learn import KerasRegressor
neural_network = KerasRegressor(build_fn=create_network,
epochs=20,
batch_size=10,
verbose=1)
X=feature_normalization(X)[0]
from sklearn.model_selection import cross_val_score
scores = cross_val_score(neural_network, X, y, cv=4)
print ('Scores:',scores)
print ('Average score:',np.average(scores))
Results:
Epoch 20/20
380/380 [==============================] - 0s 82us/step - loss: 18.5750 - mean_absolute_error: 2.9808
126/126 [==============================] - 0s 124us/step
Scores: [-27.60875144 -15.73322312 -35.57359647 -21.53566427]
Average score: -25.112808823950544
I am doing some image classification using inception_v3 model in keras, however, my train accuracy is lower than validation during the whole training process. And my validation accuracy is above 0.95 from the first epoch. I also find that train loss is much higher than validation loss. In the end, the test accuracy is 0.5, which is pretty bad.
At first, my optimizer is Adam with learning rate equals to 0.00001, the result is bad. Then I change it to SGD with learning rate of 0.00001, which doesn't make any change to the bad result. I also tried to increase the learning rate to 0.1, but the test accuracy is still around 0.5
import numpy as np
import pandas as pd
import keras
from keras import layers
from keras.applications.inception_v3 import preprocess_input
from keras.models import Model
from keras.layers.core import Dense
from keras.layers import GlobalAveragePooling2D
from keras.optimizers import Adam, SGD, RMSprop
from keras.preprocessing.image import ImageDataGenerator
from keras.utils.np_utils import to_categorical
from keras.utils import plot_model
from keras.models import model_from_json
from sklearn.metrics import confusion_matrix
import itertools
import matplotlib.pyplot as plt
import math
import copy
import pydotplus
train_path = 'data/train'
valid_path = 'data/validation'
test_path = 'data/test'
top_model_weights_path = 'model_weigh.h5'
# number of epochs to train top model
epochs = 100
# batch size used by flow_from_directory and predict_generator
batch_size = 2
img_width, img_height = 299, 299
fc_size = 1024
nb_iv3_layers_to_freeze = 172
train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input,
rotation_range=30,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
# this is the augmentation configuration we will use for testing:
# only rescaling
valid_datagen = ImageDataGenerator(preprocessing_function=preprocess_input,
rotation_range=30,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
train_batches =
train_datagen.flow_from_directory(train_path,
target_size=(img_width, img_height),
classes=None,
class_mode='categorical',
batch_size=batch_size,
shuffle=True)
valid_batches =
valid_datagen.flow_from_directory(valid_path,
target_size=(img_width,img_height),
classes=None,
class_mode='categorical',
batch_size=batch_size,
shuffle=True)
test_batches =
ImageDataGenerator().flow_from_directory(test_path,
target_size=(img_width,
img_height),
classes=None,
class_mode='categorical',
batch_size=batch_size,
shuffle=False)
nb_train_samples = len(train_batches.filenames)
# get the size of the training set
nb_classes_train = len(train_batches.class_indices)
# get the number of classes
predict_size_train = int(math.ceil(nb_train_samples / batch_size))
nb_valid_samples = len(valid_batches.filenames)
nb_classes_valid = len(valid_batches.class_indices)
predict_size_validation = int(math.ceil(nb_valid_samples / batch_size))
nb_test_samples = len(test_batches.filenames)
nb_classes_test = len(test_batches.class_indices)
predict_size_test = int(math.ceil(nb_test_samples / batch_size))
def add_new_last_layer(base_model, nb_classes):
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(fc_size, activation='relu')(x)
pred = Dense(nb_classes, activation='softmax')(x)
model = Model(input=base_model.input, output=pred)
return model
# freeze base_model layer in order to get the bottleneck feature
def setup_to_transfer_learn(model, base_model):
for layer in base_model.layers:
layer.trainable = False
model.compile(optimizer=Adam(lr=0.00001),
loss='categorical_crossentropy',
metrics=['accuracy'])
base_model = keras.applications.inception_v3.InceptionV3(weights='imagenet', include_top=False)
model = add_new_last_layer(base_model, nb_classes_train)
setup_to_transfer_learn(model, base_model)
model.summary()
train_labels = train_batches.classes
train_labels = to_categorical(train_labels, num_classes=nb_classes_train)
validation_labels = valid_batches.classes
validation_labels = to_categorical(validation_labels, num_classes=nb_classes_train)
history = model.fit_generator(train_batches,
epochs=epochs,
steps_per_epoch=nb_train_samples // batch_size,
validation_data=valid_batches,
validation_steps=nb_valid_samples // batch_size,
class_weight='auto')
# save model to json
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
# serialize model to HDF5
model.save_weights(top_model_weights_path)
print("Saved model to disk")
# model visualization
plot_model(model,
show_shapes=True,
show_layer_names=True,
to_file='model.png')
(eval_loss, eval_accuracy) = model.evaluate_generator(
valid_batches,
steps=nb_valid_samples // batch_size,
verbose=1)
print("[INFO] evaluate accuracy: {:.2f}%".format(eval_accuracy * 100))
print("[INFO] evaluate loss: {}".format(eval_loss))
test_batches.reset()
predictions = model.predict_generator(test_batches,
steps=nb_test_samples / batch_size,
verbose=0)
# print(predictions)
predicted_class_indices = np.argmax(predictions, axis=1)
# print(predicted_class_indices)
labels = train_batches.class_indices
labels = dict((v, k) for k, v in labels.items())
final_predictions = [labels[k] for k in predicted_class_indices]
# print(final_predictions)
# save as csv file
filenames = test_batches.filenames
results = pd.DataFrame({"Filename": filenames,
"Predictions": final_predictions})
results.to_csv("results.csv", index=False)
# evaluation test result
(test_loss, test_accuracy) = model.evaluate_generator(
test_batches,
steps=nb_train_samples // batch_size,
verbose=1)
print("[INFO] test accuracy: {:.2f}%".format(test_accuracy * 100))
print("[INFO] test loss: {}".format(test_loss))
Here is a brief summary of training process:
Epoch 1/100
2000/2000 [==============================] - 146s 73ms/step - loss: 0.4941 - acc: 0.7465 - val_loss: 0.1612 - val_acc: 0.9770
Epoch 2/100
2000/2000 [==============================] - 140s 70ms/step - loss: 0.4505 - acc: 0.7725 - val_loss: 0.1394 - val_acc: 0.9765
Epoch 3/100
2000/2000 [==============================] - 139s 70ms/step - loss: 0.4505 - acc: 0.7605 - val_loss: 0.1643 - val_acc: 0.9560
......
Epoch 98/100
2000/2000 [==============================] - 141s 71ms/step - loss: 0.1348 - acc: 0.9467 - val_loss: 0.0639 - val_acc: 0.9820
Epoch 99/100
2000/2000 [==============================] - 140s 70ms/step - loss: 0.1495 - acc: 0.9365 - val_loss: 0.0780 - val_acc: 0.9770
Epoch 100/100
2000/2000 [==============================] - 138s 69ms/step - loss: 0.1401 - acc: 0.9458 - val_loss: 0.0471 - val_acc: 0.9890
Here is the result that I get:
[INFO] evaluate accuracy: 98.55%
[INFO] evaluate loss: 0.05201659869024259
2000/2000 [==============================] - 47s 23ms/step
[INFO] test accuracy: 51.70%
[INFO] test loss: 7.737395915810134
I wish someone can help me deal with this problem.
As the code is now, you're not freezing the layers of the model for transfer learning. In the setup_to_transfer_learn you're freezing the layer in base_model, and then compiling the new model (containing layers from the base model), but not actually freezing on the new model. Just change setup_to_transfer_learn:
def setup_to_transfer_learn(model):
for layer in model.layers[:-3]: # since you added three new layers (which should not freeze)
layer.trainable = False
model.compile(optimizer=Adam(lr=0.00001),
loss='categorical_crossentropy',
metrics=['accuracy'])
Then call the function like this:
model = add_new_last_layer(base_model, nb_classes_train)
setup_to_transfer_learn(model)
You should see a large difference in the number of trainable parameters when calling model.summary()
Finally, I solved the problem. I forget to do image preprocessing to my test data. After I add this, everything works really fine.
I change this:
test_batches = ImageDataGenerator().flow_from_directory(test_path,
target_size=(img_width, img_height),
classes=None,
class_mode='categorical',
batch_size=batch_size,
shuffle=False)
to this:
test_datagen = ImageDataGenerator(preprocessing_function=preprocess_input)
test_batches = test_datagen.flow_from_directory(test_path,
target_size=(img_width, img_height),
classes=None,
class_mode='categorical',
batch_size=batch_size,
shuffle=False)
And the test accuracy is 0.98, test loss is 0.06.
What actually happens is that when you use preprocessing the model may actually start learning those techniques. One way to check if your model is learning good features is using Grad-CAM