I am using from tensorflow.keras.preprocessing.image import ImageDataGenerator
Here is the code :
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train, x_test = x_train.astype('float32') / 255, x_test.astype('float32') / 255
y_train, y_test = to_categorical(y_train), to_categorical(y_test)
img_gen = ImageDataGenerator(
rotation_range=10,
width_shift_range=.1,
height_shift_range=.1,
horizontal_flip=True,
vertical_flip=True,
fill_mode="nearest"
)
img_gen_train = img_gen.flow(x_train, y_train, batch_size=128, shuffle=True)
rand_tuner = RandomSearch(hypermodel=hypermodel,
objective='val_acc',
max_trials=max_trials,
project_name='cifar10')
then I am building a model using keras tuner HyperModel:
def hypermodel(hp):
units_choice = hp.Int('units', min_value=32, max_value=512, step=32, default=128)
lr_choice = hp.Float('learning_rate', 1e-5, 1e-2, sampling='LOG', default=1e-3)
dropout_rate_choice = hp.Float('rate', 0, .5, step=.1, default=.2)
filters_choice = hp.Choice('num_filters', values=[32, 64], default=64)
model = Sequential()
model.add(Conv2D(filters=16, kernel_size=3,
activation='relu', input_shape=input_shape))
model.add(Dropout(rate=dropout_rate_choice))
model.add(Flatten())
model.add(Dense(units=units_choice, activation='relu'))
model.add(Dense(num_labels, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer=Adam(lr_choice),
metrics=['accuracy'])
return model
After that, in an attempt to search best got an error :
How to deal with it?
Looks like Sparse_categorical_crossentropy wasn't the best choice for the generator and that was the whole problem.
Reference: https://github.com/keras-team/keras-tuner/issues/223
Related
I am trying to optimize hyperparameters for the IMBD dataset using keras hyperas but I am getting an error. I used this (https://www.kaggle.com/kt66nf/hyperparameter-optimization-using-keras-hyperas) code as a reference
CODE
def vectorize_sequences(sequences, dimension=10000):
# Create an all-zero matrix of shape (len(sequences), dimension)
results = np.zeros((len(sequences), dimension))
for i, sequence in enumerate(sequences):
results[i, sequence] = 1. # set specific indices of results[i] to 1s
return results
def data():
(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)
x_train = vectorize_sequences(train_data) #vectorized training data
x_test = vectorize_sequences(test_data)
y_train = np.asarray(train_labels).astype('float32') # vectorized labels
y_test = np.asarray(test_labels).astype('float32')
return x_train, y_train, x_test, y_test
def create_model(x_train, y_train, x_test, y_test):
model = models.Sequential()
model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))
model.add(layers.Dense(16, activation='relu'))
model.add(Dropout({{uniform(0.5, 1)}}))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
model.add(Dropout({{uniform(0.5, 1)}}))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', metrics=['accuracy'],
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})
model.fit(x_train, y_train,
batch_size={{choice([16, 32, 64])}},
epochs={{choice([25, 50, 75, 100])}},
validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
***then I loaded google drive and set the path to the notebook
best_run, best_model = optim.minimize(model= create_model,
data=data,
max_evals=15,
algo=tpe.suggest,
notebook_name='Copy of imbd', #name of the notebook
trials= Trials())
this last code is where I got the error
"File "", line 182
]
^
SyntaxError: invalid syntax"
I am trying to Save the entire model after each epoch using ModelCheckpoint callback.
After training if i am trying to load the saved model and evaluate, the model weights are not loaded. Why is this load_model not working for loading the model weights?
import tensorflow as tf
print(tf.__version__)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.models import load_model
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()
x_train = x_train / 255.0
x_test = x_test / 255.0
# Use smaller subset -- speeds things up
x_train = x_train[:10000]
y_train = y_train[:10000]
x_test = x_test[:1000]
y_test = y_test[:1000]
def get_test_accuracy(model, x_test, y_test):
test_loss, test_acc = model.evaluate(x=x_test, y=y_test, verbose=0)
print('accuracy: {acc:0.3f}'.format(acc=test_acc))
def get_new_model():
model = Sequential([
Conv2D(filters=16, input_shape=(32, 32, 3), kernel_size=(3, 3),
activation='relu', name='conv_1'),
Conv2D(filters=8, kernel_size=(3, 3), activation='relu', name='conv_2'),
MaxPooling2D(pool_size=(4, 4), name='pool_1'),
Flatten(name='flatten'),
Dense(units=32, activation='relu', name='dense_1'),
Dense(units=10, activation='softmax', name='dense_2')
])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model
# Create Tensorflow checkpoint object
checkpoint_path = "model_checkpoints"
checkpoint = ModelCheckpoint(filepath=checkpoint_path,
save_weights_only=False,
save_freq="epoch",
verbose=1)
# Create and fit model with checkpoint
model = get_new_model()
model.fit(x_train,
y_train,
epochs=3,
callbacks=[checkpoint])
# Get the model's test accuracy
get_test_accuracy(model,x_test,y_test)
# Reload model from scratch
model = load_model(checkpoint_path)
get_test_accuracy(model,x_test,y_test)
The accuracy after loading the saved model load_model is not same as the accuracy for trained model.
I am new in Keras. I want to implement a layer where not all the weights will update. For example, in the following code, I want the dilation layer will update in a way that some center weights are never updated. For say, the shape of each feature matrix (out of 1024) in the dilation layer is 448, 448 and a block of 8x8 at the center of all feature matrices will never be updated, i.e. the 8x8 block is a (non-trainable) mask to the feature matrices.
input_layer=Input(shape=(896,896,3))
new_layer = Conv2D(32, kernel_size=(3,3), padding="same", activation='relu', kernel_initializer='he_normal')(input_layer)
new_layer = MaxPooling2D(pool_size=(2, 2), strides=(2,2), padding='same', data_format=None)(new_layer)
new_layer = Conv2D(64, kernel_size=(3,3), padding='same', activation='relu', kernel_initializer='he_normal')(new_layer)
new_layer = Conv2D(1024, kernel_size=(7,7), dilation_rate=8, padding="same", activation='relu', kernel_initializer='he_normal', name='dialation')(new_layer)
new_layer = Conv2D(32, kernel_size=(1,1), padding="same", activation='relu', kernel_initializer='he_normal')(new_layer)
new_layer = Conv2D(32, kernel_size=(1,1), padding="same", activation='relu', kernel_initializer='he_normal')(new_layer)
model = Model(input_layer, new_layer)
I was trying with the Keras's custom layer [link], but it was difficult for me to understand. Anyone would please help.
UPDATE:
I added the following figure for a better understanding. The dilation layer contains 1024 features. I want the middle region of each feature to be non-trainable (static).
Use this mask for both cases:
mask = np.zeros((1,448,448,1))
mask[:,220:228,220:228] = 1
Replacing part of the feature
If you replace part of the feature with constant values, this means the feature will be static, but it will still participate in backpropagation (because weights will still be multiplied and summed for this part of the image and there is a connection)
constant = 0 (will annulate kernel, but not bias)
def replace(x):
return x*(1-mask) + constant*mask
#before the dilation layer
new_layer=Lambda(replace)(new_layer)
Keeping the feature value, but stopping backpropagation
Here, the weights of the dilation layer and further will be updated normally, but the weights before the dilation layer will not receive the influence of the central region.
def stopBackprop(x):
stopped=K.stop_gradients(x)
return x*(1-mask) + stopped*mask
#before the dilation layer
new_layer=Lambda(stopBackprop)(new_layer)
I am working on clustering weights and then freezing specific clusters and train the network.
I am trying to freeze the specific weights in this network using the above example. But I'm not sure how to set the shape of the mask and custom layer in run_certain_weights().
This is the code I'm using:
from keras.layers import Dense, Flatten, Lambda
from keras.utils import to_categorical
from keras.models import Sequential, load_model
from keras.datasets import mnist
from keras.losses import categorical_crossentropy
from keras.backend import stop_gradient
import numpy as np
def load_data():
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
y_train = to_categorical(y_train, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)
return x_train, y_train, x_test, y_test
def run():
x_train, y_train, x_test, y_test = load_data()
model=Sequential(Flatten(input_shape=(28, 28)))
layer = Dense(300, name='dense1', activation='relu')
layer.trainable=True
model.add(layer)
layer2 = Dense(100, name='dense2', activation='relu')
layer2.trainable=False
model.add(layer2)
layer3 = Dense(10, name='dense3', activation='softmax')
model.add(layer3)
model.compile(loss=categorical_crossentropy, optimizer='Adam',metrics ['accuracy'])
print(model.summary())
print("x_train.shape():",x_train.shape)
print("y_train.shape()",y_train.shape)
model.fit(x_train, y_train, epochs=5, verbose=2)
print(model.evaluate(x_test, y_test))
return model
def stopBackprop(x):
stopped=stop_gradient(x)
return x*(1-mask) + stopped*mask
def run_certain_weights():
x_train, y_train, x_test, y_test = load_data()
model=Sequential(Flatten(input_shape=(28, 28)))
mask = np.zeros((300,))
print(mask.shape)
mask[220:228,] = 1
layer = Dense(300, name='dense1', activation='relu')
layer.trainable=False
model.add(layer)
#before the dense2 layer
new_layer=Lambda(stopBackprop)(layer)
model.add(new_layer)
layer2 = Dense(300, name='dense2', activation='relu')
layer2.trainable=True
model.add(layer2)
layer3 = Dense(10, name='dense3', activation='softmax')
model.add(layer3)
model.compile(loss=categorical_crossentropy, optimizer='Adam',metrics = ['accuracy'])
print(model.summary())
print("x_train.shape():",x_train.shape)
print("y_train.shape()",y_train.shape)
model.fit(x_train, y_train, epochs=5, verbose=2)
print(model.evaluate(x_test, y_test))
return model
def freeze(model):
x_train, y_train, x_test, y_test = load_data()
name = 'dense2'
weightsAndBias = model.get_layer(name=name).get_weights()
# freeze the weights of this layer
model.get_layer(name=name).trainable = False
# record the weights before retrain
weights_before = weightsAndBias[0]
# retrain
print("x_train.shape():",x_train.shape)
print("y_train.shape()",y_train.shape)
model.fit(x_train, y_train, verbose=2, epochs=1)
weights_after = model.get_layer(name=name).get_weights()[0]
if (weights_before == weights_after).all():
print('the weights did not change!!!')
else:
print('the weights changed!!!!')
if __name__ == '__main__':
model = run()
freeze(model)
model = run_certain_weights()
freeze(model)
I am running MNIST prediction using Keras, with tensorflow backend.
I have code that runs with batches , using Keras fit() as
(X_train, y_train), (X_test, y_test) = mnist.load_data()
N1 = X_train.shape[0]
N2 = X_test.shape[0]
h = X_train.shape[1]
w = X_train.shape[2]
num_pixels = h*w
# reshape N1 samples to num_pixels
x_train = X_train.reshape(N1, num_pixels).astype('float32') # shape is now (60000,784)
x_test = X_test.reshape(N2, num_pixels).astype('float32') # shape is now (10000,784)
x_train = x_train / 255
x_test = x_test / 255
y_train = np_utils.to_categorical(y_train) #(60000,10)
y_test = np_utils.to_categorical(y_test) # (10000,10):
num_classes = y_test.shape[1]
def baseline_model():
# create model
model = Sequential()
model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))
model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
model = baseline_model()
batch_size = 200
epochs = 20
max_batches = 2 * len(x_train) / batch_size # 2*60000/200
# reshape to be [samples][width][height][ channel] for ImageDataGenerator
x_t = X_train.reshape(N1, w, h, 1).astype('float32')
datagen = ImageDataGenerator(rescale= 1./255)
train_gen = datagen.flow(x_t, y_train, batch_size=batch_size)
for e in range(epochs):
batches = 0
for x_batch, y_batch in train_gen:
# x_batch is of size [batch_sz,w,h,ch]: resize to [bth_sz,pixel_sz]: (200,28,28,1)-> (200,784)
# for model.fit
x_batch = np.reshape(x_batch, [-1, num_pixels])
model.fit(x_batch, y_batch,validation_split=0.15,verbose=0)
batches += 1
print("Epoch %d/%d, Batch %d/%d" % (e+1, epochs, batches, max_batches))
if batches >= max_batches:
break
scores = model.evaluate(x_test, y_test, verbose=0)
However, when I try to implement similar code using fit_generator(), I get an error.
the code is as below:
(X_train, y_train), (X_test, y_test) = mnist.load_data()
# separate data into train and validation
from sklearn.model_selection import train_test_split
# Split the data
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.15, shuffle= True)
# number of training samples
N1 = X_train.shape[0] # training size
N2 = X_test.shape[0] # test size
N3 = X_valid.shape[0] # valid size
h = X_train.shape[1]
w = X_train.shape[2]
num_pixels = h*w
y_train = np_utils.to_categorical(y_train)
y_valid = np_utils.to_categorical(y_valid)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]
def baseline_model():
# create model
model = Sequential()
model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))
model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
model = baseline_model()
batch_size = 200
epochs = 20
steps_per_epoch_tr = int(N1/ batch_size) # 51000/200
steps_per_epoch_val = int(N3/batch_size)
# reshape to be [samples][width][height][ channel] for ImageData Gnerator->datagen.flow
x_t = X_train.reshape(N1, w, h, 1).astype('float32')
x_v = X_valid.reshape(N3, w, h, 1).astype('float32')
# define data preparation
datagen = ImageDataGenerator(rescale=1./255) # scales x_t/x_v
train_gen = datagen.flow(x_t, y_train, batch_size=batch_size)
valid_gen = datagen.flow(x_v,y_valid, batch_size=batch_size)
model.fit_generator(train_gen,steps_per_epoch = steps_per_epoch_tr,validation_data = valid_gen,
validation_steps = steps_per_epoch_val,epochs=epochs)
This gives an error:
This is due to expected image dimension error, but I am not sure where/how to fix this. any help is greatly appreciated.
Thanks
sedy
In the model.fit() case, this line flattened the input before feeding it for training.
x_batch = np.reshape(x_batch, [-1, num_pixels])
But in the generator case, there is nothing to flatten the input before feeding it to the Dense layer. The Dense layer cannot process 2D input (28 x 28). Adding, a Flatten() layer to the model should do the trick as shown below.
def baseline_model():
# create model
model = Sequential()
model.add(Flatten(input_shape=(28,28,1)))
model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))
model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
Am aware that container for autoencoder has been removed in new Keras. My aim is to extract the encoding representation of an input and feed it in as an input to the next layer i.e. stacked autoencoder for classification using three hidden layers. I got this error:
Exception: Error when checking model target: expected dense_160 to have shape (None, 500) but got array with shape (60000L, 784L). I am not too sure if this code achieves my intention. I will appreciate any guide on how to resolve this. Thanks.
The code is pasted below:
from keras.layers import Input, Dense
from keras.models import Model
from keras.datasets import mnist
import numpy as np
nb_classes = 10
nb_epoch=200
batch_size=256
hidden_layer1=784
hidden_layer2=600
hidden_layer3=500
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))
print('Train samples: {}'.format(x_train.shape[0]))
print('Test samples: {}'.format(x_test.shape[0]))
from keras.utils import np_utils
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)
input_img = Input(shape=(784,))
encoded = Dense(hidden_layer1, activation='relu')(input_img)
encoded = Dense(hidden_layer2, activation='relu')(encoded)
encoded=Dense(hidden_layer3,activation='softmax')(encoded)
model = Model(input=input_img , output=encoded)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(x_train, x_train,
nb_epoch=nb_epoch,
batch_size=batch_size,
shuffle=True,
validation_data=(x_test, x_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test score before fine turning:', score[0])
print('Test accuracy after fine turning:', score[1])
I modified your code and tested, it worked.
nb_classes = 10
nb_epoch = 5
batch_size = 256
hidden_layer1 = 128
hidden_layer2 = 64
hidden_layer3 = 10 # because you have 10 categories
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))
print('Train samples: {}'.format(x_train.shape[0]))
print('Test samples: {}'.format(x_test.shape[0]))
from keras.utils import np_utils
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)
input_img = Input(shape=(784,))
encoded = Dense(hidden_layer1, activation='relu')(input_img)
encoded = Dense(hidden_layer2, activation='relu')(encoded)
encoded = Dense(hidden_layer3, activation='softmax')(encoded)
decoded = Dense(hidden_layer2, activation='relu')(encoded)
decoded = Dense(hidden_layer1, activation='relu')(decoded)
decoded = Dense(784, activation='sigmoid')(decoded)
model = Model(input=input_img, output=encoded)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(x_train, y_train,
nb_epoch=nb_epoch,
batch_size=batch_size,
shuffle=True,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=1)
print('/n')
print('Test score before fine turning:', score[0])
print('Test accuracy after fine turning:', score[1])