I have used below data augmentation for memory saving.
total number of test images = 400
batch size = 128
when i check accuracy of testset using model.evaluate_generator, it is different with final validation_accuracy from last epoch of my model.
Furthermore, output of model.evaluate_generator is changed when i repeat this.
below is my code.
please help!
train_datagen = ImageDataGenerator(
rescale=1./255,)
test_datagen = ImageDataGenerator(rescale=1./255)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
color_mode= "grayscale",
target_size=(img_width, img_height),
batch_size=128,
class_mode='categorical',)
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
color_mode= "grayscale",
target_size=(img_width, img_height),
batch_size=128,
class_mode='categorical')
#%%
hist = model.fit_generator(
train_generator,
samples_per_epoch=nb_train_samples,
nb_epoch=nb_epoch,
validation_data=validation_generator,
nb_val_samples=nb_validation_samples)
scoreSeg = model.evaluate_generator(validation_generator, 400)
print("Accuracy = ",scoreSeg[1])
Related
I am trying to overfit my model on a single batch to check model integrity. I am using Keras and TensorFlow for the implementation of my model and coding style for this project.
I know how to get the single batch and overfit the model in PyTorch but don't have an idea in Keras.
to get a single batch in PyTorch I used:
images, labels = next(iter(train_dataset))
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr = 0.0001)
for epoch in range(epochs):
print(f"Epoch [{epoch}/{epochs}]")
# for batch_idx, (data, target) in enumerate(train_loader):
data, target = data.to(device), target.to(device)
data = data.reshape(data.shape[0], -1)
# forward
score = model(data)
loss = criterion(score, target)
print(f"Loss: {loss.item()}")
# backward
optimizer.zero_grad()
loss.backward()
optimizer.step()
How to do it in keras any helping matrial?
Thank you everyone for coming here. I found a solution and here it is:
datagen = ImageDataGenerator(rescale=1 / 255.0,
rotation_range=20,
zoom_range=0.2,
width_shift_range=0.05,
height_shift_range=0.05,
shear_range=0.2,
horizontal_flip=True,
fill_mode="nearest"
)
# preprocessing_function=preprocess_input,
# Declare an image generator for validation & testing without generation
test_datagen = ImageDataGenerator(rescale = 1./255,)#preprocessing_function=preprocess_input
# Declare generators for training, validation, and testing from DataFrames
train_gen = datagen.flow_from_directory(directory_train,
target_size=(512, 512),
color_mode='rgb',
batch_size=BATCH_SIZE,
class_mode='binary',
shuffle=True)
val_gen = test_datagen.flow_from_directory(directory_val,
target_size=(512, 512),
color_mode='rgb',
batch_size=BATCH_SIZE,
class_mode='binary',
shuffle=False)
test_gen = test_datagen.flow_from_directory(directory_test,
target_size=(512, 512),
color_mode='rgb',
batch_size=BATCH_SIZE,
class_mode='binary',
shuffle=False)
train_images, train_labels = next(iter(train_gen))
val_images, val_labels = next(iter(val_gen))
test_images, test_labels = next(iter(val_gen))
#check shape for selected Batch
print("Length of Train images : {}".format(len(train_images)))
print("shape of Train images : {}".format(train_images.shape))
print("shape of Train labels : {}".format(train_labels.shape))
Length of Train images : 32
shape of Train images : (32, 512, 512, 3)
shape of Train labels : (32,)
history = model.fit(train_images, train_labels,
use_multiprocessing=True,
workers=16,
epochs=100,
class_weight=class_weights,
validation_data=(val_images, val_labels),
shuffle=True,
callbacks=call_backs)
I'm using keras to build an image classification model and all of my data is in one directory with subfolders for each class:
data/
----class1/
----class2/
----class3/
----class4/
I'm wondering if there is a way to split my data into a train, test AND validation set in keras using ImageDataGenerator.
Yes, there is a way you can do that.
train_datagen = ImageDataGenerator(rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
validation_split=0.2) # set validation split
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='binary',
subset='training') # set as training data
validation_generator = train_datagen.flow_from_directory(
train_data_dir, # same directory as training data
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='binary',
subset='validation') # set as validation data
model.fit_generator(
train_generator,
steps_per_epoch = train_generator.samples // batch_size,
validation_data = validation_generator,
validation_steps = validation_generator.samples // batch_size,
epochs = nb_epochs)
https://keras.io/preprocessing/image/
I try to apply k-fold cross validation to the cnn classification problem
let say I have a carA, carB
so I made the subfolder
car/trainCross/fold0 car/trainCross/fold1
car/validCross/fold0 car/validCross/fold1
and following code
model_path = '../carPrediction/model/'+ 'saved.hdf5'
for i in range(2):
print('training->',i,' split')
train_generator = train_datagen.flow_from_directory(TRAIN_CROPPED_PATH +'fold'+str(i),
target_size=(image_size, image_size),
batch_size=batch_size,
class_mode='categorical',
seed=2019,
color_mode='rgb')
print(VALID_CROPPED_PATH+'fold'+str(i))
validation_generator = valid_datagen.flow_from_directory(
VALID_CROPPED_PATH+'fold'+str(i),
target_size=(image_size,image_size),
batch_size=batch_size,
class_mode='categorical',
seed=2019,
color_mode='rgb'
)
test_generator = test_datagen.flow_from_dataframe(
dataframe=df_test,
directory=TEST_CROPPED_PATH,
x_col='img_file',
y_col=None,
target_size= (image_size,image_size),
color_mode='rgb',
class_mode=None,
batch_size=batch_size,
shuffle=False
)
try:
model = load_model(model_path, compile=True)
except Exception as OSError:
pass
patient = 2
callbacks1 = [
EarlyStopping(monitor='val_loss', patience=patient, mode='min', verbose=1),
ReduceLROnPlateau(monitor = 'val_loss', factor = 0.5, patience = patient / 2, min_lr=0.00001, verbose=1, mode='min'),
ModelCheckpoint(filepath=model_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min'),
]
history = model.fit_generator(
train_generator,
steps_per_epoch=get_steps(nb_train_sample, batch_size),
epochs=2,
validation_data=validation_generator,
validation_steps=get_steps(nb_validation_sample, batch_size),
verbose=1,
callbacks = callbacks1
)
but not sure in this way is correct
any thought?
I'm trying to implement a model to classify(binary) my picture,but I got this problem,could anyone help me to solve it.Thank you guys~
here is the model, and the training process
def googLeNet(input = Input(shape=(224, 224, 3))):
.................
.................
...............
averagepool1_7x7_s1 = AveragePooling2D(pool_size=(7, 7), padding='same')(inception_5b)
drop1 = Dropout(rate=0.4)(averagepool1_7x7_s1)
linear = Dense(units=1, activation='linear')(drop1)
last = Dense(units=1, activation='softmax')(linear)
model = Model(input=input, outputs=last)
return model
-----------------------------------------------------------------------
train_datagen = ImageDataGenerator(rescale=1./255)
test_datagen = ImageDataGenerator(rescale=1./255)
train_generator = train_datagen.flow_from_directory(
train_dir,
target_size=(224, 224),
batch_size=20,
class_mode='binary'
)
validation_generator = test_datagen.flow_from_directory(
test_dir,
target_size=(224, 224),
batch_size=20,
class_mode='binary'
)
model = googLeNet()
model.summary()
model.compile(loss='binary_crossentropy',
optimizer=optimizers.RMSprop(lr=1e-4),
metrics=['acc']
)
history = model.fit_generator(train_generator,
steps_per_epoch=100,
epochs=30,
validation_data=validation_generator,
validation_steps=50
)
Hello everyone I started training a network ana it got stuck, it did not finish the first epoch.
Here is the code I used:
top_model_weights_path = '/data/fc_model.h5'
img_width, img_height = 150, 150
-train_data_dir = '/data/train'
validation_data_dir = '/data/validation'
nb_train_samples = 2000
nb_validation_samples = 800
epochs = 50
batch_size = 16
model = applications.VGG16(weights='imagenet', include_top=False, input_shape=(150, 150, 3))
print('Model loaded.')
top_model = Sequential()
top_model.add(Flatten(input_shape=model.output_shape[1:]))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.5))
top_model.add(Dense(1, activation='sigmoid'))
top_model.load_weights(top_model_weights_path)
model = Model(inputs= model.input, outputs= top_model(model.output))
for layer in model.layers[:25]:
layer.trainable = False
model.compile(loss='binary_crossentropy',
optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy'])
train_datagen = ImageDataGenerator(
rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='binary')
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='binary')
model.fit_generator(
train_generator,
samples_per_epoch=nb_train_samples,
epochs=epochs,
validation_data=validation_generator,
nb_val_samples=nb_validation_samples)
I am using Transfer Learning. I followed this tutorial online :
Tutorial
Please help thank you.