I am wondering if it is an issue (possible data leakage?) when implementing leave one out cross validation by hand if the model is fit each iteration after testing on each fold? It seems like if the model is trained on all data except for "X" and after testing on "X" the model is trained on all data other than "Y" and tested on "Y" it has seen "Y" on the first iteration. Is this actually a problem, and does my implementation of LOOCV by hand appear to be correct?
Thanks for your time!
i = 0
j = 0
for i in range(0, 41):
X_copy = X_orig[(i):(i+1)] #Slice the ith element from the numpy array
y_copy = y_orig[(i):(i+1)]
X_model = X_orig
y_model = y_orig
X_model = np.delete(X_model, i, axis = 0)
y_model = np.delete(y_model, i, axis = 0)
model.fit(X_model, y_model, epochs=115, batch_size=28, verbose = 0) #verbose = 0 removes learning info
prediction = model.predict(X_copy)
prediction[prediction>=0.5] = 1
prediction[prediction<0.5] = 0
print(prediction, y_copy)
if np.array_equal(y_copy, prediction):
j = j + 1
#print(y_copy, prediction)
if np.not_equal:
#print(y_copy, prediction)
pass
print(j/41) #For 41 samples in dataset
Why don't you use this?
from sklearn.model_selection import LeaveOneOut
loo = LeaveOneOut()
model =...
test_fold_predictions = []
for train_index, test_index in loo.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model.fit(X_train, y_train)
test_fold_predictions.append(model.predict(X_test))
EDIT
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.optimizers import SGD
model = Sequential()
model.add(Dense(5000, activation='relu', input_dim=X_train.shape[1]))
model.add(Dropout(0.1))
model.add(Dense(600, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(y_train.shape[1], activation='sigmoid'))
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='binary_crossentropy',
optimizer=sgd)
from sklearn.model_selection import LeaveOneOut
loo = LeaveOneOut()
test_fold_predictions = []
for train_index, test_index in loo.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model.fit(X_train, y_train, epochs=5, batch_size=2000)
test_fold_predictions.append(model.predict(X_test))
Related
I have a question about how the cross_val_score() from the Scikit-Learn works. I tried divide the dataset in 10 folds with Kfold() and compute the log loss of both training and validation sets for each fold. However I got different answers using the cross_validation_score, setting the parameter scoring = 'neg_log_loss'.
X and y are arrays of shape (1800, 12) and (1800, 1), respectively.
kfold = KFold(n_splits=10)
train_loss = []
val_loss = []
for train_index, val_index in kfold.split(X, y):
clf_logreg = LogisticRegression()
#
X_train, X_val = X[train_index], X[val_index]
y_train, y_val = y[train_index], y[val_index]
clf_logreg.fit(X_train, y_train)
y_train_pred = clf_logreg.predict(X_train)
y_val_pred = clf_logreg.predict(X_val)
train_loss.append(log_loss(y_train, y_train_pred))
val_loss.append(log_loss(y_val, y_val_pred))
clf_logreg.fit(X,y)
y_error = cross_val_score(clf_logreg, X, y, cv=kfold, scoring='neg_log_loss')
print("cross_val log_loss: ", -y_error)
print("\ntrain_loss: ", train_loss)
print("\nval_loss: ", val_loss)
The answers I got:
cross_val log_loss: [0.18546779 0.18002459 0.21591202 0.15872213 0.22852112 0.18766844
0.28641203 0.14923009 0.21446935 0.20373971]
train_loss: [2.79298449379999, 2.7290223160363962, 2.558457002245472, 2.835624958485065, 2.5797806896386337, 2.622420660745048, 2.5797797024813125, 2.6224201671663874, 2.5797782217453302, 2.6863818513513213]
val_loss: [1.9188431218680995, 2.1107385395747733, 3.645826363693089, 2.110734097366828, 3.2620355282797417, 2.686367043991502, 3.453913177154633, 2.4944849529086657, 2.8782624616981765, 2.4944938373245567]
As Ben Reiniger noted in the comment log_loss expects probabilities in y_train_pred, y_val_pred. So you need to change
clf_logreg.predict
to:
clf_logreg.predict_proba
Example:
from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import log_loss
from sklearn.model_selection import KFold, cross_val_score
X, y = load_iris(return_X_y=True)
y = y == 1
kfold = KFold(n_splits=10, random_state=1, shuffle=True)
train_loss = []
val_loss = []
for train_index, val_index in kfold.split(X, y):
clf_logreg = LogisticRegression()
#
X_train, X_val = X[train_index], X[val_index]
y_train, y_val = y[train_index], y[val_index]
clf_logreg.fit(X_train, y_train)
y_train_pred = clf_logreg.predict_proba(X_train)
y_val_pred = clf_logreg.predict_proba(X_val)
train_loss.append(log_loss(y_train, y_train_pred))
val_loss.append(log_loss(y_val, y_val_pred))
clf_logreg.fit(X, y)
y_error = cross_val_score(clf_logreg, X, y, cv=kfold, scoring="neg_log_loss")
print("cross_val log_loss: ", -y_error)
print("\nval_loss: ", val_loss)
Results:
cross_val log_loss: [0.53548503 0.54200945 0.60324094 0.64781483 0.43323992 0.37625601
0.55101127 0.46172226 0.50216316 0.64359642]
val_loss: [0.5354850268015129, 0.5420094471965571, 0.6032409439788419, 0.647814828089315, 0.43323991804482626, 0.3762560144867495, 0.5510112741331039, 0.46172225526408, 0.5021631570133954, 0.6435964210060579]
I'm trying to train an autoencoder but have problems in reshaping my X_train to fit it to my model model().
from tensorflow import keras
from keras.layers import *
from keras.models import Model
from keras.models import Sequential
from keras.optimizers import Adam
from keras.optimizers import RMSprop
from keras.utils import plot_model
X_train = np.array(X_train, dtype=np.float)
X_test =np.array(X_train, dtype=np.float)
X_train = X_train.reshape(len(X_train), 100,1)
X_test = X_test.reshape(len(X_test), 100,1)
#inputs = Input(shape=(230, 1,100))
epoch = 100
batch = 128
def model():
m = Sequential()
# ##m.add(Reshape((,)))
m.add(Flatten())
m.add(Dense(512, activation='relu'))
m.add(Dense(128, activation = 'relu'))
m.add(Dense(2, activation = 'linear'))
m.add(Dense(128, activation = 'relu'))
m.add(Dense(512, activation = 'relu'))
m.add(Dense(784, activation = 'sigmoid'))
m.compile(loss='mean_squared_error', optimizer = 'rmsprop', metrics = ['accuracy'])
# Fit data to model m
m.fit(X_train, X_train, batch_size = batch, epochs = epoch)
m.summary()
#score = m.evaluate(X_test, Y_test, verbose = 0)
#print('Test loss:' score[0])
#print('Test accuracy:', score[1])
#m.summary()
mod = model()
The of dimension of my data is the following:
X_train = (523, 100,1)
X_test = (523, 100,1)
To fix your issue, change the following:
X_train = X_train.reshape((-1, 100))
X_test = X_test.reshape((-1, 100))
Delete the Flatten layer and use 100 neurons for the last layer as stated in the comments.
I have a dataset with two columns, path and class. I'd like to fine tune VGGface with it.
dataset.head(5):
path class
0 /f3_224x224.jpg red
1 /bc_224x224.jpg orange
2 /1c_224x224.jpg brown
3 /4b_224x224.jpg red
4 /0c_224x224.jpg yellow
I'd like to use these paths to preprocess images and feed to keras. My preprocessing functions is below:
from keras.preprocessing.image import img_to_array, load_img
def prep_image(photo):
img = image.load_img(path + photo, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = utils.preprocess_input(x, version=1)
return x
I prepare my datasets with the following code:
from sklearn.model_selection import train_test_split
path = list(dataset.columns.values)
path.remove('class')
X = dataset[path]
y = dataset['class']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
I train my model with the following code:
nb_class = 4
hidden_dim = 512
vgg_model = VGGFace(include_top=False, input_shape=(224, 224, 3))
last_layer = vgg_model.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
x = Dense(hidden_dim, activation='relu', name='fc6')(x)
x = Dense(hidden_dim, activation='relu', name='fc7')(x)
out = Dense(nb_class, activation='softmax', name='fc8')(x)
custom_vgg_model = Model(vgg_model.input, out)
custom_vgg_model.compile(
optimizer="adam",
loss="categorical_crossentropy"
)
custom_vgg_model.fit(X_train, y_train, epochs=50, batch_size=16)
test_loss, test_acc = model.evaluate(X_test, y_test)
However i get value error because i can't figure out how to preprocess images and feed the arrays. How can i transform the paths from X_train/test dataframes and replace them with the output of prep_image function?
ValueError: Error when checking input: expected input_2 to have 4 dimensions, but got array with shape (50297, 1)
So the shape should be (50297, 224, 224, 3).
X_train, X_test are basically just path names it seems. In your data preparation step you just need to modify your code like that adding those last two lines.
from sklearn.model_selection import train_test_split
path = list(dataset.columns.values)
path.remove('class')
X = dataset[path]
y = dataset['class']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
X_train = np.array([prep_image(path)[0] for path in X_train])
X_test = np.array([prep_image(path)[0] for path in X_test])
I am running MNIST prediction using Keras, with tensorflow backend.
I have code that runs with batches , using Keras fit() as
(X_train, y_train), (X_test, y_test) = mnist.load_data()
N1 = X_train.shape[0]
N2 = X_test.shape[0]
h = X_train.shape[1]
w = X_train.shape[2]
num_pixels = h*w
# reshape N1 samples to num_pixels
x_train = X_train.reshape(N1, num_pixels).astype('float32') # shape is now (60000,784)
x_test = X_test.reshape(N2, num_pixels).astype('float32') # shape is now (10000,784)
x_train = x_train / 255
x_test = x_test / 255
y_train = np_utils.to_categorical(y_train) #(60000,10)
y_test = np_utils.to_categorical(y_test) # (10000,10):
num_classes = y_test.shape[1]
def baseline_model():
# create model
model = Sequential()
model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))
model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
model = baseline_model()
batch_size = 200
epochs = 20
max_batches = 2 * len(x_train) / batch_size # 2*60000/200
# reshape to be [samples][width][height][ channel] for ImageDataGenerator
x_t = X_train.reshape(N1, w, h, 1).astype('float32')
datagen = ImageDataGenerator(rescale= 1./255)
train_gen = datagen.flow(x_t, y_train, batch_size=batch_size)
for e in range(epochs):
batches = 0
for x_batch, y_batch in train_gen:
# x_batch is of size [batch_sz,w,h,ch]: resize to [bth_sz,pixel_sz]: (200,28,28,1)-> (200,784)
# for model.fit
x_batch = np.reshape(x_batch, [-1, num_pixels])
model.fit(x_batch, y_batch,validation_split=0.15,verbose=0)
batches += 1
print("Epoch %d/%d, Batch %d/%d" % (e+1, epochs, batches, max_batches))
if batches >= max_batches:
break
scores = model.evaluate(x_test, y_test, verbose=0)
However, when I try to implement similar code using fit_generator(), I get an error.
the code is as below:
(X_train, y_train), (X_test, y_test) = mnist.load_data()
# separate data into train and validation
from sklearn.model_selection import train_test_split
# Split the data
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.15, shuffle= True)
# number of training samples
N1 = X_train.shape[0] # training size
N2 = X_test.shape[0] # test size
N3 = X_valid.shape[0] # valid size
h = X_train.shape[1]
w = X_train.shape[2]
num_pixels = h*w
y_train = np_utils.to_categorical(y_train)
y_valid = np_utils.to_categorical(y_valid)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]
def baseline_model():
# create model
model = Sequential()
model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))
model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
model = baseline_model()
batch_size = 200
epochs = 20
steps_per_epoch_tr = int(N1/ batch_size) # 51000/200
steps_per_epoch_val = int(N3/batch_size)
# reshape to be [samples][width][height][ channel] for ImageData Gnerator->datagen.flow
x_t = X_train.reshape(N1, w, h, 1).astype('float32')
x_v = X_valid.reshape(N3, w, h, 1).astype('float32')
# define data preparation
datagen = ImageDataGenerator(rescale=1./255) # scales x_t/x_v
train_gen = datagen.flow(x_t, y_train, batch_size=batch_size)
valid_gen = datagen.flow(x_v,y_valid, batch_size=batch_size)
model.fit_generator(train_gen,steps_per_epoch = steps_per_epoch_tr,validation_data = valid_gen,
validation_steps = steps_per_epoch_val,epochs=epochs)
This gives an error:
This is due to expected image dimension error, but I am not sure where/how to fix this. any help is greatly appreciated.
Thanks
sedy
In the model.fit() case, this line flattened the input before feeding it for training.
x_batch = np.reshape(x_batch, [-1, num_pixels])
But in the generator case, there is nothing to flatten the input before feeding it to the Dense layer. The Dense layer cannot process 2D input (28 x 28). Adding, a Flatten() layer to the model should do the trick as shown below.
def baseline_model():
# create model
model = Sequential()
model.add(Flatten(input_shape=(28,28,1)))
model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))
model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
I'm working on a human activity recognition problem using depth stream.
Each sample is a Matlab file of size (20,3,no_of_frames), i.e., there are 20 rows and 3 cols in every frame and the number of frames can vary for different samples.
I have padded all the samples with 0's so that all of them contains same no of frames (say 100).
So now all samples are of size (20,3,100).
Also let the total no of samples are 400 and no of classes be 10.
How can I arrange my dataset so as to use LSTM in Keras. Would you also suggest some basic LSTM model for the classification purpose?
Here is my code to create dataset:
clc;
clear all;
ctr=0;
longest=326; % length of longest sequence
gyroData = zeros(431,longest*3);
gyroLabel=zeros(431,1);
for a=1:27
for s=1:2:7
for t=1:4
fname = strcat('a', int2str(a), '_s', int2str(s), '_t', int2str(t),'_inertial.mat');
if exist(fname)
load(fname);
d_iner = d_iner(:,1:3);
r = size(d_iner,1);
d_iner = cat(1,d_iner,zeros(longest-r,3)); % do zero padding to longest sequence
ctr = ctr+1;
d_iner = d_iner';
gyroData(ctr,:,:) = d_iner(:);
gyroLabel(ctr,1)=a-1;
end
end
end
end
n1 = randperm(ctr);
for i=1:ctr
if i==1
x1=gyroData(n1(i),:);
y1=gyroLabel(n1(i),1);
else
x1=cat(1,x1,gyroData(n1(i),:));
y1=cat(1,y1,gyroLabel(n1(i),1));
end
end
%%
ctr=0;
gyroData = zeros(430,longest*3);
gyroLabel=zeros(430,1);
for a=1:27
for s=2:2:8
for t=1:4
fname = strcat('a', int2str(a), '_s', int2str(s), '_t', int2str(t),'_inertial.mat');
if exist(fname)
load(fname);
d_iner = d_iner(:,1:3);
r = size(d_iner,1);
d_iner = cat(1,d_iner,zeros(longest-r,3)); % do zero padding to longest sequence
ctr = ctr+1;
d_iner = d_iner';
gyroData(ctr,:,:) = d_iner(:);
gyroLabel(ctr,1)=a-1;
end
end
end
end
n1 = randperm(ctr);
for i=1:ctr
if i==1
x2=gyroData(n1(i),:);
y2=gyroLabel(n1(i),1);
else
x2=cat(1,x2,gyroData(n1(i),:));
y2=cat(1,y2,gyroLabel(n1(i),1));
end
end
save('inertial_padded.mat', 'x1', 'y1', 'x2', 'y2');
**And this is my LSTM code in Keras**
import numpy as np
import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
import scipy.io
# fix random seed for reproducibility
np.random.seed(7)
mat = scipy.io.loadmat('inertial_padded.mat')
x_train = mat['x1']
y_train = mat['y1']
x_test = mat['x2']
y_test = mat['y2']
data_dim = 3
timesteps = 326
num_classes = 27
x_train = x_train.reshape(x_train.shape[0], 326,3)
x_test = x_test.reshape(x_test.shape[0], 326, 3)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
model = Sequential()
model.add(LSTM(32, input_shape=(timesteps, data_dim), activation='sigmoid'))
#model.add(LSTM(32, activation='tanh')) # returns a sequence of vectors of dimension 32
#model.add(LSTM(32)) # return a single vector of dimension 32
model.add(Dense(27, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
print(model.summary())
model.fit(x_train, y_train,
batch_size=16, epochs=25,
validation_data=(x_test, y_test))
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))