Custom loss function including output of layer - python-3.x

I have a dataset that contains 1 value of y_true per case. I want to build a DNN that outputs 3 coefficients that will later be used as follows to create y_pred
y_pred = 4*coeff_1 + 5*coeff_2 + 6 *coeff_3
I am using keras and when i tried to define a custom function like this
from keras.callbacks import ModelCheckpoint
from keras.layers import advanced_activations
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error
import keras.backend as K
def custom_objective(layer):
return K.sum(layer.output)
NN_model = Sequential()
# The Input Layer :
NN_model.add(Dense(X_train.shape[1], kernel_initializer='normal',input_dim = X_train.shape[1], activation='relu'))
# The Hidden Layers :
NN_model.add(Dense(20, kernel_initializer='normal',activation='elu'))
NN_model.add(Dense(20, kernel_initializer='normal',activation='elu'))
output_layer = Dense(1, kernel_initializer='normal',activation='linear')
# The Output Layer :
NN_model.add(output_layer)
# Compile the network :
NN_model.compile(loss=custom_objective(output_layer), optimizer='Adamax', metrics=['mean_absolute_error'])
NN_model.summary()
NN_model.fit(X_train, y_train, epochs=10,verbose = 1)
print('NN train = ', mean_absolute_error(y_train , NN_model.predict(X_train)))
predictions = NN_model.predict(X_test)
MAE = mean_absolute_error(y_test , predictions)
print('NN MAE = ', MAE)
I get all
TypeError: Using a tf.Tensor as a Python bool is not allowed. Use
if t is not None: instead of if t: to test if a tensor is defined,
and use TensorFlow ops such as tf.cond to execute subgraphs
conditioned on the value of a tensor.
So my question is
How can I define a DNN that will take 1 y_true per data, output 3 values which it will combine linearly to assemble a y_pred which will be used to get the loss function and train the network
Thank you for your time

How about something along these lines?
from keras.models import Model
from keras.layers import Dense, Input, Add, Lambda
def model(inp_size):
inp = Input(shape=(inp_size, 1))
x1 = Dense(20, activation='elu')(inp)
x1 = Dense(20, activation='elu')(x1)
x1 = Dense(1, activation = 'linear')(x1)
x2 = Dense(20, activation='elu')(inp)
x2 = Dense(20, activation='elu')(x2)
x2 = Dense(1, activation = 'linear')(x2)
x3 = Dense(20, activation='elu')(inp)
x3 = Dense(20, activation='elu')(x3)
x3 = Dense(1, activation = 'linear')(x3)
x1 = Lambda(lambda x: x * 4.0)(x1)
x2 = Lambda(lambda x: x * 5.0)(x2)
x3 = Lambda(lambda x: x * 6.0)(x3)
out = Add()([x1, x2, x3])
return Model(inputs = inp, outputs = out)

Related

Add custom objects to a keras model while building it

I would like to use a custom accuracy function. I would prefer to add the custom object to the model when creating it (not saving the model and loading it again to add the object).
I first load the following libraries:
import pickle
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from keras.layers import Dense
from keras.models import Sequential
from keras.optimizers import gradient_descent_v2
from sklearn.model_selection import train_test_split
from tensorflow import keras
import keras.backend as K
from keras.models import load_model
Then, I define my custom function as below:
def cust_acc(y_true, y_pred):
acc = ((y_true == y_pred) & (y_true == 0)) | \
(y_true * y_pred > 0) | \
((y_true == 0) & (y_pred < 0)) | \
((y_true < 0) & (y_pred == 0))
return K.sum(acc) / K.size(acc)
Here, I read the input values and define the structure of the NN model:
InstNum = 'Base' # Instance number
file = open('Overtime_Prediction_Inst' + str(InstNum) + '.pkl', 'rb')
X, y, inp, b1, b2 = pickle.load(file)
file.close
nL = 3
alpha = 0.01
act = [' ',
'relu',
'linear']
nN = [0, 10, 1]
And here is where I normalize the data points and build the model:
scaler = MinMaxScaler()
scaler.fit(X)
nX = scaler.transform(X)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)
# define model
term = keras.callbacks.TerminateOnNaN()
model = Sequential()
model.add(Dense(nN[1], input_dim=nN[0], activation=act[1]))
for i in range(2, nL):
model.add(Dense(nN[i], activation=act[i]))
# compile model
model = load_model(model, custom_objects={'cust_acc': cust_acc})
model.compile(loss='MeanAbsoluteError',
optimizer=gradient_descent_v2.SGD(learning_rate=0.01, momentum=0.9),
accuracy=['cust_acc'])
# fit model
history = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=30, callbacks=[term])
# evaluate the model
train_mse = model.evaluate(X_train, y_train)
test_mse = model.evaluate(X_test, y_test)
But I get an error in line model = load_model(model, custom_objects={'amir_acc': amir_acc}) as follows:
'Unable to load model. Filepath is not an hdf5 file (or h5py is not '
OSError: Unable to load model. Filepath is not an hdf5 file (or h5py is not available) or SavedModel. Received: filepath=<keras.engine.sequential.Sequential object at 0x000001FE209BAD08>
Let me know if you want actual data to be able to reproduce the results. Thanks for the help.

Multi-input/Multi-output : Wrong output dimension when using KerasClassifier and GridSearchCV

I have built a multi-input (100 features) multi-ouput (100 predictions) ANN model using keras and tensorflow. I have been able to train my model and reach a quite satisfying accuracy on the test set using the following code :
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import backend as K
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Dropout
def my_loss_fn(y_true, y_pred) :
d = K.sum(K.abs(y_true), axis = -1)
n = K.sum((K.tanh(100000*y_true*y_pred)/2 + 0.5)*K.abs(y_true), axis = -1)
return 1 - n/d
def my_metric_fn(y_true, y_pred) :
d = K.sum(K.abs(y_true))
n = K.sum((K.tanh(100000*y_true*y_pred)/2 + 0.5)*K.abs(y_true))
return n/d
def accuracy(y_true, y_pred) :
#print(y_true.shape, y_true)
#print(y_pred.shape, y_true)
acc = np.zeros([1, len(y_true)])
for day in range(len(y_pred)) :
d = 0
n = 0
for i in range(len(y_pred[0])) :
d = d + abs(y_true[day, i])
if np.sign(y_pred[day, i])*np.sign(y_true[day, i]) > 0 :
n = n + abs(y_true[day, i])
else :
n = n + 0
acc[0, day] = n/d
return np.mean(acc, axis = -1)[0]
#Model
classifier = Sequential()
classifier.add(Dense(units = 50, input_shape = (100, ), activation = "tanh"))
classifier.add(Dropout(0.2))
classifier.add(Dense(units=100, activation = 'tanh'))
classifier.compile(optimizer = 'rmsprop', loss = my_loss_fn, metrics = ['accuracy', my_metric_fn])
#Training
callback = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', min_delta = 0.0001, patience = 20, verbose = 0, mode = 'min')
nb_epochs = 250
history = classifier.fit(X_train, y_train, epochs = nb_epochs, batch_size = 31, callbacks = [callback], verbose = True, validation_split = 0., validation_data = (X_test, y_test), use_multiprocessing = True)
#Prediction
y_pred_train = classifier.predict(X_train)
y_pred_test = classifier.predict(X_test)
acc_test = accuracy(y_test, y_pred_test)
acc_train = accuracy(y_train, y_pred_train)
I am trying to improve the performance of my model by tuning the hyperparameters so I used KerasClassifier() and GridSearchCV(). The following code illustrates my approach for the gridsearch.
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer
from tensorflow import autograph
#Building a function to create the classifier
def build_classifier(nb_layers, nb_nodes, optimizer, dropout, activation_fn):
classifier=Sequential()
classifier.add(Dense(units = nb_nodes, input_shape = (100, ), activation = activation_fn))
for i in range(nb_layers-1) :
classifier.add(Dense(units = nb_nodes, activation = activation_fn, kernel_initializer = "uniform"))
classifier.add(Dropout(dropout))
classifier.add(Dense(units = 100, activation = 'tanh'))
classifier.compile(optimizer=optimizer, loss = tf.autograph.experimental.do_not_convert(my_loss_fn), metrics= ['accuracy', tf.autograph.experimental.do_not_convert(my_metric_fn)])
return classifier
#Creating a scorer to feed to the GridSearchCV()
my_scorer = make_scorer(accuracy, greater_is_better = True)
classifier=KerasClassifier(build_fn=build_classifier)
parameters={'batch_size':[13, 31],'epochs':[100, 150], 'optimizer':['adam', 'rmsprop'], 'dropout' : [0.2, 0.1], 'nb_layers' : [2, 3], 'nb_nodes' : [45, 50, 110, 115], 'activation_fn' : ['relu', 'tanh']}
grid_search=GridSearchCV(estimator=classifier, scoring = my_scorer, param_grid=parameters, cv=5, verbose = 1)
grid_search=grid_search.fit(X_train_, y_train_raw)
When I fit my GridSearchCV() object I get the following error at the end of the first combination of hyperparameters (when the scoring is computed) :
TypeError: object of type 'numpy.int32' has no len()
I investigated by adding print commandes inside my accuracy() function
#print(y_true.shape, y_true)
#print(y_pred.shape, y_pred)
to print both the shape and the array y_true and y_pred given as inputs for my accuracy() function used as the scoring in the GridSearchCV() object.
I found out that y_true.shape == (555, 100) but y_pred.shape == (555,). The value 555 corresponds to the number of lines of the fifth validation set because cv = 5.
However, I do not understand why the prediction of the gridsearch is not a multi-output prediction even though the number of nodes of the last layer of the classifier is (100,).
This was a regression problem so I used KerasRegressor() instead and it solved the issue. I guess that for a multi-output classification problem, KerasClassifier() expect the output to be a 2D hot encoded array.

Is there any way to fit and apply deep learning algorithm on chemical smiles data in sequential model?

I have written a code for this where my input as an X
X : c1ccccc1 and Y value is water/methanol as classification category.
# multi-class classification with Keras
import pandas
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from keras.utils import np_utils
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
from sklearn.pipeline import Pipeline
# load dataset
dataframe = pandas.read_csv("ADLV3_1.csv", header=None)
dataset = dataframe.values
X = dataset[:,2:3]
Y = dataset[:,3:4]
# encode class values as integers
encoder = LabelEncoder()
encoder.fit(Y)
encoded_Y = encoder.transform(Y)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y = np_utils.to_categorical(encoded_Y)
# define baseline model
def baseline_model():
# create model
model = Sequential()
model.add(Dense(8, input_dim=4, activation='relu'))
model.add(Dense(3, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
estimator = KerasClassifier(build_fn=baseline_model, epochs=200, batch_size=5, verbose=0)
kfold = KFold(n_splits=10, shuffle=True)
results = cross_val_score(estimator, X, dummy_y, cv=kfold)
print("Baseline: %.2f%% (%.2f%%)" % (results.mean()*100, results.std()*100))
Code running successfully but getting warning as
/usr/local/lib/python3.7/dist-packages/sklearn/preprocessing/_label.py:235: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
y = column_or_1d(y, warn=True)
/usr/local/lib/python3.7/dist-packages/sklearn/preprocessing/_label.py:268: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
y = column_or_1d(y, warn=True)
/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details:
ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type float).
FitFailedWarning)
Baseline: nan% (nan%)
Is there any solution to make the algorithm workable? I can't predict any values

Keras TimeseriesGenerator: error when checking input

When I try to use the TimeSeriesGenerator function, my Keras LSTM NN starts training for a few moments but then gives a ValueError message. What's wrong? I wonder how it can start training and then get an error.
My similar implementation without this function runs smoothly but then the quality of the predictions are awful (and I'm not sure that this function, once successfully implemented, would make a difference).
See the code below:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Nadam
from tensorflow.keras.layers import Input, LSTM, Dense
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, TerminateOnNaN
from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator
data = pd.read_excel('example.xlsx',usecols=['wave','wind','current','X','Y','RZ'])
data = data.apply(lambda x: (x - np.mean(x)) / np.std(x))
n_cutoff = 200
X = np.array(data.loc[n_cutoff:,['wave','wind']])
Y = np.array(data.loc[n_cutoff:,['RZ']])
X = X.reshape(len(X),2)
X = np.append(X, [[0]*np.size(X, axis=1)], axis=0)
Y = Y.reshape(len(Y),1)
Y = np.insert(Y, 0, 0)
n_lag = 3
n_batch = 15
n = int(0.75*len(X))
generator = TimeseriesGenerator(X, Y, length=n_lag, batch_size=n_batch)
inputs = Input(shape=(n_lag,2))
hidden1 = LSTM(units=100,
activation='softmax',
recurrent_activation='linear',
dropout=0.5,
recurrent_dropout=0.5,
return_sequences=True)(inputs)
hidden2 = LSTM(units=30,
activation='softmax',
recurrent_activation='linear',
dropout=0.5,
recurrent_dropout=0.5,
return_sequences=False)(hidden1)
outputs = Dense(units=1,
activation='linear')(hidden2)
model = Model(inputs=inputs, outputs=outputs)
optimizer = Nadam(learning_rate=1e-2, beta_1=0.95, beta_2=0.9, epsilon=1e-7)
model.compile(loss='mean_squared_error', optimizer=optimizer)
history = model.fit(generator,
verbose=1,
steps_per_epoch=int(n/n_batch),
epochs=1,
shuffle=False,
callbacks=[EarlyStopping(monitor='loss', min_delta=0, patience=20, verbose=1, mode='auto'),
ReduceLROnPlateau(monitor='loss', factor=0.5, patience=10, verbose=1, mode='auto', cooldown=1),
TerminateOnNaN()])
Y_hat = model.predict(X[n:])

Make new predictions with a trained ANN - Keras

I'm new to the world of ANN and I was wondering, how do I pass a new dataset, possible a new csv, into the model that I alredy trained? I understand, for instance, that:
model.predict()
only accept arrays, and those arrays have to be of the same shape in order to work. So, how do I pass a complete new csv to generate predictions?
Here is my code, I know that probably it's gonna be a mess, but I'm working on it.
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
# Importing the dataset
dataset = pd.read_csv('Prova_1.csv')
dataset = dataset[np.isfinite(dataset['ID'])]
X = dataset.iloc[:, 3:6].values
y = dataset.iloc[:, 6].values
# Encoding
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
labelencoder_X_1 = LabelEncoder()
X[:, 0] = labelencoder_X_1.fit_transform(X[:, 0])
labelencoder_X_2 = LabelEncoder()
X[:, 2] = labelencoder_X_2.fit_transform(X[:, 2])
onehotencoder = OneHotEncoder(categorical_features = [2])
X = onehotencoder.fit_transform(X).toarray()
X = X[:, 1:]
# Splitting
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
# Feature Scaling
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
#ANN
import keras
from keras.models import Sequential
from keras.layers import Dense
# Initialising the ANN
classifier = Sequential()
# Adding the input layer and the first hidden layer
classifier.add(Dense(units = 27, kernel_initializer = 'uniform', activation = 'relu', input_dim = 55))
# Adding the second hidden layer
classifier.add(Dense(units = 27, kernel_initializer = 'uniform', activation = 'relu'))
# Adding the output layer
classifier.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))
# Compiling the ANN
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])
# Fitting the ANN to the Training set
classifier.fit(X_train, y_train, batch_size = 1, epochs = 100)
# Prediction
y_pred = classifier.predict(X_test)
y_pred = (y_pred > 0.5)
You need to do all the preprocessing operations to train data for the new data. Because the model trained over your input shape and input features. Make sure your input shape of your input data same as before, unless it will not work.

Resources