I have two custom submodels model1 and model2. I concatenate them in another model3:
class ConcatenateModel(tf.keras.model):
def __init__(self, model1, model2, **kwargs):
super().__init__(**kwargs)
self.model1 = model1
self.model2 = model2
def call(self, inputs):
x = self.model1(inputs)
x = self.model2(x)
return x
model3 = ConcatenateModel(model1, model2)
The concatenation allows me to access the individual outputs of each submodel after training model3.
To train model3, I run model3.compile(...) and model3.fit(...). As I understand the training procedure, I do NOT have to compile model1 and model2 to train model3, right?
Curiously, I am getting a warning when training model3:
WARNING:tensorflow:Gradients do not exist for variables "layer in model2" when minimizing the loss. If you're using model.compile(), did you forget to provide a loss argument?
I don't really get why this warning occurs. Yes, I didn't compile model2. Thus, it has no loss argument. But since I provided a loss in model3.compile() and run fit() on model3 that shouldn't be a problem, right?
Related
I want to create an MLP based custom CNN model (multi-scaled) consists of several parallel small networks (capsules). These simple small networks are instantiated as a custom layer (conv2d->Flatten->Dense) for each convolution scale i.e. 3x3, 5x5. The purpose of these capsule networks is to generate intermediate loss consciousness to reduce overall global loss using the CNN model. I have written some sketchy codes but I'm not able to write the correct code for computing local loss using these capsules. Here's the code:
from tensorflow.keras import layers
import tensorflow as tf
from tensorflow.keras.datasets import mnist
from tensorflow.keras.layers import Layer
class capsule(tf.keras.layers.Layer):
def __init__(self):
super(capsule, self).__init__()
self.loss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=True)
self.Flatten = tf.keras.layers.Flatten()
self.conv2D = tf.keras.layers.Conv2D(3,3,(1,1),padding='same', activation='relu',name="LocalLoss3x3")
self.classifier = tf.keras.layers.Dense(10,activation='softmax', name='capsule3Output')
def call(self, inputs):
x=self.conv2D(inputs)
x=self.Flatten(x)
x=self.classifier(x)
pred=self(x_train)
loss=self.loss_fn(pred,y_train)
#self.add_loss(self.rate * tf.reduce_sum(tf.square(inputs)))
return loss, x
(x_train, y_train), (x_test, y_test)= mnist.load_data()
from tensorflow.keras import layers
class SparseMLP(tf.keras.models.Model):
def __init__(self, output_dim):
super(SparseMLP, self).__init__()
self.dense_1 = layers.Dense(1, activation=tf.nn.relu)
self.capsule = capsule()
self.dense_2 = layers.Dense(output_dim)
def call(self, inputs):
x = self.dense_1(inputs)
loss,x = self.capsule(inputs)
return self.dense_2(x)
mlp = SparseMLP(10)
#x_train=x_train.reshape(-1,28,28,1)
y = mlp(x_train)
To include a loss within a layer , you can use add_loss function of tf.keras.layers.Layer class. This fucntion takes a loss value and adds it up to the global loss function define in compile function.
you can call self.add_loss(loss_value) from inside the call method of a custom
layer.Losses added in this way get added to the "main" loss during training
(the one passed to compile()).
So to make ur model consider the losses from intermediate layer , you should uncomment the add_loss fn , and then train the model in usual way that you train.
Please mind that it is totally fine to not declare a "main" loss in the compile function as there already is a loss that ur defining in your layer class.
Note that when you pass losses via add_loss(), it becomes possible to call compile() without a loss function, since the model already has a loss to minimize.
Please note that call function of SparseMLP model , should look like this:
x = self.dense_1(inputs)
# i dunno if u desire to do this, that is pass inputs in capsule
# instead of x.Currently the output from dense_1 is not used at all .
# so keep in mind to make sure ur passing proper inputs to layers.
# and u do not have to call loss here as it will tracked internally by
# keras.
x = self.capsule(inputs)
return self.dense_2(x)
So running your model like below should do the trick:
model.compile(loss = "define ur main loss is there is" , metrics = "define ur metrics")
model.fit(x = train_inst , y = train_targets)
Is there a way to save the entire model build using tf.keras Model subclassing API after the training is done? I know we can use save_weights to save the weights only, but is there a way to save the whole model so that I may use it for prediction later when I do not have the code available?
class MyModel(tf.keras.Model):
def __init__(self, num_classes=10):
super(MyModel, self).__init__(name='my_model')
self.num_classes = num_classes
# Define your layers here.
self.dense_1 = layers.Dense(32, activation='relu')
self.dense_2 = layers.Dense(num_classes, activation='sigmoid')
def call(self, inputs):
# Define your forward pass here,
# using layers you previously defined (in `__init__`).
x = self.dense_1(inputs)
return self.dense_2(x)
model = MyModel(num_classes=10)
# The compile step specifies the training configuration.
model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(data, labels, batch_size=32, epochs=5)
You can use the following steps for saving model after training, loading and inference:
Save Model after training
model.save(filepath="model")
# OR
tf.keras.models.save_model(model, filepath="model_")
Load Saved Model
loaded_model = tf.keras.models.load_model(filepath="model_")
Prediction using Loaded model
result = loaded_model.predict(test_db)
I'm working on a neural network in Keras that translates English sentences into a custom language. For this, I'd like to create a custom loss function that takes the prediction for each sentence and evaluates whether it complies with the grammar rules of the custom language and if not adds value to the standard loss function.
How can I evaluate a tensor after each epoch but not during compilation?
Below is my custom loss function. As during compilation of the model there is no batch yet, y_pred has the shape (None, x, y) and can't be evaluated to get the prediction. My idea to circumvent this was to assign a standard loss function during compilation and when batches arrive calculate the custom loss. Unfortunately the custom loss is never reached.
def custom_loss(tokenizer, punishment_rate):
def compile_loss(y_true, y_pred):
shape = K.int_shape(y_pred)
#standard loss function
loss = K.sparse_categorical_crossentropy(y_true, y_pred)
if shape[0] is not None:
#THIS is never reached and that's the problem
prediction = logits_to_text(K.eval(y_pred), tokenizer)
#test if prediction complies to grammar rules
compileable = compiles(prediction) ^ 1
compile_error = compileable * punishment_rate
loss = K.sparse_categorical_crossentropy(y_true, y_pred, axis=-1) * (1 + compile_error)
return loss
return compile_loss
Is there any workaround for evaluating a tensor only when it was filled with a batch? Or alternatively, change the loss function after compilation of the model via a callback without it having to recompile the model?
As per keras source, you can use a Loss Function Wrapper to create a Custom Loss Function class and then pass it to your model seamlessly.
As an example:
#Import the wrapper
from keras.losses import LossFunctionWrapper
#Create your class extending the wrapper
class MyLossFunction(LossFunctionWrapper):
#Implement the constructor - here you can give extended arguments to it.
def __init__(self,
tokenizer,
punishment_rate,
reduction=losses_utils.Reduction.SUM_OVER_BATCH_SIZE,
name='my_custom_text_function'):
super(MyLossFunction, self).__init__(
my_function,
name=name,
reduction=reduction,
tokenizer = tokenizer,
punishment_rate= punishment_rate)
#Now you have to define your function "my_function":
#Please, notice that ALL loss functions that follow keras model needs two arguments:
#y_true (correct result) and y_pred (the result obtained in the network).
def my_function(y_true, y_pred, tokenizer, punishment_rate):
shape = K.int_shape(y_pred)
if shape[0] is not None:
prediction = logits_to_text(K.eval(y_pred), tokenizer)
#test if prediction complies to grammar rules
compileable = compiles(prediction) ^ 1
compile_error = compileable * punishment_rate
return K.sparse_categorical_crossentropy(y_true, y_pred, axis=-1) * (1 + compile_error)
return K.sparse_categorical_crossentropy(y_true, y_pred)
You can then instantiate it and use in your compiler:
custom_loss= MyLossFunction(tokenizer = ..., punishment_rate = ...)
classifier.compile(optimizer=optimizer,
loss=custom_loss,
metrics= ['binary_accuracy'])
Using the keras subclass API it is easy enough to add a a batch normalization layer however the layer.losses list always appears empty. What is the correct method of including in the train loss when doing tape.gradient(loss, lossmodel.trainable_variables) where lossmodel is some separate keras subclass model defining a more complicated loss function that must include the gradient losses?
For example, this is minimal model with ONLY the batch norm layer. It has no loss AFAIK
class M(tf.keras.Model):
def __init__(self, axis):
super().__init__()
self.layer = tf.keras.layers.BatchNormalization(axis=axis, scale=False, center=True, virtual_batch_size=1, input_shape=(6,))
def call(self, x):
out = self.layer(x)
return out
m = M(1)
In [77]: m.layer.losses
Out[77]: []
I am training a model in keras and I want to plot graphs of results after each epoch. I know that keras callbacks provide "on_epoch_end" function that can be overloaded if one wants to do some computations after each epoch but my function takes some additional parameters which when given, crashes code by the meta class error. The detail is given below:
Here is how I am doing it right now, which is working fine:-
class NewCallback(Callback):
def on_epoch_end(self, epoch, logs={}): #working fine, printing epoch after each epoch
print("EPOCH IS: "+str(epoch))
epochs=5
batch_size = 16
model_saved=False
if model_saved:
vae.load_weights(args.weights)
else:
# train the autoencoder
vae.fit(x_train,
epochs=epochs,
batch_size=batch_size,
validation_data=(x_test, None),
callbacks=[NewCallback()])
But I want my callback function like this:-
class NewCallback(Callback,models,data,batch_size):
def on_epoch_end(self, epoch, logs={}):
print("EPOCH IS: "+str(epoch))
x=models.predict(data)
plt.plot(x)
plt.savefig(epoch+".png")
If I call it like this in fit:
callbacks=[NewCallback(models, data, batch_size=batch_size)]
I get this error:
TypeError: metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases
I am looking for a simpler solution to call my function or get this error of meta class resolved, any help will be much appreciated!
I think that what you would like to do is to define a class that descends from callback and takes models, data, etc... as constructor arguments. So:
class NewCallback(Callback):
""" NewCallback descends from Callback
"""
def __init__(self, models, data, batch_size):
""" Save params in constructor
"""
self.models = models
def on_epoch_end(self, epoch, logs={}):
x = self.models.predict(self.data)
In case you want to make predictions on the test data you can try this
class CustomCallback(keras.callbacks.Callback):
def __init__(self, model, x_test, y_test):
self.model = model
self.x_test = x_test
self.y_test = y_test
def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict(self.x_test, self.y_test)
print('y predicted: ', y_pred)
You need mention the callback during model.fit
model.sequence()
# your model architecture
model.fit(x_train, y_train, epochs=10,
callbacks=[CustomCallback(model, x_test, y_test)])
Similar to on_epoch_end there are many other methods provided by keras
on_train_begin, on_train_end, on_epoch_begin, on_epoch_end, on_test_begin,
on_test_end, on_predict_begin, on_predict_end, on_train_batch_begin, on_train_batch_end,
on_test_batch_begin, on_test_batch_end, on_predict_batch_begin,on_predict_batch_end