I want to write my custom training function, but I couldn't access my trainable_weights because it returns []. I can get the weights using layer.get_weight() but my trainable_variables are empty. this is my training method:
def train_on_batch(X, y_real,model):
with tf.GradientTape() as tape:
tape.watch(X)
y_pred = model(X, training= True)
print(model.trainable_variables)
loss_value = loss(y_real, y_pred)
grads = tape. gradient(loss_value, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
return loss_value
and this is my part of CNN model:
model_list = list()
# base model input
in_image = Input(shape=input_shape)
# conv 1x1
d = Conv2D(128, (1,1), padding='same', kernel_initializer=init, kernel_constraint= const)(in_image)
d = LeakyReLU(alpha=0.2)(d)
# conv 3x3 (output block)
d = MinibatchStdev()(d)
d = Conv2D(128, (3,3), padding='same', kernel_initializer=init, kernel_constraint= const)(d)
d = LeakyReLU(alpha=0.2)(d)
# conv 4x4
d = Conv2D(128, (4,4), padding='same', kernel_initializer=init, kernel_constraint= const)(d)
d = LeakyReLU(alpha=0.2)(d)
# dense output layer
d = Flatten()(d)
out_class = Dense(1, name='dense')(d)
print(type(out_class))
# define model
model = Model(in_image, out_class)
enter code here
Check the last block of code of question. This code has given the best way to get the variables.
TensorFlow 2.0 How to get trainable variables from tf.keras.layers layers, like Conv2D or Dense
Related
I'm trying to solve the code error which comes from this line (not sure)
K.function([model2.layers[0].input], [model2.layers[-1].output])
The full code can be found here CODE and in this post I show only the problematic part:
inputs2 = Input(shape=(X_train2.shape[1], X_train2.shape[2]))
lstm2 = LSTM(128, return_sequences=True, dropout=0.3)(inputs2, training=True)
lstm2 = LSTM(32, return_sequences=False, dropout=0.3)(lstm2, training=True)
dense2 = Dense(50)(lstm2)
out2 = Dense(1)(dense2)
model2 = Model(inputs2, out2)
model2.compile(loss='mse', optimizer='adam', metrics=['mse'])
history = model2.fit(X_train2, y_train2, epochs=2, batch_size=128, verbose=2, shuffle=True)
# function for stochastic dropout:
def stoc_drop2(R, NN):
trans_pred = scaler2.transform(XE[len(df_x_train1):].reshape(-1, XE.shape[-1])).reshape(-1, SEQUENCE_SIZE, XE.shape[-1])
print(">>>",trans_pred.shape)
NN_pred = NN([trans_pred, R])
return np.vstack(NN_pred)
NN = K.function([model2.layers[0].input], [model2.layers[-1].output])
for i in tqdm.tqdm(range(0, 100)):
scores2.append(mean_absolute_error(y_test2, stoc_drop2(0.5, NN)))
When running the code I get the following:
ValueError: Layer "model_1" expects 1 input(s), but it received 2 input tensors. Inputs received: [<tf.Tensor: shape=(16482, 10, 3), dtype=float64, numpy=
array(...)>, <tf.Tensor: shape=(), dtype=float32, numpy=0.5>]
The NN expects to recieve input tensor, however dropout value is also added
NN_pred = NN([trans_pred, R])
partial solution:
I removed R and use just
NN([trans_pred])
Still do not understand how to pass dropout parameter through K.function() to model.
I am struggling with scaling the output of a lambda layer. The code is as follows:
My X_train is 100*15*24 and Y_train is 100*1 (the network consists with a LSTM layer + Dense layer)
input_shape=(timesteps, num_feat)
data_input = Input(shape=input_shape, name="input_layer")
lstm1 = LSTM(10, name="lstm_layer")(data_input)
dense1 = Dense(4, activation="relu", name="dense1")(lstm1)
dense2 = Dense(1, activation = "custom_activation_1", name = "dense2")(dense1)
dense3 = Dense(1, activation = "custom_activation_2", name = "dense3")(dense1)
#dense 2 and 3 has customed activation function with range the REAL LINE (so I need to normalize it)
## custom lambda layer/ loss function ##
def custom_layer(new_input):
add_input = new_input[0]+new_input[1]
#below three lines are where problem occurs that makes the program does not work
###############################################
scaler = MinMaxScaler()
scaler.fit(add_input)
normalized = scaler.transform(add_input)
###############################################
return normalized
lambda_layer = Lambda(custom_layer, name="lambda_layer")([dense2, dense3])
model = Model(inputs=data_input, outputs=lambda_layer)
model.compile(loss='mse', optimizer='adam',metrics=['accuracy'])
model.fit(X_train, Y_train, epochs=2, batch_size=216)
How can I normalize the output of lambda_layer properly? Any ideas or suggestions are appreciated!
I don't think Scikit transformers would work in Lambda layers. If you're only interested in getting the normalized output w.r.t the data passed in, you can do,
from tensorflow.keras.layers import Input, LSTM, Dense, Lambda
from tensorflow.keras.models import Model
import tensorflow as tf
timesteps = 3
num_feat = 12
input_shape=(timesteps, num_feat)
data_input = Input(shape=input_shape, name="input_layer")
lstm1 = LSTM(10, name="lstm_layer")(data_input)
dense1 = Dense(4, activation="relu", name="dense1")(lstm1)
dense2 = Dense(1, activation = "custom_activation_1", name = "dense2")(dense1)
dense3 = Dense(1, activation = "custom_activation_2", name = "dense3")(dense1)
#dense 2 and 3 has customed activation function with range the REAL LINE (so I need to normalize it)
## custom lambda layer/ loss function ##
def custom_layer(new_input):
add_input = new_input[0]+new_input[1]
normalized = (add_input - tf.reduce_min(add_input, axis=0, keepdims=True))/(tf.reduce_max(add_input, axis=0, keepdims=True) - tf.reduce_max(add_input, axis=0, keepdims=True))
return normalized
lambda_layer = Lambda(custom_layer, name="lambda_layer")([dense2, dense3])
model = Model(inputs=data_input, outputs=lambda_layer)
model.compile(loss='mse', optimizer='adam',metrics=['accuracy'])
I just need to concatenate a flatten layer and a feature vector in Keras. This is the code:
#custom parameters
n_features = 38
vgg_model = VGGFace(include_top=False, input_shape=(224, 224, 3))
last_layer = vgg_model.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
# feature vector
feature_vector = Input(shape = (n_features,))
conc = concatenate(([x, feature_vector]), axis=1)
layer_intermediate = Dense(128, activation='relu', name='fc6')(conc)
layer_intermediate1 = Dense(32, activation='relu', name='fc7')(layer_intermediate)
out = Dense(5, activation='softmax', name='fc8')(layer_intermediate1)
custom_vgg_model = Model(vgg_model.input, out)
But I'm getting this error:
---> 20 custom_vgg_model = Model(vgg_model.input, out)
ValueError: Graph disconnected: cannot obtain value for tensor Tensor("input_88:0", shape=(?, 38), dtype=float32) at layer "input_88". The following previous layers were accessed without issue: ['input_87', 'conv1_1', 'conv1_2', 'pool1', 'conv2_1', 'conv2_2', 'pool2', 'conv3_1', 'conv3_2', 'conv3_3', 'pool3', 'conv4_1', 'conv4_2', 'conv4_3', 'pool4', 'conv5_1', 'conv5_2', 'conv5_3', 'pool5', 'flatten']
Btw the shape of the flatten layer is (None, 25088)
Since your feature_vector is also Input. Try to add feature_vector into inputs when you define the Model.
custom_vgg_model = Model([vgg_model.input,feature_vector], out)
I am trying to implement a neural network where I merge/concatenate a fully connected neural network with a convolution neural network. But when I fit the model, I get the following error:
ValueError: All input arrays (x) should have the same number of
samples. Got array shapes: [(1, 100, 60, 4500), (100, 4500)]
I have two different inputs:
image(dimensions: 1,100,60,4500) where 1 is the channel, 100: # of sample, 60*4500 (dimension of my image). This goes to my convolution neural network
positions(dimensions: 100,4500): where 100 refers to samples.
Dimension for my output is 100,2.
The code for my neural network is:
###Convolution neural network
b1 = Sequential()
b1.add(Conv2D(128*2, kernel_size=3,activation='relu',data_format='channels_first',
input_shape=(100,60,4500)))
b1.add(Conv2D(128*2, kernel_size=3, activation='relu'))
b1.add(Dropout(0.2))
b1.add(Conv2D(128*2, kernel_size=4, activation='relu'))
b1.add(Dropout(0.2))
b1.add(Flatten())
b1.summary()
###Fully connected feed forward neural network
b2 = Sequential()
b2.add(Dense(64, input_shape = (4500,), activation='relu'))
b2.add(Dropout(0.1))
b2.summary()
model = Sequential()
###Concatenating the two networks
concat = concatenate([b1.output, b2.output], axis=-1)
x = Dense(256, activation='relu', kernel_initializer='normal')(concat)
x = Dropout(0.25)(x)
output = Dense(2, activation='softmax')(x)
model = Model([b1.input, b2.input], [output])
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(),
metrics=['accuracy'])
history = model.fit([image, positions], Ytest, batch_size=10,
epochs=1,
verbose=1)
Also, the reason why my 'image' array is 4 dimensional is because in the beginning it was just (100,60,4500) but then I ran into the following error:
ValueError: Error when checking input: expected conv2d_10_input to
have 4 dimensions, but got array with shape (100, 60, 4500)
And upon googling I found out that it expects # of channels as an input too. And after I added the # of channel, this error went away but then I ran into the other error that I mentioned in the beginning.
So can someone tell me how to solve for the error (the one I specified in the beginning)? Help would be appreciated.
It is not a good practice to mix Sequential and Functional API.
You can implement the model like this
i1 = Input(shape=(1, 60, 4500))
c1 = Conv2D(128*2, kernel_size=3,activation='relu',data_format='channels_first')(i1)
c1 = Conv2D(128*2, kernel_size=3, activation='relu')(c1)
c1 = Dropout(0.2)(c1)
c1 = Conv2D(128*2, kernel_size=4, activation='relu')(c1)
c1 = Dropout(0.2)(c1)
c1 = Flatten()(c1)
i2 = Input(shape=(4500, ))
c2 = Dense(64, input_shape = (4500,), activation='relu')(i2)
c2 = Dropout(0.2)(c2)
c = concatenate([c1, c2])
x = Dense(256, activation='relu', kernel_initializer='normal')(c)
x = Dropout(0.25)(x)
output = Dense(2, activation='softmax')(x)
model = Model([i1, i2], [output])
model.summary()
Note the shape of i1 is shape=(1, 60, 4500). You have set data_format='channels_first' in Conv2D layer hence you need 1 in the beginning.
Compiled the model like this
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
Placeholder data
import numpy as np
X_img = np.zeros((100, 1, 60, 4500))
X_pos = np.ones((100, 4500))
Y = np.zeros((100, 2))
Training
history = model.fit([X_img, X_pos], Y, batch_size=1,
epochs=1,
verbose=1)
You number of samples (batch size) should always be the first dimension. So, your data should have shape (100, 1, 60, 4500) for image and (100, 4500) for positions. The argument channels_first for the Conv2D layer means that the channels is the first non-batch dimension.
You also need to change the input shape to (1, 60, 4500) in the first Conv2D layer.
I have a trained cnn model. I am trying to extract the output from each convolutional layer and plot the results to explore which regions of the image have high activations. Any ideas on how to do this?
Below is the network I have trained.
input_shape = (3,227,227)
x = Input(input_shape)
# Conv Layer 1
x = Convolution2D(96, 7,7,subsample=(4,4),activation='relu',
name='conv_1', init='he_normal')(x_input)
x = MaxPooling2D((3, 3), strides=(2,2), name='maxpool')(x)
x = BatchNormalization()(x)
x = ZeroPadding2D((2,2))(x)
# Conv Layer 2
x = Convolution2D(256, 5,5,activation='relu',name='conv_2', init='he_normal')(x)
x = MaxPooling2D((3, 3), strides=(2,2),name='maxpool2')(x)
x = BatchNormalization()(x)
x = ZeroPadding2D((2,2))(x)
# Conv Layer 3
x = Convolution2D(384, 3,3,activation='relu',
name='conv_3', init='he_normal')(x)
x = MaxPooling2D((3, 3), strides=(2,2),name='maxpool3')(x)
x = Flatten()(x)
x = Dense(512, activation = "relu")(x)
x = Dropout(0.5)(x)
x = Dense(512, activation ="relu")(x)
x = Dropout(0.5)(x)
predictions = Dense(2, activation="softmax")(x)
model = Model(inputs = x_input, outputs = predictions)
Thanks!
Look at this GitHub issue and the FAQ How can I obtain the output of an intermediate layer?. It seems the easiest way to do that is defining new models with the outputs that you want. For example:
input_shape = (3,227,227)
x = Input(input_shape)
# Conv Layer 1
# Save layer in a variable
conv1 = Convolution2D(96, 7, 7, subsample=(4,4), activation='relu',
name='conv_1', init='he_normal')(x_input)
x = conv1
x = MaxPooling2D(...)(x)
# ...
conv2 = Convolution2D(...)(x)
x = conv2
# ...
conv3 = Convolution2D(...)(x)
x = conv3
# ...
predictions = Dense(2, activation="softmax")(x)
# Main model
model = Model(inputs=x_input, outputs=predictions)
# Intermediate evaluation model
conv_layers_model = Model(inputs=x_input, outputs=[conv1, conv2, conv3])
# After training is done, retrieve intermediate evaluations for data
conv1_val, conv2_val, conv3_val = conv_layers_model.predict(data)
Note that since you are using the same objects in both models the weights are automatically shared between them.
A more complete example of activation visualization can be found here. In that case they use the K.function approach.