I have designed a layer in Keras. This is the first layer of the network. The input to this layer must be an RGB image ie of shape (height , width , 3). However when i run the code , i get the following error.
ValueError: Layer sequential_1 was called with an input that isn't a symbolic tensor. Received type: . Full input: [<main.CountPix object at 0x7fa9a5e81518>]. All inputs to the layer should be tensors.
How should I input my image or what should I modify in my layer?
class CountPix(Layer):
def __init__(self, **kwargs):
super(CountPix, self).__init__(**kwargs)
def build(self, input_shape):
# Create a trainable weight variable for this layer.
self.kernel = self.add_weight(name='kernel', shape=((200,200,3)),initializer='uniform',trainable=True)
super(MyLayer, self).build(input_shape) # Be sure to call this somewhere!
You need to define an input.
from keras.layers import Input
input_X = Input(shape=(height, width, 3), dtype='float32', name='input_image')
Also, if your self.kernel line you need to explicitly tell keras it is has an input shape similar to this example:
from keras.layers import Dense
from keras.models import Sequential
model = Sequential()
model.add(Dense(32, input_shape=(height, width, 3)))
Related
I would like to create a sequential model in keras with one hidden layer with as many nodes as there are input nodes. Each input node should be connected to only one of the hidden nodes. All nodes in the hidden layer should be connected to a single output node: as in this image
I would like to be able to specify the activation function of the hidden layer.
Is it possible to achieve that with a Sequential() model in keras?
Here is a custom layer where you can do everything you want:
import keras
import tensorflow as tf
from keras.layers import *
from keras import Sequential
import numpy as np
tf.set_random_seed(10)
class MyDenseLayer(keras.layers.Layer):
def __init__(self):
super(MyDenseLayer, self).__init__()
def parametric_relu(self, _x):
# some more or less complicated activation
# with own weight
pos = tf.nn.relu(_x)
neg = self.alphas * (_x - abs(_x)) * 0.5
return pos + neg
def build(self, input_shape):
# main weight
self.kernel = self.add_weight("kernel",
shape=[int(input_shape[-1]),],
initializer=tf.random_normal_initializer())
# any additional weights here
self.alphas = self.add_weight('alpha', shape=[int(input_shape[-1]),],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
self.size = int(input_shape[-1])
def call(self, input):
linear = tf.matmul(input, self.kernel*tf.eye(self.size))
nonlinear = self.parametric_relu(linear)
return nonlinear
model = Sequential()
model.add(MyDenseLayer())
model.build((None, 4))
print(model.summary())
x = np.ones((5,4))
print(model.predict(x))
In this notebook https://nbviewer.jupyter.org/github/krasserm/bayesian-machine-learning/blob/master/bayesian_neural_networks.ipynb, the author defines the function
def mixture_prior_params(sigma_1, sigma_2, pi):
params = K.variable([sigma_1, sigma_2, pi], name='mixture_prior_params')
sigma = np.sqrt(pi * sigma_1 ** 2 + (1 - pi) * sigma_2 ** 2)
return params, sigma
which creates a variable and returns a tuple. This method is then called
prior_params, prior_sigma = mixture_prior_params(sigma_1=1.0, sigma_2=0.1, pi=0.2)
Then, in the class DenseVariational, which is a custom layer, in the method build, the prior_params global variable is added to the private list _trainable_weights
def build(self, input_shape):
self._trainable_weights.append(prior_params)
...
Why would one need or want to do this? If I attempt to print the trainable parameters of either the custom layer or a model made of this custom layer, for example
# Create the model with DenseVariational layers
model = Model(x_in, x_out)
print("model.trainable_weights =", model.trainable_weights)
I can see that each DenseVariational layer contains a mixture_prior_params trainable parameter. Why should one declare mixture_prior_params, more specifically, sigma_1, sigma_2 and pi, outside of the layer, if they are trainable parameters of the layer?
After having looked at this question Can I share weights between keras layers but have other parameters differ? and its answer (https://stackoverflow.com/a/45258859/3924118) and having printed the values of the trainable variables of the model after the model has been trained, it seems like this is a way of sharing a variable across different layers, given that the value of that variable seems to be equal across layers, after the model has been trained.
I have created a simple example (with TensorFlow 2.0.0 and Keras 2.3.1) that shows this
import numpy as np
from keras import activations, initializers
from keras import backend as K
from keras import optimizers
from keras.layers import Input
from keras.layers import Layer
from keras.models import Model
shared_variable = K.variable([0.3], name='my_shared_variable')
class MyLayer(Layer):
def __init__(self, output_dim, activation=None, **kwargs):
self.output_dim = output_dim
self.activation = activations.get(activation)
super().__init__(**kwargs)
def build(self, input_shape):
self._trainable_weights.append(shared_variable)
self.my_weight = self.add_weight(name='my_weight',
shape=(input_shape[1], self.output_dim),
initializer=initializers.normal(),
trainable=True)
super().build(input_shape)
def call(self, x):
return self.activation(K.dot(x, self.my_weight * shared_variable))
def compute_output_shape(self, input_shape):
return input_shape[0], self.output_dim
if __name__ == "__main__":
# Define the architecture of the model.
x_in = Input(shape=(1,))
h1 = MyLayer(20, activation='relu')(x_in)
h2 = MyLayer(20, activation='relu')(h1)
x_out = MyLayer(1)(h2)
model = Model(x_in, x_out)
print("h1.trainable_weights (before training) =", model.layers[1].trainable_weights[0])
print("h2.trainable_weights (before training) =", model.layers[2].trainable_weights[0])
# Prepare the model for training.
model.compile(loss="mse", optimizer=optimizers.Adam(lr=0.03))
# Generate dataset.
X = np.linspace(-0.5, 0.5, 100).reshape(-1, 1)
y = 10 * np.sin(2 * np.pi * X)
# Train the model.
model.fit(X, y, batch_size=1, epochs=100, verbose=0)
print("h1.trainable_weights (after training) =", model.layers[1].trainable_weights[0])
print("h2.trainable_weights (after training) =", model.layers[2].trainable_weights[0])
The output is
h1.trainable_weights (before training) = <tf.Variable 'my_shared_variable:0' shape=(1,) dtype=float32, numpy=array([0.3], dtype=float32)>
h2.trainable_weights (before training) = <tf.Variable 'my_shared_variable:0' shape=(1,) dtype=float32, numpy=array([0.3], dtype=float32)>
h1.trainable_weights (after training) = <tf.Variable 'my_shared_variable:0' shape=(1,) dtype=float32, numpy=array([0.7049409], dtype=float32)>
h2.trainable_weights (after training) = <tf.Variable 'my_shared_variable:0' shape=(1,) dtype=float32, numpy=array([0.7049409], dtype=float32)>
I have a problem with creating a Dense model with 3 Layers in which the activation function is cRelu.
cRelu concatenates two relu (a negative and a positive) and creates a tensor twice the size in it's output.
When trying to add another layer after it, I always get a size mismatch error
model = Sequential()
model.add(Dense(N, input_dim=K, activation=crelu))
model.add(Dense(N//2, activation=crelu))
How do I tell the next layer to expect a 2N input and to N?
Keras doesn't expect the activation function to change the output shape. If you want to change it, you should wrap the crelu functionality in a layer and specify the corresponding output shape:
import tensorflow as tf
from keras.layers import Layer
class cRelu(Layer):
def __init__(self, **kwargs):
super(cRelu, self).__init__(**kwargs)
def build(self, input_shape):
super(cRelu, self).build(input_shape)
def call(self, x):
return tf.nn.crelu(x)
def compute_output_shape(self, input_shape):
"""
All axis of output_shape, except the last one,
coincide with the input shape.
The last one is twice the size of the corresponding input
as it's the axis along which the two relu get concatenated.
"""
return (*input_shape[:-1], input_shape[-1]*2)
Then you can use it as follows
model = Sequential()
model.add(Dense(N, input_dim=K))
model.add(cRelu())
model.add(Dense(N//2))
model.add(cRelu())
I'm new to Keras
my neural network structure is here:
neural network structure
my idea is :
import keras.backend as KBack
import tensorflow as tf
#...some code here
model = Sequential()
hidden_units = 4
layer1 = Dense(
hidden_units,
input_dim=len(InputIndex),
activation='sigmoid'
)
model.add(layer1)
# layer1_bias = layer1.get_weights()[1][0]
layer2 = Dense(
1, activation='sigmoid',
use_bias=False
)
model.add(layer2)
# KBack.bias_add(model.output, layer1_bias[0])
I know this is not working cause layer1_bias[0] is not tensor, but I have no idea how to fix it. Or somebody has other solution.
Thanks.
You get the error because bias_add expects a Tensor and you are passing it a float (the actual value of the bias). Also, be aware that your hidden layer actually has 3 biases (one for each node). If you want to add the bias of the first node to your output layer, this should work:
import keras.backend as K
from keras.layers import Dense, Activation
from keras.models import Sequential
model = Sequential()
layer1 = Dense(3, input_dim=2, activation='sigmoid')
layer2 = Dense(1, activation=None, use_bias=False)
activation = Activation('sigmoid')
model.add(layer1)
model.add(layer2)
K.bias_add(model.output, layer1.bias[0:1]) # slice like this to not lose a dimension
model.add(activation)
print(model.summary())
Note that, to be 'correct' (according to the definition of what a dense layer does), you should add the bias first, then the activation.
Also, your code is not really in line with the picture of your network. In the picture, one single shared bias is added to each of the nodes in the network. You can do this with the functional API. The idea is to disable the use of biases in the hidden layer and the output layers, and to manually add a bias variable that you define yourself and that will be shared by the layers. I'm using tensorflow for tf.add() since that supports broadcasting:
from keras.layers import Dense, Lambda, Input, Add
from keras.models import Model
import keras.backend as K
import tensorflow as tf
# Define the shared bias as a custom keras variable
shared_bias = K.variable(value=[0], name='shared_bias')
input_layer = Input(shape=(2,))
# Disable biases in the hidden layer
dense_1 = Dense(units=3, use_bias=False, activation=None)(input_layer)
# Manually add the shared bias
dense_1 = Lambda(lambda x: tf.add(x, shared_bias))(dense_1)
# Disable bias in output layer
output_layer = Dense(units=1, use_bias=False)(dense_1)
# Manually add the bias variable
output_layer = Lambda(lambda x: tf.add(x, shared_bias))(output_layer)
model = Model(inputs=input_layer, outputs=output_layer)
print(model.summary())
This assumes that your shared bias is not trainable though.
I am trying to introduce sparsity in the training samples. My data matrix has a size of (say) NxP and I want to pass it through a layer (keras layer) which has weights of size same as the input size. That is trainable weight matrix W has a shape of NxP. I want to do an hadamard product (element-wise multiplication) of Input matrix to this layer. W multiplied element-wise with input. How to get a trainable layer for W in this case ?
EDIT:
By the way, thank you so much for the quick reply. However, the hadamard product I want to do is between two matrices, one is the input, lets call it X and my X is shape of NxP. And I want my kernel in the hadamard layer to be the same size as X. So kernel should have a size of NxP too. And element wise multiplication of two matrices is achived by the call function.
But the current implementation gives the kernel size as P only. Also,I tried changing the shape of the kernel in the build as follows:
self.kernel = self.add_weight(name='kernel',
shape=input_shape,
initializer='uniform',
trainable=True)
But it gives me the error below:
TypeError: Failed to convert object of type to Tensor. Contents: (None, 16). Consider casting elements to a supported type.
Here P is 16 and I will get my N during the runtime and N is similar to the number of training samples.
Thank you in advance for the help.
Take the example of the documentation to create a layer, and in the call function just define it to be x * self.kernel.
This is my POC:
from keras import backend as K
from keras.engine.topology import Layer
from keras.models import Sequential
from keras.layers import Dense, Activation
import numpy as np
np.random.seed(7)
class Hadamard(Layer):
def __init__(self, **kwargs):
super(Hadamard, self).__init__(**kwargs)
def build(self, input_shape):
# Create a trainable weight variable for this layer.
self.kernel = self.add_weight(name='kernel',
shape=(1,) + input_shape[1:],
initializer='uniform',
trainable=True)
super(Hadamard, self).build(input_shape) # Be sure to call this somewhere!
def call(self, x):
print(x.shape, self.kernel.shape)
return x * self.kernel
def compute_output_shape(self, input_shape):
print(input_shape)
return input_shape
N = 10
P = 64
model = Sequential()
model.add(Dense(128, input_shape=(N, P), activation='relu'))
model.add(Dense(64))
model.add(Hadamard())
model.add(Activation('relu'))
model.add(Dense(32))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
print(model.summary())
model.fit(np.ones((10, N, P)), np.ones((10, N, 1)))
print(model.predict(np.ones((20, N, P))))
If you need to use it as the first layer you should include the input shape parameter:
N = 10
P = 64
model = Sequential()
model.add(Hadamard(input_shape=(N, P)))
model.compile(loss='mean_squared_error', optimizer='adam')
print(model.summary())
This results in:
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
hadamard_1 (Hadamard) (None, 10, 64) 640
=================================================================
Total params: 640
Trainable params: 640
Non-trainable params: 0