I have implemented an autoencoder in Pytorch and wish to extract the representations (output) from a specified encoding layer. This setup is similar to making predictions using sub-models that we used to have in Keras.
However, implementing something similar in Pytorch looks a bit challenging. I tried forward hooks as explained in How to get the output from a specific layer from a PyTorch model? and https://pytorch.org/tutorials/beginner/former_torchies/nnft_tutorial.html but to no avail.
Could you help me getting outputs from a specific layer?
I have attached my code below:
class Autoencoder(torch.nn.Module):
# Now defining the encoding and decoding layers.
def __init__(self):
super().__init__()
self.enc1 = torch.nn.Linear(in_features = 784, out_features = 256)
self.enc2 = torch.nn.Linear(in_features = 256, out_features = 128)
self.enc3 = torch.nn.Linear(in_features = 128, out_features = 64)
self.enc4 = torch.nn.Linear(in_features = 64, out_features = 32)
self.enc5 = torch.nn.Linear(in_features = 32, out_features = 16)
self.dec1 = torch.nn.Linear(in_features = 16, out_features = 32)
self.dec2 = torch.nn.Linear(in_features = 32, out_features = 64)
self.dec3 = torch.nn.Linear(in_features = 64, out_features = 128)
self.dec4 = torch.nn.Linear(in_features = 128, out_features = 256)
self.dec5 = torch.nn.Linear(in_features = 256, out_features = 784)
# Now defining the forward propagation step
def forward(self,x):
x = F.relu(self.enc1(x))
x = F.relu(self.enc2(x))
x = F.relu(self.enc3(x))
x = F.relu(self.enc4(x))
x = F.relu(self.enc5(x))
x = F.relu(self.dec1(x))
x = F.relu(self.dec2(x))
x = F.relu(self.dec3(x))
x = F.relu(self.dec4(x))
x = F.relu(self.dec5(x))
return x
autoencoder_network = Autoencoder()
I have to take the output from encoder layers marked enc1, enc2 .., enc5.
The simplest way is to explicitly return the activations you need:
def forward(self,x):
e1 = F.relu(self.enc1(x))
e2 = F.relu(self.enc2(e1))
e3 = F.relu(self.enc3(e2))
e4 = F.relu(self.enc4(e3))
e5 = F.relu(self.enc5(e4))
x = F.relu(self.dec1(e5))
x = F.relu(self.dec2(x))
x = F.relu(self.dec3(x))
x = F.relu(self.dec4(x))
x = F.relu(self.dec5(x))
return x, e1, e2, e3, e4, e5
You can define a global dictionary, like activations = {}, then in the forward function just assign values to it, like activations['enc1'] = x.clone().detach() and so on.
Related
i am building a federated learning model, i would need to json encode my model so that i can exchange it with new workers i add to the network.
could you help me?
is it possible to encode a model in a json file so that it can be sent?
below i put my net class i tried with the class methods but of course i get the error:
object not serialisable in json.
thanks for your help
class net(nn.Module):
def __init__(self):
super( net, self).__init__()
self.c1 = nn.Conv2d(3, 32, 3)
self.b1 = nn.BatchNorm2d(32)
self.c2 = nn.Conv2d(32, 64, 3)
self.p1 = nn.MaxPool2d(2, 2)
self.c3 = nn.Conv2d(64, 128, 3)
self.d1 = nn.Dropout(0.5)
self.c4 = nn.Conv2d(128, 256, 5)
self.p2 = nn.MaxPool2d(2, 2)
self.c5 = nn.Conv2d(256, 128, 3)
self.p3 = nn.MaxPool2d(2, 2)
self.d2 = nn.Dropout(0.2)
self.l1 = nn.Linear(29*29*128, 128)
self.l2 = nn.Linear(128, 32)
self.l3 = nn.Linear(32 , 10)
self.weights_initialization()
def forward(self, x):
x = F.relu(self.c1(x))
x = self.b1(x)
x = F.relu(self.c2(x))
x = self.p1(x)
x = F.relu(self.c3(x))
x = self.d1(x)
x = F.relu(self.c4(x))
x = self.p2(x)
x = F.relu(self.c5(x))
x = self.p3(x)
x = self.d2(x)
x = x.view(x.size(0), -1)
x = F.relu(self.l1(x))
x = self.d2(x)
x = F.relu(self.l2(x))
x = self.l3(x)
def forward(self, x):
out = F.relu(self.fc1(x))
out = F.relu(self.fc2(out))
return self.output(out)
def weights_initialization(self):
for m in self.modules():
if isinstance(m, nn.Linear):
nn.init.xavier_normal_(m.weight)
nn.init.constant_(m.bias, 0)
model = net()
I have tried the classical methods but have not succeeded in any way in doing this encoding. I currently have no idea how to do this, can you help me?
I am trying to design densenet using model-subclass method. In which I created one block of 5 different layers which is repeated (using for loop) as per user's input. problem is after 1st iteration when output is given to first layer of block. it is giving error. Please help me with how to input give to input to the block
Densenet code is here:
class CNN(keras.Model):
def __init__(self,nfilters,sfilters):
super(CNN,self).__init__()
self.num_filters = nfilters[0]
self.dropout_rate = dropout_rate
self.eps = eps
self.num_blocks = num_blocks
#conv1
self.conv1 = tf.keras.layers.Conv2D(self.num_filters, kernel_size=(sfilters[0],sfilters[0]), use_bias=False, kernel_initializer='he_normal', kernel_regularizer=tf.keras.regularizers.l2(1e-4))
#H_block
self.h_bn = tf.keras.layers.BatchNormalization(epsilon=self.eps)
self.h_act = tf.keras.layers.Activation('relu')
self.h_zp = tf.keras.layers.ZeroPadding2D((1,1))
self.h_do = tf.keras.layers.Dropout(rate=self.dropout_rate)
self.concat = tf.keras.layers.Concatenate()
#trans_block
self.compression_factor = compress_factor
# compression_factor is the 'θ'
self.tran_bn = tf.keras.layers.BatchNormalization( epsilon=eps )
self.tran_act = tf.keras.layers.Activation('relu')
#self.num_feature_maps = input.shape[1]
#The value of 'm'
self.tran_do = tf.keras.layers.Dropout(rate=self.dropout_rate)
self.tran_avgp = tf.keras.layers.AveragePooling2D(pool_size=(3,3))
self.num_layers = num_layers_per_block
self.growth_rate = growth_rate
self.globalaverage = keras.layers.GlobalAveragePooling2D()
self.dense = keras.layers.Dense(37) # Num Classes for CIFAR-10
self.activation = keras.layers.Activation( 'softmax' )
def call(self, inputs, training=False):
inputs = tf.keras.layers.Input(shape=input_shape)
x = self.conv1(inputs)
for i in range( self.num_blocks ):
#x, num_filters = dense_block( x, num_layers_per_block , num_filters, growth_rate , dropout_rate )
for i in range(self.num_layers): # num_layers is the value of 'l'
#H_block
x = self.h_bn(x)
x = self.h_act(x)
x = self.h_zp(x)
h_conv2d = tf.keras.layers.Conv2D(self.num_filters, kernel_size=(sfilters[0], sfilters[0]), use_bias=False , kernel_initializer='he_normal')
x = h_conv2d(x)
#x = self.h_conv2d(x)
x = self.h_do(x)
#inputs = tf.keras.layers.Concatenate()([conv_outputs, inputs])
x = tf.concat([x,self.conv1(inputs)],-1)
#x = self.concat(x,inputs)
#transititon
# compression_factor is the 'θ'
x = self.tran_bn(x)
x = self.tran_act(x)
#num_feature_maps = input.shape[1]
self.num_feature_maps = x.shape[1] # The value of 'm'
tran_conv2d = tf.keras.layers.Conv2D(np.floor(self.compression_factor*self.num_feature_maps).astype(np.int),kernel_size=(1,1), use_bias=False, padding='same', kernel_initializer='he_normal', kernel_regularizer=tf.keras.regularizers.l2(1e-4))
x = tran_conv2d(x)
x = self.tran_do(x)
x = self.tran_avgp(x)
#num_filters += growth_rate # To increase the number of filters for each layer.
self.num_filters += self.growth_rate # To increase the number of filters for each layer.
x = self.globalaverage(x)
x = self.dense(x) # Num Classes for CIFAR-10
x = self.activation(x)
Error is here:
ValueError: Exception encountered when calling layer "cnn_3" (type CNN).
in user code:
File "<ipython-input-6-86c881729324>", line 40, in call *
x = self.h_bn(x)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 67, in error_handler **
raise e.with_traceback(filtered_tb) from None
ValueError: Exception encountered when calling layer "batch_normalization_6" (type BatchNormalization).
Dimensions must be equal, but are 16 and 8 for '{{node batch_normalization_6/FusedBatchNormV3}} = FusedBatchNormV3[T=DT_FLOAT, U=DT_FLOAT, data_format="NHWC", epsilon=1.1e-05, exponential_avg_factor=0.01, is_training=true](Placeholder, batch_normalization_6/ReadVariableOp, batch_normalization_6/ReadVariableOp_1, batch_normalization_6/FusedBatchNormV3/ReadVariableOp, batch_normalization_6/FusedBatchNormV3/ReadVariableOp_1)' with input shapes: [?,126,126,16], [8], [8], [8], [8].
Call arguments received:
• inputs=tf.Tensor(shape=(None, 126, 126, 16), dtype=float32)
• training=True
Call arguments received:
• inputs=tf.Tensor(shape=(None, 128, 128, 1), dtype=float32)
• training=True
I'm trying to do attention mechanism while returning the tensor I'm getting the following error
ValueError: Shape mismatch: The shape of labels (received (64, 53)) should equal the shape of logits except for the last dimension (received (64, 1, 500)).
Please find the below code
Here is code for attention please correct me if it is wrong
class Attention(tf.keras.layers.Layer):
def __init__(self):
super().__init__()
def call(self,enc_op,hidden_state):
# print(enc_op.shape,hidden_state.shape)
query_with_time_axis = tf.expand_dims(hidden_state, 1)
context_vector = tf.matmul(enc_op,tf.transpose(query_with_time_axis,perm=[0,2,1]))
context_vector = tf.nn.softmax(context_vector,axis=1)
context_vector = context_vector * enc_op
context_vector = tf.reduce_sum(context_vector, axis=1)
return context_vector
Here is decoder part I'm calling the attention from here
class Decoder(tf.keras.layers.Layer):
def init(self,vocab_size,embedding_dim,input_length,dec_units):
super().init()
self.vocab_size = vocab_size
self.embedding_dim = embedding_dim
self.dec_units = dec_units
self.input_length = input_length
self.attention = Attention()
def build(self,input_shape):
self.embedding = Embedding(input_dim=self.vocab_size,output_dim = self.embedding_dim,input_shape = input_shape,
mask_zero = True, name = "embedding_layer_decoder")
self.lstm = LSTM(self.dec_units,return_sequences=True,return_state=True,name = "Decoder_LSTM")
def call(self,target_sentances,enc_op,hidden_state,cell_state):
target_embed = self.embedding(target_sentances)
for i in range(target_embed.shape[1]):
context_vector = self.attention(enc_op,hidden_state)
y = tf.concat([context_vector, target_embed[:,i,:]], axis=-1)
y = tf.expand_dims(y, 1)
lstm_output,hidden_state,_ = self.lstm(y,initial_state = [hidden_state,cell_state])
return lstm_output
class Mymodel(Model):
def __init__(self,encoder_inputs_length,decoder_inputs_length,output_vocab_size):
super().__init__()
self.encoder = Encoder(vocab_size = 500, embedding_dim = 50, input_length = encoder_inputs_length, enc_units=64)
self.decoder = Decoder(vocab_size = 500, embedding_dim = 50, input_length = decoder_inputs_length, dec_units=64)
self.dense = Dense(output_vocab_size,activation = "softmax")
def call(self,data):
input,output = data[0],data[1]
print(input.shape,output.shape)
encoder_output,encoder_h,encoder_c = self.encoder(input)
print("="*20, "ENCODER", "="*20)
print("-"*35)
print(encoder_output)
print("ENCODER ==> OUTPUT SHAPE",encoder_output.shape)
print("ENCODER ==> HIDDEN STATE SHAPE",encoder_h.shape)
print("ENCODER ==> CELL STATE SHAPE", encoder_c.shape)
print("="*20,"Decoder","="*20)
decoder_output = self.decoder(output,encoder_output,encoder_h,encoder_c)
output1 = self.dense(decoder_output)
print("-"*35)
print("Final output shape",output.shape)
print("="*50)
return output1
model = Mymodel(encoder_inputs_length=30,decoder_inputs_length=20,output_vocab_size=500)
ENCODER_SEQ_LEN = 30
DECODER_SEQ_LEN = 20
optimizer = tf.keras.optimizers.Adam()
model.compile(optimizer=optimizer,loss=tf.keras.losses.SparseCategoricalCrossentropy())
for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):
model.fit([inp, targ], targ, steps_per_epoch=1)
The shape of my input and target is
(64, 55) (64, 53)
64 is batch size
I cannot reproduce the same results after loading a model using pytorch.
I am training a model 'net' and in the same file, after training (kfold) then the model is saved and also tested in 1 specific testing file:
class model(nn.Module):
def __init__(self,size_net):
print('Initialize net with size: ',size_net)
self.T = size_net
# Layer 1
self.conv1 = nn.Conv2d(1, 16, (1,16), padding = 0)
self.batchnorm1 = nn.BatchNorm2d(16, False)
# Layer 2
self.padding1 = nn.ZeroPad2d((16, 17, 0, 1))
self.conv2 = nn.Conv2d(1, 4, (2, 32))
self.batchnorm2 = nn.BatchNorm2d(4, False)
self.pooling2 = nn.MaxPool2d(2, 4)
# Layer 3
self.padding2 = nn.ZeroPad2d((2, 1, 4, 3))
self.conv3 = nn.Conv2d(4, 4, (8, 4))
self.batchnorm3 = nn.BatchNorm2d(4, False)
self.pooling3 = nn.MaxPool2d((2, 4))
# FC Layer
# NOTE: This dimension will depend on the number of timestamps per sample in your data.
# I have 120 timepoints.
self.fc1 = nn.Linear(int(self.T/2), 2)
def forward(self, x):
# Layer 1
x = F.elu(self.conv1(x))
x = self.batchnorm1(x)
x = F.dropout(x, 0.25)
x = x.permute(0, 3, 1, 2)
#print "layer 1"
# Layer 2
x = self.padding1(x)
x = F.elu(self.conv2(x))
x = self.batchnorm2(x)
x = F.dropout(x, 0.25)
x = self.pooling2(x)
#print "layer 2"
# Layer 3
x = self.padding2(x)
x = F.elu(self.conv3(x))
x = self.batchnorm3(x)
x = F.dropout(x, 0.25)
x = self.pooling3(x)
#print "layer 3"
# FC Layer
#print ('view:',x.shape)
x = x.view(-1, int(self.T/2))
#x = torch.sigmoid(self.fc1(x))
x= torch.softmax(self.fc1(x),1)
#print "layer 4"
return x
#now call the model and train
net = model(SIZE_NET)
....
eval.train_Kfold_validation(n_epochs=25)
## save models state
"""
net = EEGNet(SIZE_NET)
save_path = './eeg_net_{}.pt'.format(date.today().strftime("%Y%m%d"))
torch.save(net.state_dict(), save_path)
'''
TEST
'''
testfile = '1_testonline_1_20190202-163051.csv'
kun_1 = np.genfromtxt( '../'+ testfile, delimiter=',').astype('float32')[:-1, :]
kun_1 = kun_1[:, :SIZE_NET]
X, y = prep.list_2darrays_to_3d([kun_1], -1)
print(X.shape)
array_dstack = np.array(X)
array_dstack_reshaped = np.reshape(array_dstack,(1, 1, SIZE_NET, 16))
inputs = Variable(torch.from_numpy(array_dstack_reshaped))
pred = net(inputs)
print('prob: '+str(pred)) #Converted to probabilities
For example for this file I got: pred=tensor([[0.5912, 0.4088]], grad_fn=)
When instead I load the saved model in a new script and I attempt inference again on the same testfile:
prep= Data_prep()
fileName = '1_testonline_1_20190202-163051.csv'
kun_1 = np.genfromtxt(file_dir+fileName, delimiter=',').astype('float32')[:-1,:]
kun_1 = kun_1[:,:SIZE_NET]
X , y = prep.list_2darrays_to_3d([kun_1],[-1])
# Load pre-trained model
net = model(SIZE_NET)
load_path = file_dir+'/model_colors/model_20190205.pt'
net.load_state_dict(torch.load(load_path))
net.eval()
array_dstack = np.array(X)
print(X.shape)
# (#samples, 1, #timepoints, #channels)
array_dstack_reshaped = np.reshape(array_dstack,(1, 1, SIZE_NET, 16))
inputs = Variable(torch.from_numpy(array_dstack_reshaped))
pred = net(inputs)
print(pred)
When I run the test script the prob values are different and even worse NOT stable: running multiple times give different predictions... Any help appreciated
As #Jatentaki pointed out the solution is to ALWAYS fix the seed in all scripts that need to use the model in pytorch
torch.manual_seed(0)
I am trying to build my first NN with pytroch and got an issue.
TypeError: new() received an invalid combination of arguments - got (float, int, int, int), but expected one of:
* (torch.device device)
* (torch.Storage storage)
* (Tensor other)
* (tuple of ints size, torch.device device)
* (object data, torch.device device)
Now I know what this is saying in that I am not passing the right type to the method or init. But I dont know what I should pass as it looks right to me.
def main():
#Get the time and data
now = datetime.datetime.now()
hourGlassToStack = 2 #Hourglasses to stack
numModules= 2 #Residual Modules for each hourglass
numFeats = 256 #Number of features in each hourglass
numRegModules = 2 #Depth regression modules
print("Creating Model")
model = HourglassNet3D(hourGlassToStack, numModules, numFeats,numRegModules).cuda()
print("Model Created")
this is the main method that created the model.
It then calls this methods.
class HourglassNet3D(nn.Module):
def __init__(self, nStack, nModules, nFeats, nRegModules):
super(HourglassNet3D, self).__init__()
self.nStack = nStack
self.nModules = nModules
self.nFeats = nFeats
self.nRegModules = nRegModules
self.conv1_ = nn.Conv2d(3, 64, bias = True, kernel_size = 7, stride = 2, padding = 3)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace = True)
self.r1 = Residual(64, 128)
self.maxpool = nn.MaxPool2d(kernel_size = 2, stride = 2)
self.r4 = Residual(128, 128)
self.r5 = Residual(128, self.nFeats)
_hourglass, _Residual, _lin_, _tmpOut, _ll_, _tmpOut_, _reg_ = [], [], [], [], [], [], []
for i in range(self.nStack):
_hourglass.append(Hourglass(4, self.nModules, self.nFeats))
for j in range(self.nModules):
_Residual.append(Residual(self.nFeats, self.nFeats))
lin = nn.Sequential(nn.Conv2d(self.nFeats, self.nFeats, bias = True, kernel_size = 1, stride = 1),
nn.BatchNorm2d(self.nFeats), self.relu)
_lin_.append(lin)
_tmpOut.append(nn.Conv2d(self.nFeats, 16, bias = True, kernel_size = 1, stride = 1))
_ll_.append(nn.Conv2d(self.nFeats, self.nFeats, bias = True, kernel_size = 1, stride = 1))
_tmpOut_.append(nn.Conv2d(16, self.nFeats, bias = True, kernel_size = 1, stride = 1))
for i in range(4):
for j in range(self.nRegModules):
_reg_.append(Residual(self.nFeats, self.nFeats))
self.hourglass = nn.ModuleList(_hourglass)
self.Residual = nn.ModuleList(_Residual)
self.lin_ = nn.ModuleList(_lin_)
self.tmpOut = nn.ModuleList(_tmpOut)
self.ll_ = nn.ModuleList(_ll_)
self.tmpOut_ = nn.ModuleList(_tmpOut_)
self.reg_ = nn.ModuleList(_reg_)
self.reg = nn.Linear(4 * 4 * self.nFeats,16 )
And this then call this
class Residual(nn.Module):
#set the number ofinput and output for each layer
def __init__(self, numIn, numOut):
super(Residual, self).__init__()
self.numIn = numIn
self.numOut = numOut
self.bn = nn.BatchNorm2d(self.numIn)
self.relu = nn.ReLU(inplace = True)
self.conv1 = nn.Conv2d(self.numIn, self.numOut / 2, bias = True, kernel_size = 1)
self.bn1 = nn.BatchNorm2d(self.numOut / 2)
self.conv2 = nn.Conv2d(self.numOut / 2, self.numOut / 2, bias = True, kernel_size = 3, stride = 1, padding = 1)
self.bn2 = nn.BatchNorm2d(self.numOut / 2)
self.conv3 = nn.Conv2d(self.numOut / 2, self.numOut, bias = True, kernel_size = 1)
if self.numIn != self.numOut:
self.conv4 = nn.Conv2d(self.numIn, self.numOut, bias = True, kernel_size = 1)
all of this looks fine to me, but I dont know how I am suppose to pass this then if I am doing it wrong.
Thank you for any help
You might have to look out as to what you are passing to your convolutional layers in the Residual class. Per default, Python 3 will convert any division operation into a float variable.
Try casting your variables back to an integer, and see if that helps. Fixed code for Residual:
class Residual(nn.Module):
#set the number ofinput and output for each layer
def __init__(self, numIn, numOut):
super(Residual, self).__init__()
self.numIn = numIn
self.numOut = numOut
self.bn = nn.BatchNorm2d(self.numIn)
self.relu = nn.ReLU(inplace = True)
self.conv1 = nn.Conv2d(self.numIn, int(self.numOut / 2), bias = True, kernel_size = 1)
self.bn1 = nn.BatchNorm2d(int(self.numOut / 2))
self.conv2 = nn.Conv2d(int(self.numOut / 2), int(self.numOut / 2), bias = True, kernel_size = 3, stride = 1, padding = 1)
self.bn2 = nn.BatchNorm2d(int(self.numOut / 2))
self.conv3 = nn.Conv2d(int(self.numOut / 2), self.numOut, bias = True, kernel_size = 1)
if self.numIn != self.numOut:
self.conv4 = nn.Conv2d(self.numIn, self.numOut, bias = True, kernel_size = 1)