torch.nn.Linear(in_features, out_features, bias=True, device=None, dtype=None)
I have a dataset of [914,19] shape. should my in_features be 914? And I want to predict 5 different values so should my output feature be 5?
class NeuralNetwork(nn.Module):
def __init__(self):
super(NeuralNetwork, self).__init__()
self.linear1 = nn.Linear(914,512)
self.linear2 = nn.Linear(512,512)
self.linear3 = nn.Linear(512,512)
self.linear4 = nn.Linear(512,5)
def forward(self, x):
x = F.relu(self.linear1(x))
x = F.relu(self.linear2(x))
x = F.relu(self.linear3(x))
x = self.linear4(x)
return x
NeuralNet = NeuralNetwork()
print(NeuralNet)
Your input data is shaped (914, 19), assuming 914 refers to your batch size here, then the in_features corresponds to 19. This can be read as a tensor containing 914 19-feature-long input vectors.
In this case, the in_features of linear1 would be set to 19.
Related
I am trying to build a GAN which accepts a label to generate new image.
class Generator(nn.Module):
def __init__(self):
super(Generator, self).__init__()
self.fc1 = nn.Linear(100, 256)
self.fc2 = nn.Linear(256, 512)
self.fc3 = nn.Linear(512, 1024)
self.fc4 = nn.Linear(1024, 784)
def forward(self, x):
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
x = F.relu(self.fc3(x))
x = torch.tanh(self.fc4(x))
return x
# set label
label = 3
# create one-hot encoded vector
one_hot = torch.zeros(1, 10)
one_hot[0][label] = 1
# set noise vector
noise = torch.randn(1, 100)
# concatenate label and noise
noise_with_label = torch.cat([one_hot, noise], dim=1)
# pass through generator
generated_image = generator(noise_with_label)
But its throwing :
112
113 def forward(self, input: Tensor) -> Tensor:
--> 114 return F.linear(input, self.weight, self.bias)
115
116 def extra_repr(self) -> str:
RuntimeError: mat1 and mat2 shapes cannot be multiplied (1x110 and 100x256)
I am using MNIST dataset.
I tried to resolve it, but couldn’t find a way to fix it.
Original comment by #jodag helped to fix the issue:
The input you're providing to your model has shape [1,110] because you're concatenating noise (shape [1,100]) and one_hot (shape [1,10]). But the first layer expects inputs of shape [..., 100]. One way to get rid of the error would be to change the first layer of the model to self.fc1 = nn.Linear(110, 256). An alternative would be to define noise as noise = torch.randn(1, 90)
For a pytorch module, I suppose I could use .named_children, .named_modules, etc. to obtain a list of the submodules. However, I suppose the list is not given in order, right? An example:
In [19]: import transformers
In [20]: model = transformers.DistilBertForSequenceClassification.from_pretrained('distilb
...: ert-base-cased')
In [21]: [name for name, _ in model.named_children()]
Out[21]: ['distilbert', 'pre_classifier', 'classifier', 'dropout']
The order of .named_children() in the above model is given as distilbert, pre_classifier, classifier, and dropout. However, if you examine the code, it is evident that dropout happens before classifier. So how do I get the order of these submodules?
In Pytorch, the results of print(model) or .named_children(), etc are listed based on the order they are declared in __init__ of the model's class e.g.
Case 1
class Model(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
self.fc1 = nn.Linear(320, 50)
self.fc2 = nn.Linear(50, 10)
self.conv2_drop = nn.Dropout2d()
def forward(self, x):
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
x = x.view(-1, 320)
x = F.relu(self.fc1(x))
x = F.dropout(x, p=0.6)
x = self.fc2(x)
return F.log_softmax(x, dim=1)
model = Model()
print(model)
[name for name, _ in model.named_children()]
# output
['conv1', 'conv2', 'fc1', 'fc2', 'conv2_drop']
Case 2
Changed order of fc1 and fc2 layers in constructor.
class Model(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
self.fc2 = nn.Linear(50, 10)
self.fc1 = nn.Linear(320, 50)
self.conv2_drop = nn.Dropout2d()
def forward(self, x):
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
x = x.view(-1, 320)
x = F.relu(self.fc1(x))
x = F.dropout(x, p=0.6)
x = self.fc2(x)
return F.log_softmax(x, dim=1)
model = Model()
print(model)
[name for name, _ in model.named_children()]
# output
['conv1', 'conv2', 'fc2', 'fc1', 'conv2_drop']
That's why classifier is printed before dropout as it's declared so in constructor:
class DistilBertForSequenceClassification(DistilBertPreTrainedModel):
...
self.distilbert = DistilBertModel(config)
self.pre_classifier = nn.Linear(config.dim, config.dim)
self.classifier = nn.Linear(config.dim, config.num_labels)
self.dropout = nn.Dropout(config.seq_classif_dropout)
Nevertheless, you can play with model's submodules using .modules(), etc. but they'll be listed only in the order they are declared in __init__. If you only want to print structure based on forward method, you may try using pytorch-summary.
I am struggling with understanding how to get hidden layers and concatenate them.
I am using the following code as an example:
class classifier(nn.Module):
#define all the layers used in model
def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers,
bidirectional, dropout):
#Constructor
super().__init__()
self.batch = BATCH_SIZE
self.hidden = hidden_dim
self.layers = n_layers
if(bidirectional):
self.directions = 2
else:
self.directions = 1
#embedding layer
self.embedding = nn.Embedding(vocab_size, embedding_dim)
#lstm layer
self.gru = nn.GRU(embedding_dim,
hidden_dim,
num_layers=n_layers,
bidirectional=bidirectional,
dropout=dropout,
batch_first=True)
#dense layer
self.fc = nn.Linear(hidden_dim * 2, output_dim)
#activation function
self.act = nn.Sigmoid()
def forward(self, text, text_lengths):
#text = [batch size,sent_length]
embedded = self.embedding(text)
#embedded = [batch size, sent_len, emb dim]
#packed sequence
packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths,batch_first=True)
packed_output, (hidden, cell) = self.lstm(packed_embedded)
#hidden = [batch size, num layers * num directions,hid dim]
#cell = [batch size, num layers * num directions,hid dim]
#concat the final forward and backward hidden state
hidden = torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1)
#hidden = [batch size, hid dim * num directions]
dense_outputs=self.fc(hidden)
#Final activation function
outputs=self.act(dense_outputs)
return outputs
The line hidden = torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1) I did not get it.
As per my understanding, I was doing this which did not work.
hidden2 = hidden.view(batch_size,self.layers,self.directions,self.hidden)
hidden2 = torch.cat((hidden2[:,:,0,:],hidden2[:,:,1,:]),dim=1)
dense_outputs=self.fc(hidden2)
Can somebody please explain. I went through PyTorch documentation but did not get.
The shape[0] of hidden output for bidirectional GRU is 2. You should just concat two hidden output on dim=1:
hid_enc = torch.cat([hid_enc[0,:, :], hid_enc[1,:,:]], dim=1).unsqueeze(0)
As the explanation for usage of -1 and -2 as the index , as you know in python lists, the object in index -1 is the last object of the list(second object in our tensor list) and index -2 refers to the object before last object(first object in our case). So the code you did not understand is equivalent to the code in my answer
I'm using stacked Autoencoder, which is a bunch of Conv layers.
However, I'm having a tensor mismatch error, and I'm not sure about the reason. Everything done in the Encoder is reversed in the Decoder!
This is for time-series data. Input shape is (bactch_size, 1, 3000)
Here's the code
class CDAutoEncoder(nn.Module):
def __init__(self, input_size, output_size, kernel, stride):
super(CDAutoEncoder, self).__init__()
self.forward_pass = nn.Sequential(
nn.Conv1d(input_size, output_size, kernel_size=kernel, stride=stride, padding=0),
nn.PReLU(),
)
self.backward_pass = nn.Sequential(
nn.ConvTranspose1d(output_size, input_size, kernel_size=kernel, stride=stride, padding=0),
nn.PReLU(),
)
def forward(self, x):
y = self.forward_pass(x)
return y
def reconstruct(self, x):
return self.backward_pass(x)
class StackedAutoEncoder(nn.Module):
def __init__(self):
super(StackedAutoEncoder, self).__init__()
self.ae1 = CDAutoEncoder(1, 32, 50, 10)
self.ae2 = CDAutoEncoder(32, 64, 10, 3)
self.ae3 = CDAutoEncoder(64, 64, 5, 1)
def forward(self, x):
a1 = self.ae1(x)
a2 = self.ae2(a1)
a3 = self.ae3(a2)
return self.reconstruct(a3)
def reconstruct(self, x):
a2_reconstruct = self.ae3.reconstruct(x)
a1_reconstruct = self.ae2.reconstruct(a2_reconstruct)
x_reconstruct = self.ae1.reconstruct(a1_reconstruct)
return x_reconstruct
The error:
RuntimeError: The size of tensor a (2990) must match the size of tensor b (3000) at non-singleton dimension 2
I've tried adding padding and it worked, but when I changed the kernel size I get different tensor-size-mismatch-error.
Apparently, there's nothing like 'same' padding, so is there automated solution for this?
I have train dataset of 8000 images and labels. Validation set consists of 1957 images and labels. The test set contains 2487 images. Each image contains White Blood Cell images. WBC is divided innto 4 categories: Eosinophil, Neutrophil, Monocyte and Lymphocyte. Eosinophil and Neutrophil are Polynuclear while the remaining two are Mononuclear. The cells need to be classified between the two classes : Polynuclear and Mononuclear.
# import libraries
def get_data(folder):
X = []
y = []
for wbc_type in os.listdir(folder):
if not wbc_type.startswith('.'):
if wbc_type in ['NEUTROPHIL', 'EOSINOPHIL']:
label = 'POLYNUCLEAR'
else:
label = 'MONONUCLEAR'
for image_filename in tqdm(os.listdir(folder + wbc_type)):
img_file = cv2.imread(folder + wbc_type + '/' + image_filename)
if img_file is not None:
# Downsample the image to 120, 160, 3
img_file = scipy.misc.imresize(arr=img_file, size=(120, 160, 3))
img_arr = np.asarray(img_file)
X.append(img_arr)
y.append(label)
X = np.asarray(X)
y = np.asarray(y)
return X,y
X_train, y_train = get_data('C:/Users/Neerajan/Desktop/blood-cells/dataset2-master/dataset2-master/images/TRAIN/')
X_test, y_test = get_data('C:/Users/Neerajan/Desktop/blood-cells/dataset2-master/dataset2-master/images/TEST/')
encoder = LabelEncoder()
encoder.fit(y_train)
y_train = encoder.transform(y_train)
y_test = encoder.transform(y_test)
X_train=np.array((X_train), dtype = np.float32)
X_train=X_train/255.0
X_test=np.array((X_test), dtype = np.float32)
X_test=X_test/255.0
y_train = y_train.astype(int)
y_train = y_train.flatten()
from chainer.datasets import split_dataset_random
from chainer.dataset import DatasetMixin
class MyDataset(DatasetMixin):
def __init__(self, X, labels):
super(MyDataset, self).__init__()
self.X_ = X
self.labels_ = labels
self.size_ = X.shape[0]
def __len__(self):
return self.size_
def get_example(self, i):
return np.transpose(self.X_[i, ...], (2, 0, 1)), self.labels_[i]
batch_size = 32
dataset = MyDataset(X_train, y_train)
dataset_train, valid = split_dataset_random(dataset, 8000, seed=0)
train_iter = iterators.SerialIterator(dataset_train, batch_size)
valid_iter = iterators.SerialIterator(valid, batch_size, repeat=False, shuffle=False)
from chainer.dataset import concat_examples
batch_image, batch_label = concat_examples(next(train_iter))
print("batch_image.shape\n{}".format(batch_image.shape))
print("batch_label.shape\n{}".format(batch_label.shape))
batch_image.shape : (32,3,120,160) batch_label.shape : (32,)
class MyModel(chainer.Chain):
def __init__(self, n_out):
super(MyModel, self).__init__()
with self.init_scope():
self.conv1=L.Convolution2D(None, 32, 3, 3, 1)
self.conv2=L.Convolution2D(32, 64, 3, 3, 1)
self.conv3=L.Convolution2D(64, 128, 3, 3, 1)
self.fc4=L.Linear(None, 32)
self.fc5=L.Linear(32, n_out)
def __call__(self, x):
h = F.relu(self.conv1(x))
h = F.relu(self.conv2(h))
h = F.relu(self.conv3(h))
h = F.leaky_relu(self.fc4(h))
h = F.sigmoid(self.fc5(h))
return h
from chainer import training
def train(model_object, batchsize=32, gpu_id=-1, max_epoch=14):
model = L.Classifier(model_object)
if gpu_id >=0:
model.to_gpu(gpu_id)
#serializers.save_npz('kankata',model)
# 4. Optimizer
optimizer = optimizers.Adam()
optimizer.setup(model)
serializers.save_npz('my.state',optimizer)
# 5. Updater
updater = training.StandardUpdater(train_iter, optimizer, device=gpu_id)
# 6. Trainer
trainer = training.Trainer(updater, (max_epoch, 'epoch'), out='C:/Users/Neerajan/Desktop/ReportDump'.format(model_object.__class__.__name__))
# 7. Evaluator
class TestModeEvaluator(extensions.Evaluator):
def evaluate(self):
model = self.get_target('main')
ret = super(TestModeEvaluator, self).evaluate()
return ret
trainer.extend(extensions.LogReport())
trainer.extend(TestModeEvaluator(valid_iter, model, device=gpu_id))
trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'main/accuracy', 'validation/main/loss', 'validation/main/accuracy', 'elapsed_time']))
trainer.extend(extensions.PlotReport(['main/loss', 'validation/main/loss'], x_key='epoch', file_name='loss.png'))
trainer.extend(extensions.PlotReport(['main/accuracy', 'validation/main/accuracy'], x_key='epoch', file_name='accuracy.png'))
trainer.run()
del trainer
return model
gpu_id = -1 # Set to -1 if you don't have a GPU
model = train(MyModel(2), gpu_id=gpu_id)
It is recommended that for binary classification we use sigmoid activation function in the last layer of model and binary_cross_entropy in classifier.
How do I implement binary_cross_entropy as the loss_function in the classifier?
see this example for binary classification.
43 model = L.Classifier(
44 MLP(44, 1), lossfun=F.sigmoid_cross_entropy, accfun=F.binary_accuracy)
feeding lossfun=F.sigmoid_cross_entropy to L.Classifier is a good solution.