I have been following the textbook, but there is an error in the imported contents and I don't know what to do.
this is the Net class
import torch
import torch.nn as nn
class Net(nn.Module):
def __init__(self, n_input, n_output):
super().__init__()
self.l1 = nn.Linear(n_input,n_output)
self.sigmoid = nn.Sigmoid()
self.l1.weight.data.fill_(1.0)
self.l1.bias.data.fill_(1.0)
def forward(self,x):
x1 = self.l1(x)
x2 = self.Sigmoid(x1)
return x2
and the optimizer and loss functions and data
from torch import optim
criterion = nn.BCELoss()
lr = 0.01
optimizer = optim.SGD(net.parameters(),lr = lr)
I have been following the textbook, but there is an error in the imported contents and I don't know what to do.
Related
I'm trying to one time run my model and see if it's working. I've searched the error and suggested answers was to instantiate the model once. I already did that. However, even after instantiating the model. it gives an error once requesting access to model parametes. What's the problem?
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, TensorDataset
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from IPython import display
display.set_matplotlib_formats('svg')
data = np.array([[1,1]])
label = np.array([2])
for i in range(-50,51,1):
data = np.append(data, [[i,i]], axis=0)
label = np.append(label, [i+i])
# conver to tensor
T_data = torch.tensor(data).float()
T_label = torch.tensor(label).long()
# split data
train_data, test_data, train_label, test_label = train_test_split(T_data, T_label, test_size= .2)
# convert into Pytorch dataset
train_data = TensorDataset(train_data, train_label)
test_data = TensorDataset(test_data, test_label)
# translate into dataloader
batchsize = 32
train_loader = DataLoader(train_data, batch_size= batchsize, shuffle =True, drop_last=True)
test_loader = DataLoader(test_data, batch_size=test_data.tensors[0].shape[0])
class AddNN(nn.Module):
def __init__(self):
super().__init__()
self.input = nn.Linear(2,16)
## hidden layer
self.fc1 = nn.Linear(16,32)
self.fc2 = nn.Linear(32,1)
#output layer
self.output = nn.Linear(1,1)
# forward pass
def forward(self, x):
x = F.relu(self.input(x))
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
return self.output(x)
net =AddNN()
lossfun = nn.MSELoss()
optimizer = torch.optim.Adam(AddNN.parameters(), lr = .05)
It returns TypeError: parameters() missing 1 required positional argument: 'self'
What it means to have missing a required positional argument here??
After model instantiation, you should use that model object instead of the model class inside Adam and with .parameters()
net = AddNN()
lossfun = nn.MSELoss()
optimizer = torch.optim.Adam(net.parameters(), lr = .05)
I'm trying to use a gradient method to find a minimizer of a function f, where x(k+1)=x(k)-α▽f(x), and α=0.1. The function f is f(x) = 2*(x^3+1)^(-1/2), x(0)=1
Here is the pytorch sample
import torch
import torch.nn as nn
from torch import optim
alpha = 0.1
class MyModel(nn.Module):
def __init__(self):
super().__init__()
self.x = 1
def forward(self, x):
return 2*pow((pow(x,3)+1),(-1/2))
model = MyModel()
optimizer = optim.SGD(model.parameters(),lr=alpha)
terminationCond = False
while not terminationCond:
f = model.forward(x)
f.backward()
optimizer.step()
if x.grad < 0.001:
terminationCond = True
optimizer.zero_grad()
But I cannot output the correct value of x, how to modify my code in order to find a minizer of a function f?
There are a few things that need to be considered ...
For x to be a parameter (model.parameter()) x should be nn.Parameter(torch.as_tensor([1.]))
You are passing what (and why) x to the forward method?
The function f(x) = 2*(x^3+1)^(-1/2) is inversely proportional to x. When x goes up in value, the function goes down. The SGD will minimize the value of the function, hence maximize x.
f(0) = 2.0
f(1) = 1.41421
f(2) = 0.66666
Here is a working example of minimizing f(x) ie maximizing x.
import torch
import torch.nn as nn
from torch import optim
alpha = 0.1
class MyModel(nn.Module):
def __init__(self):
super().__init__()
self.x = nn.Parameter(torch.as_tensor([1.0]))
def forward(self):
return 2*pow((pow(self.x, 3)+1), (-1/2))
model = MyModel()
optimizer = optim.SGD(model.parameters(), lr=alpha)
terminationCond = False
print(model.x)
f = model()
f.backward()
optimizer.step()
optimizer.zero_grad()
print(model.x)
In keras, we can use a Lambda layer to create a custom layer, like this:
def f(x):
return x**2
model.add(Lambda(f))
Now my question is, how to make such custom function trainable? How to make this function such that it raises an input to the power w, where w is trainable. Like this:
def f(x):
return x**w
The problem can be solved by making a new layer via subclassing,
import tensorflow as tf
from tensorflow.keras.layers import *
from tensorflow.keras.models import *
from tensorflow.keras.optimizers import *
import numpy as np
class ScaleLayer(tf.keras.layers.Layer):
def __init__(self):
super(ScaleLayer, self).__init__()
self.scale = tf.Variable(1., trainable=True)
def call(self, inputs):
return inputs ** self.scale
x = np.array([1,2,3,4,5,6,7,8,9,10,11,12,13]).reshape(-1,1)
y = x**3.25
l = ScaleLayer()
a1 = tf.keras.layers.Input(shape=1)
a2 = l(a1)
model = tf.keras.models.Model(a1,a2)
model.compile(optimizer=Adam(learning_rate=0.01), loss='mse')
model.fit(x,y, epochs=500, verbose=0)
print(l.weights) # This prints 3.25
More about this can be found here
Is there a way to get the train/validation loss from a Skorch fit in e.g a list (if you want to do some plotting, statistics)?
You can use the history (which allows slicing over time) to get this information. For example:
train_loss = net.history[:, 'train_loss']
which returns the train_loss for each recorded epoch.
Here is an example based on the following.
import numpy as np
import torch
import torch.nn.functional as F
from matplotlib import pyplot as plt
from sklearn.datasets import make_classification
from skorch import NeuralNetClassifier
from torch import nn
torch.manual_seed(0)
class ClassifierModule(nn.Module):
def __init__(
self,
num_units=10,
nonlin=F.relu,
dropout=0.5,
):
super(ClassifierModule, self).__init__()
self.num_units = num_units
self.nonlin = nonlin
self.dropout = dropout
self.dense0 = nn.Linear(20, num_units)
self.nonlin = nonlin
self.dropout = nn.Dropout(dropout)
self.dense1 = nn.Linear(num_units, 10)
self.output = nn.Linear(10, 2)
def forward(self, X, **kwargs):
X = self.nonlin(self.dense0(X))
X = self.dropout(X)
X = F.relu(self.dense1(X))
X = F.softmax(self.output(X), dim=-1)
return X
net = NeuralNetClassifier(
ClassifierModule,
max_epochs=20,
lr=0.1,
# device='cuda', # uncomment this to train with CUDA
)
X, y = make_classification(1000, 20, n_informative=10, random_state=0)
X, y = X.astype(np.float32), y.astype(np.int64)
net.fit(X, y)
train_loss = net.history[:, 'train_loss']
valid_loss = net.history[:, 'valid_loss']
plt.plot(train_loss, 'o-', label='training')
plt.plot(valid_loss, 'o-', label='validation')
plt.legend()
plt.show()
And results:
I have the following trimmed down model:
import torch.nn as nn
import torch
import argparse
import torch
import torch.utils.data
from torch import nn, optim
from torch.autograd import Variable
from torch.nn import functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
import json
import numpy as np
import datetime
import os
class EncoderRNN(nn.Module):
def __init__(self, input_size=8, hidden_size=10, num_layers=2):
super(EncoderRNN, self).__init__()
self.input_size = input_size
self.hidden_size = hidden_size
self.num_layers = num_layers
self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
self.relu = nn.ReLU()
self.sigmoid = nn.Sigmoid()
#initialize weights
nn.init.xavier_uniform(self.lstm.weight_ih_l0, gain=np.sqrt(2))
nn.init.xavier_uniform(self.lstm.weight_hh_l0, gain=np.sqrt(2))
def forward(self, input):
tt = torch
print input.shape
h0 = Variable(tt.FloatTensor(self.num_layers, input.size(0), self.hidden_size))
c0 = Variable(tt.FloatTensor(self.num_layers, input.size(0), self.hidden_size))
encoded_input, hidden = self.lstm(input, (h0, c0))
encoded_input = self.sigmoid(encoded_input)
return encoded_input
train_x = torch.from_numpy(np.random.random((2000,19,8))).float()
train_loader = torch.utils.data.DataLoader(train_x,
batch_size=64, shuffle=True)
model = EncoderRNN()
optimizer = optim.Adam(model.parameters(), lr=1e-6)
optimizer.zero_grad()
loss_function = torch.nn.BCELoss(reduce=True)
def train(epoch):
model.train()
train_loss = 0
for batch_idx, (data_x) in enumerate(train_loader):
x = model(Variable(data_x))
print("output has nan: " + str(np.isnan(x.detach().numpy()).any()))
train(0)
To summarize, I think I'm basically just feeding an input into an LSTM with random initialized hidden values, and then taking the sigmoid of the output of that LSTM. Then I'm feeding that output to a decoder LSTM, and taking the sigmoid of the output of the decoder output and using that as my final value.
Unfortunately, even in the first iteration, the model often outputs a vector of the correct shape (batch_size, seq_length, seq_dim), but contains at least one and sometimes all NaN values. What am I doing wrong?
Thanks!
What I've tried so far:
changing LSTM to GRU
changing sigmoid to relu
changing the dimension of the hidden representation
getting the failing input down to the encoder
Edit: Apologies to everyone who tried to help when I had broken code - I really do value your time, and thanks so much for helping!