I'm trying to one time run my model and see if it's working. I've searched the error and suggested answers was to instantiate the model once. I already did that. However, even after instantiating the model. it gives an error once requesting access to model parametes. What's the problem?
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, TensorDataset
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from IPython import display
display.set_matplotlib_formats('svg')
data = np.array([[1,1]])
label = np.array([2])
for i in range(-50,51,1):
data = np.append(data, [[i,i]], axis=0)
label = np.append(label, [i+i])
# conver to tensor
T_data = torch.tensor(data).float()
T_label = torch.tensor(label).long()
# split data
train_data, test_data, train_label, test_label = train_test_split(T_data, T_label, test_size= .2)
# convert into Pytorch dataset
train_data = TensorDataset(train_data, train_label)
test_data = TensorDataset(test_data, test_label)
# translate into dataloader
batchsize = 32
train_loader = DataLoader(train_data, batch_size= batchsize, shuffle =True, drop_last=True)
test_loader = DataLoader(test_data, batch_size=test_data.tensors[0].shape[0])
class AddNN(nn.Module):
def __init__(self):
super().__init__()
self.input = nn.Linear(2,16)
## hidden layer
self.fc1 = nn.Linear(16,32)
self.fc2 = nn.Linear(32,1)
#output layer
self.output = nn.Linear(1,1)
# forward pass
def forward(self, x):
x = F.relu(self.input(x))
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
return self.output(x)
net =AddNN()
lossfun = nn.MSELoss()
optimizer = torch.optim.Adam(AddNN.parameters(), lr = .05)
It returns TypeError: parameters() missing 1 required positional argument: 'self'
What it means to have missing a required positional argument here??
After model instantiation, you should use that model object instead of the model class inside Adam and with .parameters()
net = AddNN()
lossfun = nn.MSELoss()
optimizer = torch.optim.Adam(net.parameters(), lr = .05)
Related
I'm getting following error and I'm not able to figure out why:
RuntimeError: Model-building function did not return a valid Keras Model instance, found (<tensorflow.python.keras.engine.functional.Functional object at 0x7f74d8b849d0>, <tensorflow.python.keras.engine.functional.Functional object at 0x7f74d8b80810>)
I have read the answers here and here which seem to telling to import keras from tensorflow instead of stand alone keras which I'm doing but still getting the error. I would very much appreciate your help in figuring this out. Below is my entire code:
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from numba import njit
import tensorflow as tf
import numpy as np
from sklearn.model_selection import KFold
from sklearn.model_selection._split import _BaseKFold, indexable, _num_samples
from sklearn.utils.validation import _deprecate_positional_args
import pandas as pd
import kerastuner as kt
import gc
from tqdm import tqdm
from random import choices
import warnings
warnings.filterwarnings('ignore')
class MyTuner(kt.Tuner):
def run_trial(self, trial, x, y):
cv = PurgedGroupTimeSeriesSplit(n_splits=5, group_gap = 20)
val_losses = []
for train_indices, test_indices in cv.split(x, groups=x[0]):
x_train, y_train = x[train_indices, 1:], y[train_indices]
x_test, y_test = x[test_indices, 1:], y[test_indices]
x_train = apply_transformation(x_train)
x_test = apply_transformation(x_test)
model = self.hypermodel.build(trial.hyperparameters)
model.fit(x_train, y_train, batch_size = hp.Int('batch_size', 500, 5000, step=500, default=4000),
epochs = hp.Int('epochs', 100, 1000, step=200, default=500))
val_losses.append(model.evaluate(x_test, y_test))
self.oracle.update_trial(trial.trial_id, {'val_loss': np.mean(val_losses)})
self.save_model(trial.trial_id, model)
def create_autoencoder(hp, input_dim, output_dim):
i = Input(input_dim)
encoded = BatchNormalization()(i)
encoded = GaussianNoise(hp.Float('gaussian_noise', 1e-2, 1, sampling='log', default=5e-2))(encoded)
encoded = Dense(hp.Int('encoder_dense', 100, 300, step=50, default=64), activation='relu')(encoded)
decoded = Dropout(hp.Float('decoder_dropout_1', 1e-1, 1, sampling='log', default=0.2))(encoded)
decoded = Dense(input_dim,name='decoded')(decoded)
x = Dense(hp.Int('output_x', 32, 100, step=10, default=32),activation='relu')(decoded)
x = BatchNormalization()(x)
x = Dropout(hp.Float('x_dropout_1', 1e-1, 1, sampling='log', default=0.2))(x)
x = Dense(hp.Int('output_x', 32, 100, step=10, default=32),activation='relu')(x)
x = BatchNormalization()(x)
x = Dropout(hp.Float('x_dropout_2', 1e-1, 1, sampling='log', default=0.2))(x)
x = Dense(output_dim,activation='sigmoid',name='label_output')(x)
encoder = Model(inputs=i,outputs=encoded)
autoencoder = Model(inputs=i,outputs=[decoded, x])
# optimizer = hp.Choice('optimizer', ['adam', 'sgd'])
autoencoder.compile(optimizer=Adam(hp.Float('lr', 0.00001, 0.1, default=0.001)),
loss='sparse_binary_crossentropy',
metrics=['accuracy'])
return autoencoder, encoder
build_model = lambda hp: create_autoencoder(hp, X[:, 1:].shape[1], y.shape[1])
tuner = MyTuner(
oracle=kt.oracles.BayesianOptimization(
objective=kt.Objective('val_loss', 'min'),
max_trials=20),
hypermodel=build_model,
directory='./',
project_name='autoencoders')
tuner.search(X, (X,y), callbacks=[EarlyStopping('val_loss',patience=5),
ReduceLROnPlateau('val_loss',patience=3)])
encoder_hp = tuner.get_best_hyperparameters(1)[0]
print("Best Encoder Hyper-parameter:", encoder_hp)
best_autoencoder = tuner.get_best_models(1)[0]
RuntimeError: Model-building function did not return a valid Keras Model instance, found (<tensorflow.python.keras.engine.functional.Functional object at 0x7f74d8b849d0>, <tensorflow.python.keras.engine.functional.Functional object at 0x7f74d8b80810>)
(<tensorflow.python.keras.engine.functional.Functional object at 0x7f74d8b849d0>, <tensorflow.python.keras.engine.functional.Functional object at 0x7f74d8b80810>)
As you can see this a tuple of two Keras Model instance. This is output of create_autoencoder(hp, input_dim, output_dim).
def create_autoencoder(hp, input_dim, output_dim):
# some lines of codes
return autoencoder, encoder
From my understanding, you are not using encoder. Therefore, you can remove it in your function.
That function will be look like this
def create_autoencoder(hp, input_dim, output_dim):
# some lines of codes
return autoencoder
It will only return a Keras Model Instance.
I'm learning to use pytorch and I got an error that won't let me continue programming.
My code:
import torch.nn as nn
from skorch import NeuralNetClassifier #integracao com sklearn
from sklearn.model_selection import cross_val_score,GridSearchCV
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
import torch
import torch.nn.functional as F
from torch import nn,optim
class classificadorFinal(nn.Module):
def __init__(self, activation=F.tanh, neurons=16, initializer=torch.nn.init.uniform_, dropout=0.3):
##from melhores_parametros
super().__init__()
self.dense0 = nn.Linear(4, neurons)
initializer(self.dense0.weight)
self.activation0 = activation
self.dense1 = nn.Linear(neurons, neurons)
initializer(self.dense1.weight)
self.activation1 = activation
self.dense2 = nn.Linear(neurons, 3)
self.dropout = nn.Dropout(dropout)
def forward(self, X):
X = self.dense0(X)
X = self.activation0(X)
X = self.dropout(X)
X = self.dense1(X)
X = self.activation1(X)
X = self.dropout(X)
X = self.dense2(X)
return X
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(classificador.parameters(), lr = 0.001, weight_decay = 0.0001)
#treino
for epoch in range(200):##from melhores_parametros
running_loss = 0.
running_accuracy = 0.
for data in train_loader:
inputs, labels = data
optimizer.zero_grad()
outputs = classificadorFinal(inputs)
loss = criterion(outputs, labels)###erro
loss.backward()
optimizer.step()
running_loss += loss.item()
ps = F.softmax(outputs)
top_p, top_class = ps.topk(k = 1, dim = 1)
equals = top_class == labels.view(*top_class.shape)
running_accuracy += torch.mean(equals.type(torch.float))
print('Época {:3d}: perda {:3.5f} - precisão {:3.5f}'.format(epoch + 1, running_loss/len(train_loader), running_accuracy/len(train_loader)))
The error occurs exactly on loss = criterion(outputs, labels):
AttributeError: 'classificadorFinal' object has no attribute 'log_softmax'
I found out this error is well known, but I did not understand the proposed solution:
disable aux_logits when the model is created aux_logits=False.
A little help, please!
The outputs are not actually the output of the model, but rather the model itself. classificadorFinal is the class, calling it creates an object/instance of that class, and inputs will be the first argument to the __init__ method, namely activation.
# Creates an instance of the model
outputs = classificadorFinal(inputs)
You first have to create the model (an instance), which should be done once, then call that model with the inputs. It looks like you have already created the model before, as you are using classificador.parameters() for the optimiser, hence classificador is presumably the instance of the model. You need to call classificador (instance) not classificadorFinal (class) to create the outputs.
# Call the instance of the model, not the class
outputs = classificador(inputs)
Is there a way to get the train/validation loss from a Skorch fit in e.g a list (if you want to do some plotting, statistics)?
You can use the history (which allows slicing over time) to get this information. For example:
train_loss = net.history[:, 'train_loss']
which returns the train_loss for each recorded epoch.
Here is an example based on the following.
import numpy as np
import torch
import torch.nn.functional as F
from matplotlib import pyplot as plt
from sklearn.datasets import make_classification
from skorch import NeuralNetClassifier
from torch import nn
torch.manual_seed(0)
class ClassifierModule(nn.Module):
def __init__(
self,
num_units=10,
nonlin=F.relu,
dropout=0.5,
):
super(ClassifierModule, self).__init__()
self.num_units = num_units
self.nonlin = nonlin
self.dropout = dropout
self.dense0 = nn.Linear(20, num_units)
self.nonlin = nonlin
self.dropout = nn.Dropout(dropout)
self.dense1 = nn.Linear(num_units, 10)
self.output = nn.Linear(10, 2)
def forward(self, X, **kwargs):
X = self.nonlin(self.dense0(X))
X = self.dropout(X)
X = F.relu(self.dense1(X))
X = F.softmax(self.output(X), dim=-1)
return X
net = NeuralNetClassifier(
ClassifierModule,
max_epochs=20,
lr=0.1,
# device='cuda', # uncomment this to train with CUDA
)
X, y = make_classification(1000, 20, n_informative=10, random_state=0)
X, y = X.astype(np.float32), y.astype(np.int64)
net.fit(X, y)
train_loss = net.history[:, 'train_loss']
valid_loss = net.history[:, 'valid_loss']
plt.plot(train_loss, 'o-', label='training')
plt.plot(valid_loss, 'o-', label='validation')
plt.legend()
plt.show()
And results:
Using this mnist image classification model :
%reset -f
import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms
import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms
import torch.utils.data as data_utils
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_moons
from matplotlib import pyplot
from pandas import DataFrame
import torchvision.datasets as dset
import os
import torch.nn.functional as F
import time
import random
import pickle
from sklearn.metrics import confusion_matrix
import pandas as pd
import sklearn
trans = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
root = './data'
if not os.path.exists(root):
os.mkdir(root)
train_set = dset.MNIST(root=root, train=True, transform=trans, download=True)
test_set = dset.MNIST(root=root, train=False, transform=trans, download=True)
batch_size = 64
train_loader = torch.utils.data.DataLoader(
dataset=train_set,
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(
dataset=test_set,
batch_size=batch_size,
shuffle=True)
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self).__init__()
self.fc1 = nn.Linear(28*28, 500)
self.fc2 = nn.Linear(500, 256)
self.fc3 = nn.Linear(256, 2)
def forward(self, x):
x = x.view(-1, 28*28)
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
x = self.fc3(x)
return x
num_epochs = 2
random_sample_size = 200
values_0_or_1 = [t for t in train_set if (int(t[1]) == 0 or int(t[1]) == 1)]
values_0_or_1_testset = [t for t in test_set if (int(t[1]) == 0 or int(t[1]) == 1)]
print(len(values_0_or_1))
print(len(values_0_or_1_testset))
train_loader_subset = torch.utils.data.DataLoader(
dataset=values_0_or_1,
batch_size=batch_size,
shuffle=True)
test_loader_subset = torch.utils.data.DataLoader(
dataset=values_0_or_1_testset,
batch_size=batch_size,
shuffle=False)
train_loader = train_loader_subset
# Hyper-parameters
input_size = 100
hidden_size = 100
num_classes = 2
# learning_rate = 0.00001
learning_rate = .0001
# Device configuration
device = 'cpu'
print_progress_every_n_epochs = 1
model = NeuralNet().to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
N = len(train_loader)
# Train the model
total_step = len(train_loader)
most_recent_prediction = []
test_actual_predicted_dict = {}
rm = random.sample(list(values_0_or_1), random_sample_size)
train_loader_subset = data_utils.DataLoader(rm, batch_size=4)
for epoch in range(num_epochs):
for i, (images, labels) in enumerate(train_loader_subset):
# Move tensors to the configured device
images = images.reshape(-1, 2).to(device)
labels = labels.to(device)
# Forward pass
outputs = model(images)
loss = criterion(outputs, labels)
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (epoch) % print_progress_every_n_epochs == 0:
print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, i+1, total_step, loss.item()))
predicted_test = []
model.eval() # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)
probs_l = []
predicted_values = []
actual_values = []
labels_l = []
with torch.no_grad():
for images, labels in test_loader_subset:
images = images.to(device)
labels = labels.to(device)
outputs = model(images)
_, predicted = torch.max(outputs.data, 1)
predicted_test.append(predicted.cpu().numpy())
sm = torch.nn.Softmax()
probabilities = sm(outputs)
probs_l.append(probabilities)
labels_l.append(labels.cpu().numpy())
predicted_values.append(np.concatenate(predicted_test).ravel())
actual_values.append(np.concatenate(labels_l).ravel())
if (epoch) % 1 == 0:
print('test accuracy : ', 100 * len((np.where(np.array(predicted_values[0])==(np.array(actual_values[0])))[0])) / len(actual_values[0]))
I'm to attempting to integrate 'Local Interpretable Model-Agnostic Explanations for machine learning classifiers' : https://marcotcr.github.io/lime/
It appears PyTorch support is not enabled as it is not mentioned in doc and following tutorial :
https://marcotcr.github.io/lime/tutorials/Tutorial%20-%20images.html
With my updated code for PyTorch :
from lime import lime_image
import time
explainer = lime_image.LimeImageExplainer()
explanation = explainer.explain_instance(images[0].reshape(28,28), model(images[0]), top_labels=5, hide_color=0, num_samples=1000)
Causes error :
/opt/conda/lib/python3.6/site-packages/skimage/color/colorconv.py in gray2rgb(image, alpha)
830 is_rgb = False
831 is_alpha = False
--> 832 dims = np.squeeze(image).ndim
833
834 if dims == 3:
AttributeError: 'Tensor' object has no attribute 'ndim'
So appears tensorflow object is expected here ?
How to integrate LIME with PyTorch image classification ?
Here's my solution:
Lime expects an image input of type numpy. This is why you get the attribute error and a solution would be to convert the image (from Tensor) to numpy before passing it to the explainer object. Another solution would be to select a specific image with the test_loader_subset and convert it with img = img.numpy().
Secondly, in order to make LIME work with pytorch (or any other framework), you'll need to specify a batch prediction function which outputs the prediction scores of each class for each image. The name of this function (here I've called it batch_predict) is then passed to explainer.explain_instance(img, batch_predict, ...). The batch_predict needs to loop through all images passed to it, convert them to Tensor, make a prediction and finally return the prediction score list (with numpy values). This is how I got it working.
Note also that the images need to have shape (... ,... ,3) or (... ,... ,1) in order to be properly segmented by the default segmentation algorithm. This means that you might have to use np.transpose(img, (...)). You may specify the segmentation algorithm as well if the results are poor.
Finally you'll need to display the LIME image mask on top of the original image. This snippet shows how this may be done:
from skimage.segmentation import mark_boundaries
temp, mask = explanation.get_image_and_mask(explanation.top_labels[0], positive_only=False, num_features=5, hide_rest=False)
img_boundry = mark_boundaries(temp, mask)
plt.imshow(img_boundry)
plt.show()
This notebook is a good reference:
https://github.com/marcotcr/lime/blob/master/doc/notebooks/Tutorial%20-%20images%20-%20Pytorch.ipynb
I am trying to use 20 news groups data set available in sklearn to train a LSTM to do incremental learning (classification). I used the sklearn's TfidfVectorizer to pre-process the data. Then I turned the resulting sparse matrix into a numpy array before feeding it. After that when coding the below line:
outputs, final_state = tf.nn.dynamic_rnn(cell, inputs_, initial_state=initial_state)
It gave an error saying that the 'inputs_' should have 3 dimensions. so I used:
inputs_ = tf.expand_dims(inputs_, 0)
To expand the dimension. But when I do that i get the error:
ValueError: Input size (depth of inputs) must be accessible via shape
inference, but saw value None.
The shape of 'input_' is:
(1, 134410)
I already went through this post, but it did not help.
I cannot seem to understand how to solve this issue. Any help is much appreciated. Thank you in advance!
show below is my complete code:
import os
from collections import Counter
import tensorflow as tf
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.datasets import fetch_20newsgroups
import matplotlib as mplt
from matplotlib import cm
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from sklearn.metrics import f1_score, recall_score, precision_score
from string import punctuation
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelBinarizer
def pre_process():
newsgroups_data = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))
vectorizer = TfidfVectorizer()
features = vectorizer.fit_transform(newsgroups_data.data)
lb = LabelBinarizer()
labels = np.reshape(newsgroups_data.target, [-1])
labels = lb.fit_transform(labels)
return features, labels
def get_batches(x, y, batch_size=1):
for ii in range(0, len(y), batch_size):
yield x[ii:ii + batch_size], y[ii:ii + batch_size]
def plot_error(errorplot, datapoint, numberOfWrongPreds):
errorplot.set_xdata(np.append(errorplot.get_xdata(), datapoint))
errorplot.set_ydata(np.append(errorplot.get_ydata(), numberOfWrongPreds))
errorplot.autoscale(enable=True, axis='both', tight=None)
plt.draw()
def train_test():
features, labels = pre_process()
#Defining Hyperparameters
epochs = 1
lstm_layers = 1
batch_size = 1
lstm_size = 30
learning_rate = 0.003
print(lstm_size)
print(batch_size)
print(epochs)
#--------------placeholders-------------------------------------
# Create the graph object
graph = tf.Graph()
# Add nodes to the graph
with graph.as_default():
tf.set_random_seed(1)
inputs_ = tf.placeholder(tf.float32, [None,None], name = "inputs")
# labels_ = tf.placeholder(dtype= tf.int32)
labels_ = tf.placeholder(tf.int32, [None,None], name = "labels")
#getting dynamic batch size according to the input tensor size
# dynamic_batch_size = tf.shape(inputs_)[0]
#output_keep_prob is the dropout added to the RNN's outputs, the dropout will have no effect on the calculation of the subsequent states.
keep_prob = tf.placeholder(tf.float32, name = "keep_prob")
# Your basic LSTM cell
lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)
# Add dropout to the cell
drop = tf.contrib.rnn.DropoutWrapper(lstm, output_keep_prob=keep_prob)
#Stack up multiple LSTM layers, for deep learning
cell = tf.contrib.rnn.MultiRNNCell([drop] * lstm_layers)
# Getting an initial state of all zeros
initial_state = cell.zero_state(batch_size, tf.float32)
inputs_ = tf.expand_dims(inputs_, 0)
outputs, final_state = tf.nn.dynamic_rnn(cell, inputs_, initial_state=initial_state)
#hidden layer
hidden = tf.layers.dense(outputs[:, -1], units=25, activation=tf.nn.relu)
logit = tf.contrib.layers.fully_connected(hidden, 1, activation_fn=None)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logit, labels=labels_))
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
saver = tf.train.Saver()
# ----------------------------online training-----------------------------------------
with tf.Session(graph=graph) as sess:
tf.set_random_seed(1)
sess.run(tf.global_variables_initializer())
iteration = 1
state = sess.run(initial_state)
wrongPred = 0
errorplot, = plt.plot([], [])
for ii, (x, y) in enumerate(get_batches(features, labels, batch_size), 1):
feed = {inputs_: x.toarray(),
labels_: y,
keep_prob: 0.5,
initial_state: state}
predictions = tf.round(tf.nn.softmax(logit)).eval(feed_dict=feed)
print("----------------------------------------------------------")
print("Iteration: {}".format(iteration))
print("Prediction: ", predictions)
print("Actual: ",y)
pred = np.array(predictions)
print(pred)
print(y)
if not ((pred==y).all()):
wrongPred += 1
if ii % 27 == 0:
plot_error(errorplot,ii,wrongPred)
loss, states, _ = sess.run([cost, final_state, optimizer], feed_dict=feed)
print("Train loss: {:.3f}".format(loss))
iteration += 1
saver.save(sess, "checkpoints/sentiment.ckpt")
errorRate = wrongPred/len(labels)
print("ERROR RATE: ", errorRate )
if __name__ == '__main__':
train_test()
ValueError: Input size (depth of inputs) must be accessible via shape inference, but saw value None.
This error is given because you don't specify the size nor the amount of inputs.
I got the script working like this:
inputs_ = tf.placeholder(tf.float32, [1,None], name = "inputs")
inputs_withextradim = tf.expand_dims(inputs_, 2)
outputs, final_state = tf.nn.dynamic_rnn(cell, inputs_withextradim, initial_state=initial_state)