I am trying to calculate sklearn log loss but continuously getting value error. how to resolve the error. The code is simple - fit the label encoder to array and then use sklearn logloss that takes three arguments - the labels, the ground truth and the probability values of each class.
from sklearn import preprocessing
le = preprocessing.LabelEncoder()
le.fit([2.5, 3.0, 3.5, 3.8, 4.0, 4.5, 5.0, 5.5, 6.0])
from sklearn.metrics import log_loss
le.classes_
log_loss([6.0], [[0., 0., 0., 0., 0.28571429, 0.14285714, 0., 0.57142857, 0. ]], labels=list(le.classes_))
Error
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
C:\Users\PRANAV~1\AppData\Local\Temp/ipykernel_25368/2311544075.py in <module>
----> 1 log_loss([6.0], [[0., 0., 0., 0., 0.28571429, 0.14285714,
2 0., 0.57142857, 0. ]], labels=list(le.classes_))
~\AppData\Roaming\Python\Python39\site-packages\sklearn\utils\validation.py in inner_f(*args, **kwargs)
61 extra_args = len(args) - len(all_args)
62 if extra_args <= 0:
---> 63 return f(*args, **kwargs)
64
65 # extra_args > 0
~\AppData\Roaming\Python\Python39\site-packages\sklearn\metrics\_classification.py in log_loss(y_true, y_pred, eps, normalize, sample_weight, labels)
2233
2234 if labels is not None:
-> 2235 lb.fit(labels)
2236 else:
2237 lb.fit(y_true)
~\AppData\Roaming\Python\Python39\site-packages\sklearn\preprocessing\_label.py in fit(self, y)
295
296 self.sparse_input_ = sp.issparse(y)
--> 297 self.classes_ = unique_labels(y)
298 return self
299
~\AppData\Roaming\Python\Python39\site-packages\sklearn\utils\multiclass.py in unique_labels(*ys)
96 _unique_labels = _FN_UNIQUE_LABELS.get(label_type, None)
97 if not _unique_labels:
---> 98 raise ValueError("Unknown label type: %s" % repr(ys))
99
100 ys_labels = set(chain.from_iterable(_unique_labels(y) for y in ys))
ValueError: Unknown label type: ([2.5, 3.0, 3.5, 3.8, 4.0, 4.5, 5.0, 5.5, 6.0],)
What you are doing is not valid.
Log_loss excepts as input arguments y_true, y_pred), which are the ground truth (correct) labels for n_samples samples and the predicted probabilities, as returned by a classifier’s predict_proba method, respectively.
To solve this, convert the numerical (invalid) labels into strings:
log_loss(['6.0'], [[0., 0., 0., 0., 0.28571429, 0.14285714, 0., 0.57142857, 0.]],
labels=list(le.classes_.astype(str)))
# 34.53877639491069
The problem: you have floats as labels and this breaks the function. In sklearn, numerical labels need to be integers.
Here is a full numerical example:
log_loss([6], [[0., 0., 0., 0., 0.28571429, 0.14285714, 0., 0.57142857, 0.]],
...: labels=[0,1,2,3,4,5,6,7,8])
Here is another problematic case:
log_loss([6], [[0., 0., 0., 0., 0.28571429, 0.14285714, 0., 0.57142857, 0.]],
...: labels=[0,1.1,2,3,4,5,6,7,8])
# ValueError: Unknown label type: ([0, 1.1, 2, 3, 4, 5, 6, 7, 8],)
Related
When I run 'loss.backward()' and 'weight.grad' I get a tensor containing all zeros. Also, 'weight.grad_fn' retruns NONE.
However, it all seems to return the correct result for the second layer 'w2'.
If I play with simple operations such as x*2 or x**2 'backward()' and '.grad' return correct results
Here's my code:
import torch
from torch import nn
import torch.nn.functional as F
from torchvision import datasets, transforms
# Getting MNIST data
num_workers = 0
batch_size = 64
transform = transforms.ToTensor()
train_data = datasets.MNIST(root='data', train=True, download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, num_workers=num_workers)
dataiter = iter(train_loader)
images, labels = dataiter.next()
#####################################
#####################################
#### NN Part
def activation(x):
return 1/(1+torch.exp(-x))
inputs = torch.from_numpy(images.view())
# Flatten the inputs format from (64,1,28,28) into (64,784)
inputs = inputs.reshape(images.shape[0], int(images.shape[1]*images.shape[2]*images.shape[3]))
w1 = torch.randn(784, 256, requires_grad=True)# n_input, n_hidden
b1 = torch.randn(256)# n_hidden
w2 = torch.randn(256, 10, requires_grad=True)# n_hidden, n_output
b2 = torch.randn(10)# n_output
h = activation(torch.mm(inputs, w1) + b1)
y = torch.mm(h, w2) + b2
#print(h)
#print(y)
y.sum().backward()
print(w1.grad)
print(w1.grad_fn)
#print(w2.grad)
#print(w2.grad_fn)
By the way it gives me the same problem if I try to run it this way also:
images = images.reshape(images.shape[0], -1)
model = nn.Sequential(nn.Linear(784, 128),
nn.ReLU(),
nn.Linear(128, 64),
nn.ReLU(),
nn.Linear(64, 10),
nn.LogSoftmax(dim=1))
logits = model(images)
criterion = nn.NLLLoss()
loss = criterion(logits, labels)
print(loss)
print(loss.grad_fn)
print('Before backward pass: ', model[0].weight.grad)
loss.backward()
print('After: ', model[0].weight.grad)
#print('After: ', model[2].weight.grad)
#print('After: ', model[4].weight.grad)
The gradients of w1 are not all zero, there are simply a lot of zeros, especially around the border, because the MNIST images have a lot of black pixels (zeros). When multiplying with zero, the resulting gradients are also zero.
By printing w1.grad you only see a very small part of the values (borders), and you just can't see the non-zero values.
w1.grad
# => tensor([[0., 0., 0., ..., 0., 0., 0.],
# [0., 0., 0., ..., 0., 0., 0.],
# [0., 0., 0., ..., 0., 0., 0.],
# ...,
# [0., 0., 0., ..., 0., 0., 0.],
# [0., 0., 0., ..., 0., 0., 0.],
# [0., 0., 0., ..., 0., 0., 0.]])
# Indices of non-zero elements
w1.grad.nonzero()
# => tensor([[ 71, 0],
# [ 71, 1],
# [ 71, 2],
# ...,
# [746, 253],
# [746, 254],
# [746, 255]])
I found some weird activity by using add_loss function in Keras model compare to use loss='something like mse' in compile function while I was coding exercises for auto-encoder.
The same model of that uses loss in compile function requires y_train and y_test(if I use validation_data) as usual, but add_loss(mse(inputs,outputs)) emits the error if I provide y data(y_train/ y_test).
Here is the simple code I have made to train mnist, flattened to be 784 dim from 28 x 28.
the common body of the sample code is as below:
encode_dim = 32
inputs = Input(shape=(784,))
encoded = Dense(128, activation='relu')(inputs)
encoded = Dense(64, activation='relu')(encoded)
encoded = Dense(encode_dim, activation='relu')(encoded)
encoder = Model(inputs=inputs, outputs=encoded)
print(encoder.summary())
encoded_inputs = Input(shape=(encode_dim, ))
decoded = Dense(64, activation='relu')(encoded_inputs)
decoded = Dense(128, activation='relu')(decoded)
decoded = Dense(784, activation='sigmoid')(decoded)
decoder = Model(inputs=encoded_inputs, outputs=decoded)
print(decoder.summary())
outputs = decoder(encoder(inputs))
ae = Model(inputs, outputs)
print(ae.summary())
with providing loss by name in compile function would be like below:
ae.compile(optimizer='adam', loss='mse')
ae.fit(x_train, y_train, epochs=1, batch_size=256)
would work,
2. but with the case below:
ae_loss = mse(inputs, outputs)
ae.add_loss(ae_loss )
ae.compile(optimizer='adam')
ae.fit(x_train, y_train, epochs=1, batch_size=256)
will give the error like below:
ValueError Traceback (most recent call
last) in
1 deep_vaestyle_model2 = AutoEncoderTester(DeepModelVAEStyle())
2 deep_vaestyle_model2.train(x_train=x_train_flat, y_train=x_train_flat, x_test=x_test_flat, y_test=x_test_flat,
----> 3 epochs=1, batch_size=1024, verbose=1)
4 deep_vaestyle_model2.test(x_test=x_test_flat)
in train(self, x_train, y_train,
x_test, y_test, epochs, batch_size, verbose)
16 histogram_freq=0,
17 write_graph=True,
---> 18 write_grads=True,
19 # batch_size=batch_size,
20 # write_images=True
d:\igs_projects\realtime_eeg_analyzer\venv\lib\site-packages\keras\engine\training.py
in fit(self, x, y, batch_size, epochs, verbose, callbacks,
validation_split, validation_data, shuffle, class_weight,
sample_weight, initial_epoch, steps_per_epoch, validation_steps,
**kwargs)
950 sample_weight=sample_weight,
951 class_weight=class_weight,
--> 952 batch_size=batch_size)
953 # Prepare validation data.
954 do_validation = False
d:\igs_projects\realtime_eeg_analyzer\venv\lib\site-packages\keras\engine\training.py
in _standardize_user_data(self, x, y, sample_weight, class_weight,
check_array_lengths, batch_size)
787 feed_output_shapes,
788 check_batch_axis=False, # Don't enforce the batch size.
--> 789 exception_prefix='target')
790
791 # Generate sample-wise weight values given the sample_weight and
d:\igs_projects\realtime_eeg_analyzer\venv\lib\site-packages\keras\engine\training_utils.py
in standardize_input_data(data, names, shapes, check_batch_axis,
exception_prefix)
61 raise ValueError('Error when checking model ' +
62 exception_prefix + ': '
---> 63 'expected no data, but got:', data)
64 return []
65 if data is None:
ValueError: ('Error when checking model target: expected no data, but
got:', array([[0., 0., 0., ..., 0., 0., 0.],
[0., 0., 0., ..., 0., 0., 0.],
[0., 0., 0., ..., 0., 0., 0.],
...,
[0., 0., 0., ..., 0., 0., 0.],
[0., 0., 0., ..., 0., 0., 0.],
[0., 0., 0., ..., 0., 0., 0.]]))
but if I give y data(y_train, and y_test) as None, then it runs as I expected.
I tried to find out why keras consider those two are different but I couldn't. Can someone explain????
I'm trying to tile a length 18 1 hot vector into a 40x40 grid.
Looking at pytorch docs, expand dims seems to be what i need.
But I cannot get it to work. Any idea what I'm doing wrong?
one_hot = torch.zeros(18).unsqueeze(0)
one_hot[0,1] = 1.0
one_hot
tensor([[0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])
one_hot.expand(-1,-1,40,40)
Traceback (most recent call last):
File "<input>", line 1, in <module>
RuntimeError: The expanded size of the tensor (40) must match the existing size (18) at non-singleton dimension 3
I'm expecting a tensor of shape (1, 18, 40,40)
expand works along singleton dimensions of the input tensor. In your example, you are trying to expand a 1-by-18 tensor along its (non-existent) third and fourth dimensions - this is why you are getting an error. The only singleton dimension (=dimension with size==1) you have is the first dimension.
fix
one_hot = torch.zeros(1,18,1,1, dtype=torch.float) # create the tensor with all singleton dimensions in place
one_hot[0,1,0,0] = 1.
one_hot.expand(-1,-1,40,40)
I want to use 'KDtree'(this is the best option. Other 'KNN' algorithms aren't optimal for my project) with custom distance metric. I checked some answers here for similar questions, and this should work...but doesn't.
distance_matrix is symetric as should be by definition:
array([[ 1., 0., 5., 5., 0., 3., 2.],
[ 0., 1., 0., 0., 0., 0., 0.],
[ 5., 0., 1., 5., 0., 2., 3.],
[ 5., 0., 5., 1., 0., 4., 4.],
[ 0., 0., 0., 0., 1., 0., 0.],
[ 3., 0., 2., 4., 0., 1., 0.],
[ 2., 0., 3., 4., 0., 0., 1.]])
I know my metric is not 'formally metric', but in documentation it says that my function has to be 'formally metric', only when I'm using 'ball tree'(under User-defined distance:).
Here is my code:
from sklearn.neighbors import DistanceMetric
def dist(x, y):
dist = 0
for elt_x, elt_y in zip(x, y):
dist += distance_matrix[elt_x, elt_y]
return dist
X = np.array([[1,0], [1,2], [1,3]])
tree = KDtree(X, metric=dist)
I get this error:
NameError
Traceback (most recent call last)
<ipython-input-27-b5fac7810091> in <module>()
7 return dist
8 X = np.array([[1,0], [1,2], [1,3]])
----> 9 tree = KDtree(X, metric=dist)
NameError: name 'KDtree' is not defined
I tried also:
from sklearn.neighbors import KDTree
def dist(x, y):
dist = 0
for elt_x, elt_y in zip(x, y):
dist += distance_matrix[elt_x, elt_y]
return dist
X = np.array([[1,0], [1,2], [1,3]])
tree = KDTree(X, metric=lambda a,b: dist(a,b))
I get this error:
ValueError
Traceback (most recent call last)
<ipython-input-27-b5fac7810091> in <module>()
7 return dist
8 X = np.array([[1,0], [1,2], [1,3]])
----> 9 tree = KDtree(X, metric=dist)
ValueError: metric PyFuncDistance is not valid for KDTree
I also tried:
from sklearn.neighbors import NearestNeighbors
nbrs = NearestNeighbors(n_neighbors=1, algorithm='kd_tree', metric=dist_metric)
I get following error:
ValueError Traceback (most recent call last)
<ipython-input-32-c78d02cacb5a> in <module>()
1 from sklearn.neighbors import NearestNeighbors
----> 2 nbrs = NearestNeighbors(n_neighbors=1, algorithm='kd_tree', metric=dist_metric)
/usr/local/lib/python3.5/dist-packages/sklearn/neighbors/unsupervised.py in __init__(self, n_neighbors, radius, algorithm, leaf_size, metric, p, metric_params, n_jobs, **kwargs)
121 algorithm=algorithm,
122 leaf_size=leaf_size, metric=metric, p=p,
--> 123 metric_params=metric_params, n_jobs=n_jobs, **kwargs)
/usr/local/lib/python3.5/dist-packages/sklearn/neighbors/base.py in _init_params(self, n_neighbors, radius, algorithm, leaf_size, metric, p, metric_params, n_jobs)
138 raise ValueError(
139 "kd_tree algorithm does not support callable metric '%s'"
--> 140 % metric)
141 elif metric not in VALID_METRICS[alg_check]:
142 raise ValueError("Metric '%s' not valid for algorithm '%s'"
ValueError: kd_tree algorithm does not support callable metric '<function dist_metric at 0x7f58c2b3fd08>'
I tried all other algorithms (auto, brute,...), but it puts out same error.
I have to use distance matrix for elements of vectors as element is code for characteristics, and 5 can be closer to 1 than is 3. What I need is to get top 3 neighbors(sorted from closest to furthest).
Scikit-learn's KDTree does not support custom distance metrics. The BallTree does support custom distance metrics, but be careful: it is up to the user to make certain the provided metric is actually a valid metric: if it is not, the algorithm will happily return results of a query, but the results will be incorrect.
Also, you should be aware that using a custom Python function as a metric is generally too slow to be useful, because of the overhead of Python callbacks within the traversal of the tree.
Sorry for the title but I could't come up with a better description here.
I am trying to apply batches for training on a model which should have 13 fully connected output layers. Each output layer has only two nodes (but are fully connected as stated).
Building the model's output looks like this:
outputs = list()
for i in range(num_labels):
out_y = Dense(2, activation='softmax', name='out_{:d}'.format(i))(convolution_layer)
outputs.append(out_y)
self.model = Model(input=inputs, output=outputs)
However, I can't manage to feed this model. I've tried to go with a [batch_size, 13, 1, 2] sized output array:
y = np.zeros((batch_size, 13, 1, 2))
But for a batch of size 2 I get:
ValueError: The model expects 13 input arrays, but only received one array. Found: array with shape (2, 13, 1, 2)
I've tried several other things but it's simply not clear to me how the input for the model looks like.
How can I train this model?
I have also tried to pass a list of lists of numpy arrays:
where the first level of the batch represent the sample (here 2) and the second level is the sample with the list of 13 numpy arrays. Yet I am getting:
ValueError: Error when checking model target: you are passing a list as input to your model, but the model expects a list of 13 Numpy arrays instead. The list you passed was: [[array([ 0., 1.]), array([ 0., 1.]), array([ 0., 1.]), array([ 0., 1.]), array([ 0., 1.]), array([ 0., 1.]), array([ 0., 1.]), array([ 0., 1.]), array([ 0., 1.]), array([ 1., 0.]), array([
As suggested, I also tried to return a list() of numpy arrays of size [13,2]:
Where the error becomes:
ValueError: Error when checking model target: the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 13 arrays but instead got the following list of 2 arrays: [array([[ 0., 1.],
[ 0., 1.],
[ 0., 1.],
[ 0., 1.],
[ 0., 1.],
[ 0., 1.],
[ 0., 1.],
[ 0., 1.],
[ 0., 1.],
[ 1., 0.],
[ ...
The code
Below you can find the current code which generates one sample in sample_generator and a full batch in batch_generator (which uses sample_generator).
def batch_generator(w2v, file_path, meta_info, batch_size, sample_generator_fn, embedding_size):
Please note: The code shows now how I generate a list() of [13,2] ndarrays whereas the number of such ndarrays in that list is defined by batch_size.
try:
x = np.zeros((batch_size, meta_info.max_sequence_length, embedding_size, 1))
y = list() #np.zeros((batch_size, 13, 1, 2))
file = open(file_path)
while True:
x[:] = 0.0
#y[:] = 0.0
for batch in range(batch_size):
sentence_info_json = file.readline()
if sentence_info_json == '':
file.seek(0)
sentence_info_json = file.readline()
sample = sample_generator_fn(w2v, sentence_info_json, meta_info)
if not sample:
continue
sentence_embedding = sample[0]
final_length = len(sentence_embedding)
x[batch, :final_length, :, 0] = sentence_embedding
y.append(sample[1])
shuffled = np.asarray(range(batch_size))
np.random.shuffle(shuffled)
x = x[shuffled]
#y = y[shuffled]
y = [y[i] for i in shuffled]
yield x, y
except Exception as e:
print('Error in generator.')
print(e)
raise e
def sample_generator(w2v, sentence_info_json, meta_info):
if not sentence_info_json:
print('???')
sentence_info = json.loads(sentence_info_json)
tokens = [token['word'] for token in sentence_info['corenlp']['tokens']]
sentence = Sentence(tokens=tokens)
sentence_embedding = w2v.get_word_vectors(sentence.tokens.tolist())
sentence_embedding = np.asarray([word_vector for word_vector in sentence_embedding if word_vector is not None])
final_length = len(sentence_embedding)
if final_length == 0:
return None
y = np.zeros((2, len(meta_info.category_dict)))
y[1, :] = 1.
#y_list = []
y_tar = np.zeros((len(meta_info.category_dict), 2))
for i in range(len(meta_info.category_dict)):
y_tar[i][1] = 1.0
# y_list.append(np.asarray([0.0, 1.0]))
for opinion in sentence_info['opinions']:
index = meta_info.category_dict[opinion['category']]
y_tar[index][0] = 1.0
y_tar[index][1] = 0.0
#y_list[index][0] = 1.0
#y_list[index][1] = 0.0
return sentence_embedding, y_tar
As requested, the call to fit_generator()
cnn.model.fit_generator(generator=batch_generator(word2vec,
train_file, train_meta_info,
num_batches, sample_generator,
embedding_size),
samples_per_epoch=2000,
nb_epoch=2,
# validation_data=batch_generator(test_file_path, train_meta_info),
# nb_val_samples=100,
verbose=True)
Your output should be a list as specified in the error. Each element of the list should be a numpy array of size [batch_size, nb_outputs]. So a list of 13 elements of size [batch_size,2] in your case.