I am building an average weighted ensemble model, everything is going ok with the code. But, I could not download the model since there is no name for it. How can I assign a name for the model? The code is as below
models = [ EfficientNetB0_model, EfficientNetB1_model, DenseNet121_model, DenseNet169_model, DenseNet201_model
,EfficientNetB2_model, EfficientNetB3_model, EfficientNetB4_model, EfficientNetB5_model, EfficientNetB6_model,
EfficientNetB7_model, EfficientNetV2B0_model, EfficientNetV2B1_model, EfficientNetV2B2_model, EfficientNetV2B3_model,
EfficientNetV2L_model, EfficientNetV2M_model, EfficientNetV2S_model, InceptionResNetV2_model, InceptionV3_model,
ResNet50_model, ResNet50V2_model, ResNet101_model, ResNet101V2_model, ResNet152_model,
ResNet152V2_model, VGG16_model, VGG19_model, Xception_model, MobileNet_model , MobileNetV2_model
]
preds = [model.predict(X_test) for model in models]
preds=np.array(preds)
weights = [0.2, 0.3, 0.0, 0.1, 0.0,
0.3, 0.2, 0.1, 0.0, 0.3,
0.1, 0.3, 0.3, 0.1, 0.0,
0.1, 0.2, 0.1, 0.1, 0.1,
0.4, 0.0, 0.2, 0.1, 0.4,
0.0, 0.0, 0.1, 0.1, 0.0, 0.0
]
#Use tensordot to sum the products of all elements over specified axes.
weighted_preds = np.tensordot(preds, weights, axes=((0),(0)))
weighted_ensemble_prediction = np.argmax(weighted_preds, axis=1)
weighted_accuracy = accuracy_score(y_test, weighted_ensemble_prediction)
print('Accuracy Score for model1 = ', accuracy1)
print('Accuracy Score for model2 = ', accuracy2)
print('Accuracy Score for model3 = ', accuracy3)
print('Accuracy Score for model4 = ', accuracy4)
print('Accuracy Score for model5 = ', accuracy5)
print('Accuracy Score for model6 = ', accuracy6)
print('Accuracy Score for model7 = ', accuracy7)
print('Accuracy Score for model8 = ', accuracy8)
print('Accuracy Score for model9 = ', accuracy9)
print('Accuracy Score for model10 = ', accuracy10)
print('Accuracy Score for model11 = ', accuracy11)
print('Accuracy Score for model12 = ', accuracy12)
print('Accuracy Score for model13 = ', accuracy13)
print('Accuracy Score for model14 = ', accuracy14)
print('Accuracy Score for model15 = ', accuracy15)
print('Accuracy Score for model16 = ', accuracy16)
print('Accuracy Score for model17 = ', accuracy17)
print('Accuracy Score for model18 = ', accuracy18)
print('Accuracy Score for model19 = ', accuracy19)
print('Accuracy Score for model20 = ', accuracy20)
print('Accuracy Score for model21 = ', accuracy21)
print('Accuracy Score for model22 = ', accuracy22)
print('Accuracy Score for model23 = ', accuracy23)
print('Accuracy Score for model24 = ', accuracy24)
print('Accuracy Score for model25 = ', accuracy25)
print('Accuracy Score for model26 = ', accuracy26)
print('Accuracy Score for model27 = ', accuracy27)
print('Accuracy Score for model28 = ', accuracy28)
print('Accuracy Score for model29 = ', accuracy29)
print('Accuracy Score for model30 = ', accuracy30)
print('Accuracy Score for model31 = ', accuracy31)
print('Accuracy Score for average ensemble = ', ensemble_accuracy)
print('Accuracy Score for weighted average ensemble = ', weighted_accuracy)
I have tried to search for a solution, but I could not find can you help me with that? and I am expecting to get a solution of how to download the weighted average ensemble model. Thank you!
Related
I have 5 classes in validation set and i want to draw a graph based on top1 results per class in validation loop using wandb . I have tried a single accuracy graph based on the average of 5 classes and it works fine but i want to do a separate way like top1 accuracy for each class. I am unable to achieve, are there any way to achieve it?
Validation Loader
val_loaders = []
for nuisance in val_nuisances:
val_loaders.append((nuisance, torch.utils.data.DataLoader(
datasets.ImageFolder(os.path.join(valdir, nuisance), transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True,
)))
val_nuisances = ['shape', 'pose', 'texture', 'context', 'weather']
Validation Loop
def validate(val_loaders, model, criterion, args):
overall_top1 = 0
for nuisance, val_loader in val_loaders:
batch_time = AverageMeter('Time', ':6.3f', Summary.NONE)
losses = AverageMeter('Loss', ':.4e', Summary.NONE)
top1 = AverageMeter('Acc#1', ':6.2f', Summary.AVERAGE)
top5 = AverageMeter('Acc#5', ':6.2f', Summary.AVERAGE)
progress = ProgressMeter(
len(val_loader),
[batch_time, losses, top1, top5],
prefix=f'Test {nuisance}: ')
# switch to evaluate mode
model.eval()
with torch.no_grad():
end = time.time()
for i, (images, target) in enumerate(val_loader):
if args.gpu is not None:
images = images.cuda(args.gpu, non_blocking=True)
if torch.cuda.is_available():
target = target.cuda(args.gpu, non_blocking=True)
# compute output
output = model(images)
loss = criterion(output, target)
# measure accuracy and record loss
acc1, acc5 = accuracy(output, target, topk=(1, 5))
losses.update(loss.item(), images.size(0))
top1.update(acc1[0], images.size(0))
top5.update(acc5[0], images.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
progress.display(i)
progress.display_summary()
overall_top1 += top1.avg
overall_top1 /= len(val_loaders)
return top1.avg
I don't see any log to W&B in your code, but logging the top1 accuracy per class would just be
class_names = ['shape', 'pose', 'texture', 'context', 'weather']
top1_accuracies = [0.9, 0.8, 0.9, 0.9, 0.8]
wandb.log({class_names[0]: top1_accuracies[0], class_names[1]: top1_accuracies[1], ...}
In the above example, it looks like you're not actually creating a variable for the top1 accuracy of each class. You'll want to do that first. Taken from https://stackoverflow.com/a/50977153/3959708
You can use sklearn's confusion matrix to get the accuracy
from sklearn.metrics import confusion_matrix
import numpy as np
y_true = [0, 1, 2, 2, 2]
y_pred = [0, 0, 2, 2, 1]
target_names = ['class 0', 'class 1', 'class 2']
#Get the confusion matrix
cm = confusion_matrix(y_true, y_pred)
#array([[1, 0, 0],
# [1, 0, 0],
# [0, 1, 2]])
#Now the normalize the diagonal entries
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
#array([[1. , 0. , 0. ],
# [1. , 0. , 0. ],
# [0. , 0.33333333, 0.66666667]])
#The diagonal entries are the accuracies of each class
cm.diagonal()
#array([1. , 0. , 0.66666667])
this is the code where I was working on Image Classification using Pytorch and I'm not able to get the accuracy right.
the accuracy is exceeding 100 ,can anyone help me to find the error.
def trained_model(criterion, optimizer, epochs=5):
epoch_loss = 0.0
epoch_accuracy = 0
running_loss = 0
running_accuracy = 0
total = 0
for epoch in range(epochs):
print('epoch : {}/{}'.format(epoch+1, epochs))
for images, labels in train_loader:
images, labels = images.to(device), labels.to(device)
optimizer.zero_grad()
outputs = model(images)
loss = criterion(outputs, labels)
_, predictions = torch.max(outputs, dim=1)
loss.backward()
optimizer.step()
running_loss += loss.item()
running_accuracy += torch.sum(predictions == labels.data)
epoch_loss = running_loss / len(train_dataset)
epoch_accuracy = running_accuracy / len(train_dataset)
print('Loss:{:.4f} , Accuracy : {:.4f} '.format(epoch_loss, epoch_accuracy))
return model
You should probably use torch.argmax to get the class predictions from your model output, instead of torch.max.
Assuming you are working with indices as labels. Something like the following will get you the average accuracy of the current batch:
>>> outputs = torch.rand(16, 5)
>>> pred = torch.argmax(outputs, axis=0)
tensor([14, 11, 13, 15, 7])
>>> labels = torch.tensor([14, 6, 13, 5, 8])
>>> accuracy = (pred == labels).float().mean()
tensor(0.4000)
def show3D(searcher, grid_param_1, grid_param_2, name_param_1, name_param_2, rot=0):
scores_mean = searcher.cv_results_['mean_test_score']
scores_mean = np.array(scores_mean).reshape(len(grid_param_2), len(grid_param_1))
scores_sd = searcher.cv_results_['std_test_score']
scores_sd = np.array(scores_sd).reshape(len(grid_param_2), len(grid_param_1))
print('Best params = {}'.format(searcher.best_params_))
print('Best score = {}'.format(scores_mean.max()))
_, ax = plt.subplots(1,1)
# Param1 is the X-axis, Param 2 is represented as a different curve (color line)
for idx, val in enumerate(grid_param_2):
ax.plot(grid_param_1, scores_mean[idx, :], '-o', label=name_param_2 + ': ' + str(val))
ax.tick_params(axis='x', rotation=rot)
ax.set_title('Grid Search Scores')
ax.set_xlabel(name_param_1)
ax.set_ylabel('CV score')
ax.legend(loc='best')
ax.grid('on')
from sklearn.linear_model import SGDClassifier
metrics = ['hinge', 'log', 'modified_huber', 'perceptron', 'huber', 'epsilon_insensitive']
penalty = ['l2', 'l1', 'elasticnet']
searcher = GridSearchCV(SGDClassifier(max_iter=10000), {'loss': metrics,
'penalty': penalty},
scoring='roc_auc')
searcher.fit(train_x, train_y)
show3D(searcher, metrics, penalty, 'loss', 'penalty', 80)
searcher.cv_results_['mean_test_score']
The graph shows that the optimal value is huber + l2, however best_params gives a different result, how can this be? The plotting seems to be right, took from here: How to graph grid scores from GridSearchCV?
The best_params are correct, as they come from searcher.best_params_. The show3D must be updated as the cv results are wrongly assigned to params:
def show3D(searcher, grid_param_1, grid_param_2, name_param_1, name_param_2, rot=0):
scores_mean = searcher.cv_results_['mean_test_score']
scores_mean = np.array(scores_mean).reshape(len(grid_param_1), len(grid_param_2)).T
print('Best params = {}'.format(searcher.best_params_))
print('Best score = {}'.format(scores_mean.max()))
_, ax = plt.subplots(1,1)
# Param1 is the X-axis, Param 2 is represented as a different curve (color line)
for idx, val in enumerate(grid_param_2):
ax.plot(grid_param_1, scores_mean[idx, :], '-o', label=name_param_2 + ': ' + str(val))
ax.tick_params(axis='x', rotation=rot)
ax.set_title('Grid Search Scores')
ax.set_xlabel(name_param_1)
ax.set_ylabel('CV score')
ax.legend(loc='best')
ax.grid('on')
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.datasets import make_classification
train_x, train_y = make_classification(10000,10,2)
grid_param_1 = ['hinge', 'log', 'modified_huber', 'perceptron', 'huber', 'epsilon_insensitive']
grid_param_2 = ['l2', 'l1', 'elasticnet']
searcher = GridSearchCV(SGDClassifier(max_iter=10000), param_grid = {'loss': grid_param_1,
'penalty': grid_param_2},
scoring='roc_auc')
searcher.fit(train_x, train_y)
searcher.best_params_
show3D(searcher, grid_param_1, grid_param_2, 'loss', 'penalty', 80)
searcher.cv_results_['mean_test_score']
Best params = {'loss': 'huber', 'penalty': 'elasticnet'}
Best score = 0.9730321844671845
array([0.97055738, 0.97121098, 0.97126158, 0.97163018, 0.97188638,
0.97186598, 0.96557938, 0.97176798, 0.97196198, 0.95864618,
0.96608918, 0.92235953, 0.96921638, 0.97070898, 0.97303218,
0.96587218, 0.97211978, 0.96902218])
A bit ugly manual proof that params {'loss': 'huber', 'penalty': 'elasticnet'} produce highest cv results indeed:
searcher.cv_results_['params'][np.argmax(searcher.cv_results_['mean_test_score'])]
{'loss': 'huber', 'penalty': 'elasticnet'}
I build a Neural Network with two hidden layer.
from collections import namedtuple
def multilayer_perceptron():
tf.reset_default_graph()
inputs = tf.placeholder(tf.float32, shape=[None,train_x.shape[1]])
y = tf.placeholder(tf.float32, shape=[None, 1])
weights = {
'h1': tf.Variable(tf.random_normal([train_x.shape[1], n_hidden_1])),
'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
'out': tf.Variable(tf.random_normal([n_hidden_2, 1]))
}
biases = {
'b1': tf.Variable(tf.random_normal([n_hidden_1])),
'b2': tf.Variable(tf.random_normal([n_hidden_2])),
'out': tf.Variable(tf.random_normal([1]))
}
# Hidden layer con funzione di attivazione ReLU
layer_1 = tf.add(tf.matmul(inputs, weights['h1']), biases['b1'])
layer_1 = tf.nn.relu(layer_1)
# Hidden layer with ReLU activation
layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
layer_2 = tf.nn.relu(layer_2)
# Output layer with linear activation
out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
learning_rate = tf.placeholder(tf.float32)
is_training=tf.Variable(True,dtype=tf.bool)
cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y,logits=out_layer )
cost = tf.reduce_mean(cross_entropy)
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
predicted = tf.nn.sigmoid(out_layer)
correct_pred = tf.equal(tf.round(predicted), y)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Export the nodes
export_nodes = ['inputs', 'y', 'learning_rate','is_training', 'out_layer',
'cost', 'optimizer', 'predicted', 'accuracy']
Graph = namedtuple('Graph', export_nodes)
local_dict = locals()
graph = Graph(*[local_dict[each] for each in export_nodes])
return graph
pred1 = multilayer_perceptron()
Next I create a function for determinate the batch for input and output value:
def get_batch(data_x,data_y,batch_size=32):
batch_n=len(data_x)//batch_size
for i in range(batch_n):
batch_x=data_x[i*batch_size:(i+1)*batch_size]
batch_y=data_y[i*batch_size:(i+1)*batch_size]
yield batch_x,batch_y
epochs = 25
train_collect = 20
train_print=train_collect*2
learning_rate_value = 0.001
batch_size=400
x_collect = []
train_loss_collect = []
train_acc_collect = []
valid_loss_collect = []
valid_acc_collect = []
saver = tf.train.Saver()
Finally I launch the session that print a Loss function and accuracy of train model. I save the result on file .ckpt
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
iteration=0
for e in range(epochs):
for batch_x,batch_y in get_batch(train_x,train_y,batch_size):
iteration+=1
feed = {pred1.inputs: train_x,
pred1.y: train_y,
pred1.learning_rate: learning_rate_value,
pred1.is_training:True
}
train_loss, _, train_acc = sess.run([pred1.cost, pred1.optimizer, pred1.accuracy], feed_dict=feed)
if iteration % train_collect == 0:
x_collect.append(e)
train_loss_collect.append(train_loss)
train_acc_collect.append(train_acc)
if iteration % train_print==0:
print("Epoch: {}/{}".format(e + 1, epochs),
"Train Loss: {:.4f}".format(train_loss),
"Train Acc: {:.4f}".format(train_acc))
feed = {pred1.inputs: valid_x,
pred1.y: valid_y,
pred1.is_training:False
}
val_loss, val_acc = sess.run([pred1.cost, pred1.accuracy], feed_dict=feed)
valid_loss_collect.append(val_loss)
valid_acc_collect.append(val_acc)
if iteration % train_print==0:
print("Epoch: {}/{}".format(e + 1, epochs),
"Validation Loss: {:.4f}".format(val_loss),
"Validation Acc: {:.4f}".format(val_acc))
saver.save(sess, "./insurance2.ckpt")
When I launch the session for data set the code give me an error:
model=multilayer_perceptron()
restorer=tf.train.Saver()
with tf.Session() as sess:
restorer.restore(sess,"./insurance2.ckpt")
feed={
pred1.inputs:test_data,
pred1.is_training:False
}
test_predict=sess.run(pred1.predicted,feed_dict=feed)
The two error are:
ValueError: Tensor Tensor("Placeholder:0", shape=(?, 125), dtype=float32) is not an element of this graph.
TypeError: Cannot interpret feed_dict key as Tensor: Tensor Tensor("Placeholder:0", shape=(?, 125), dtype=float32) is not an element of this graph.
In my graph I export input.
I am new to TensorFlow and need to implement a deep neural network for a regression task. I assume there are no such sample codes on the internet where regression is performed using deep neural network (at least I could not find any. Please post any helpful link, if available). So, I have tried to merge the tutorials on deep neural networks for classification and regression together for my purpose. As expected, I am bombarded with errors. The error message reads:
InvalidArgumentError: In[0] is not a matrix
[[Node: MatMul_35 = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/cpu:0"](_recv_Placeholder_36_0, Variable_72/read)]]
The code:
import tensorflow as tf
import numpy
import matplotlib.pyplot as plt
n_nodes_hl1 = 100
n_nodes_hl2 = 100
batch_size = 100
n_input = 1;
n_output = 1;
learning_rate = 0.01
train_X = numpy.asarray([3.3,4.4,5.5,6.71,6.93,4.168,9.779,6.182,7.59,2.167,
7.042,10.791,5.313,7.997,5.654,9.27,3.1])
train_Y = numpy.asarray([1.7,2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221,
2.827,3.465,1.65,2.904,2.42,2.94,1.3])
x = tf.placeholder('float')
y = tf.placeholder('float')
def neural_network_model(data):
hidden_1_layer = {'weights':tf.Variable(tf.random_normal([n_input, n_nodes_hl1])),
'biases':tf.Variable(tf.random_normal([n_nodes_hl1]))}
hidden_2_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])),
'biases':tf.Variable(tf.random_normal([n_nodes_hl2]))}
l1 = tf.add(tf.matmul(data,hidden_1_layer['weights']), hidden_1_layer['biases'])
l1 = tf.nn.relu(l1)
l2 = tf.add(tf.matmul(l1,hidden_2_layer['weights']), hidden_2_layer['biases'])
l2 = tf.nn.relu(l2)
output = tf.reduce_sum(l2)
return output
def train_neural_network(x):
prediction = neural_network_model(x)
cost = tf.square(y - prediction)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
hm_epochs = 5
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(hm_epochs):
epoch_loss = 0
for (X, Y) in zip(train_X, train_Y):
_, c = sess.run([optimizer, cost], feed_dict={x: X, y: Y})
epoch_loss += c
print('Epoch', epoch, 'completed out of',hm_epochs,'loss:',epoch_loss)
plt.plot(train_X, train_Y, 'ro', label='Original data')
plt.plot(train_X, prediction, label='Fitted line')
plt.legend()
plt.show()
test_X = numpy.asarray([6.83, 4.668, 8.9, 7.91, 5.7, 8.7, 3.1, 2.1])
test_Y = numpy.asarray([1.84, 2.273, 3.2, 2.831, 2.92, 3.24, 1.35, 1.03])
print("Testing Data")
correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
print('Accuracy:',accuracy.eval({x:test_X, y:test_Y}))
train_neural_network(x)
As far I guess there is an issue with the dimensions of the hidden layer weights and/or biases (I may be wrong).
Side note: Here I have just tried to make a simple model of my project where the training and testing data points have been taken from the internet examples. My actual data would be pixel values of several images.
Change this line (working for me) :
Inputs to matmul() functions should be a matrix - you are feeding a value.
_, c = sess.run([optimizer, cost], feed_dict={x: [[X]], y: [[Y]]})
Output:
('Epoch', 0, 'completed out of', 5, 'loss:', array([[ 1.20472407e+14]], dtype=float32))
('Epoch', 1, 'completed out of', 5, 'loss:', array([[ 6.82631159]], dtype=float32))
('Epoch', 2, 'completed out of', 5, 'loss:', array([[ 8.83840561]], dtype=float32))
('Epoch', 3, 'completed out of', 5, 'loss:', array([[ 8.00222397]], dtype=float32))
('Epoch', 4, 'completed out of', 5, 'loss:', array([[ 7.6564579]], dtype=float32))
Hope this helps !
Comment: This is not a good example to explore if you're going to work
with images.