I am running a neural network with logging of training accuracy,Validation accuracy and validation loss. here is my code snippet.
def show_progress(epoch, feed_dict_train, feed_dict_validate, val_loss):
acc = session.run(accuracy, feed_dict=feed_dict_train)
val_acc = session.run(accuracy, feed_dict=feed_dict_validate)
msg = "Training Epoch {0} --- Training Accuracy: {1:>6.1%}, Validation Accuracy: {2:>6.1%}, Validation Loss: {3:.3f}"
print(msg.format(epoch + 1, acc, val_acc, val_loss))
return acc,val_acc
total_iterations = 0
#writer=tf.summary.FileWriter(options.tensorboard,session)
saver = tf.train.Saver()
def train(num_iteration):
global total_iterations
writer=tf.summary.FileWriter(options.tensorboard,session.graph)
#global writer
for i in range(total_iterations,
total_iterations + num_iteration):
x_batch, y_true_batch, _, cls_batch = data.train.next_batch(batch_size)
x_valid_batch, y_valid_batch, _, valid_cls_batch = data.valid.next_batch(batch_size)
feed_dict_tr = {x: x_batch,
y_true: y_true_batch}
feed_dict_val = {x: x_valid_batch,
y_true: y_valid_batch}
session.run(optimizer, feed_dict=feed_dict_tr)
if i % 10 == 0:
val_loss = session.run(cost, feed_dict=feed_dict_val)
epoch = int(i /10)
accu,valid_accu=show_progress(epoch, feed_dict_tr, feed_dict_val, val_loss)
#getting values for visualising inside the tensorboard
tf.summary.scalar("training_accuracy",accu)
tf.summary.scalar("Validation_accuracy",valid_accu)
tf.summary.scalar("Validation_loss",val_loss)
#tf.summary.scalar("epoch",epoch)
#merging all the values (serializing)
merged=tf.summary.merge_all()
summary=session.run(merged)
#adding them to the events directory
writer.add_summary(summary,epoch)
saver.save(session, options.save)
total_iterations += num_iteration
train(num_iteration=10)
Now I am getting a tensor board output, as for each epoch the accuracy,validation accuracy and validation loss as separate plots with single points.
For each epoch I am getting these three plots again with another point.
I want to get a continuous points for these three plots so as it forms a line graph.
Each of your call to tf.summary.scalar() will create a node in the computation graph. Specifically, in your code, the calls are inside the training loop and therefore metrics at different epochs get written to different plots.
tf.summary.scalar("training_accuracy", accu)
tf.summary.scalar("Validation_accuracy", valid_accu)
tf.summary.scalar("Validation_loss", val_loss)
What you can do is to define the summary ops before the loop with placeholders. Then, in the eval loop, you can feed these tensor with real values.
# Define a placeholder and wire it to the summary op.
accu_tensor = tf.placeholder(tf.float32)
tf.summary.scalar("training_accuracy", accu_tensor)
summary_op = tf.summary.merge_all()
# Create a session after defining ops.
sess = tf.Session()
writer = tf.summary.FileWriter(<some-directory>, sess.graph)
for i in range(total_iterations,
total_iterations + num_iteration):
# run training ops to get values for accu
# ...
# run the summary op with a feed_dict to feed the value.
summaries = sess.run(summary_op, feed_dict={accu_tensor: accu})
writer.add_summary(summaries, epoch)
Related
I am working on textclassification with transformer models (PyTorch, Huggingface, running on GPU).
I have already my model and my training loop and it works fine but to better understand wrong predictions of the model, I want to "dig a little deeper" and look at the attention weights the model gave the tokens of the misclassified predictions (for evaluation during training and later for predictions on the test set).
I tried already so many things and I already found a way the model outputs attention weights (will attach an example), BUT a) it outputs too many attention weights for each misclassified example (can it be because it gives the weights for several layers?) and b) the attention weights are not connected to the tokens. I want it in a more interpretable way, like coloring the tokens in a darker color the more weight is on that token.
This is my code so far (irrelevant snippets left out):
model = AutoModelForSequenceClassification.from_pretrained(checkpoint,
num_labels=len(label_dict),
output_attentions=True,
output_hidden_states=True)
def get_wrong_predictions(predictions, true_vals):
wrong_predictions = []
for i in range(len(true_vals)):
if np.argmax(predictions[i]) != true_vals[i]:
wrong_predictions.append((true_vals[i], np.argmax(predictions[i])))
return wrong_predictions
seed_val = 17
random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)
def evaluate(dataloader_val):
model.eval()
loss_val_total = 0
predictions, true_vals = [], []
attentions = []
for batch in dataloader_val:
batch = tuple(b.to(device) for b in batch)
inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'labels': batch[2],
}
with torch.no_grad():
outputs = model(**inputs)
loss = outputs[0]
logits = outputs[1]
attention_scores = outputs.attentions[-1].detach().cpu().numpy() # extract the attention scores from the last layer
loss_val_total += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = inputs['labels'].cpu().numpy()
predictions.append(logits)
true_vals.append(label_ids)
attentions.append(attention_scores)
loss_val_avg = loss_val_total/len(dataloader_val)
predictions = np.concatenate(predictions, axis=0)
true_vals = np.concatenate(true_vals, axis=0)
attentions = np.concatenate(attentions, axis=0)
preds = np.argmax(predictions, axis=1)
misclassified = np.where(preds != true_vals)[0]
texts = df["description"].tolist()
global misclassified_examples
misclassified_examples = []
#misclassified = []
for idx in misclassified[:min(len(attention_scores), len(misclassified))]:
text = texts[idx]
true_label = true_vals[idx]
pred_label = preds[idx]
attention_weights = attentions[idx] # get the attention weights for the misclassified examples
misclassified_examples.append({
'text': text,
'true_label': true_label,
'pred_label': pred_label,
'attention_weights' : attention_weights
})
return loss_val_avg, predictions, true_vals
train_losses = [] #to plot later
val_losses = []
overall_accuracy = []
for epoch in tqdm(range(1, epochs+1)):
model.train()
loss_train_total = 0
progress_bar = tqdm(dataloader_train, desc='Epoch {:1d}'.format(epoch), leave=False, disable=False)
for batch in progress_bar:
model.zero_grad()
batch = tuple(b.to(device) for b in batch)
inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'labels': batch[2],
}
outputs = model(**inputs)
loss = outputs[0]
loss_train_total += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
scheduler.step()
progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item()/len(batch))})
loss_train_avg = loss_train_total/len(dataloader_train) #to plot later
train_losses.append(loss_train_avg) # to plot later
tqdm.write(f'\nEpoch {epoch}')
loss_train_avg = loss_train_total/len(dataloader_train)
tqdm.write(f'Training loss: {loss_train_avg}')
val_loss, predictions, true_vals = evaluate(dataloader_validation)
val_f1 = f1_score_func(predictions, true_vals)
tqdm.write(f'Validation loss: {val_loss}')
tqdm.write(f'F1 Score (Weighted): {val_f1}')
val_overallaccuracy = overallaccuracy(true_vals, predictions)
tqdm.write(f'Overall Accuracy: {val_overallaccuracy}')
val_mcc = mcc_score_func(true_vals, predictions)
tqdm.write(f'MCC: {val_mcc}')
val_acc = accuracy_per_class(predictions, true_vals)
tqdm.write(f'Accuracy per class: {val_acc}')
val_losses.append(val_loss)
overall_accuracy.append(val_overallaccuracy)
The image shows exemplarily what kind of output I get, but that is "too much" attention as I expected one number for one token and then of course some way to relate that number of attention weight back to that token.
[enter image description here](https://i.stack.imgur.com/ICmvp.png)
Does anyone know how I could adjust my code to include attention weight alignment for the tokens? I would be VERY thankful!!
I've been looking through samples but am unable to understand how to integrate the precision, recall and f1 metrics for my model. My code is as follows:
for epoch in range(num_epochs):
#Calculate Accuracy (stack tutorial no n_total)
n_correct = 0
n_total = 0
for i, (words, labels) in enumerate(train_loader):
words = words.to(device)
labels = labels.to(dtype=torch.long).to(device)
# Forward pass
outputs = model(words)
loss = criterion(outputs, labels)
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
#feedforward tutorial solution
_, predicted = torch.max(outputs, 1)
n_correct += (predicted == labels).sum().item()
n_total += labels.shape[0]
accuracy = 100 * n_correct/n_total
#Push to matplotlib
train_losses.append(loss.item())
train_epochs.append(epoch)
train_acc.append(accuracy)
#Loss and Accuracy
if (epoch+1) % 10 == 0:
print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item():.2f}, Acc: {accuracy:.2f}')
Since you have the predicted and the labels variables, you can aggregate them during the epoch loop and convert them to numpy arrays to calculate the required metrics.
At the beginning of the epoch, initialize two empty lists; one for true labels and one for ground truth labels.
for epoch in range(num_epochs):
predicted_labels, ground_truth_labels = [], []
...
Then, keep appending the respective entries to each list during the epoch:
...
_, predicted = torch.max(outputs, 1)
n_correct += (predicted == labels).sum().item()
# appending
predicted_labels.append(predicted.cpu().detach().numpy())
ground_truth_labels.append(labels.cpu().detach().numpy())
...
Then, at the epoch end, you could use precision_recall_fscore_support with predicted_labels and ground_truth_labels as inputs.
Notes:
You'll probably have to refer something like this to flatten the above two lists.
Read about torch.no_grad() to apply it as a good practice during the calculations of metrics.
I'm using a VGG16 model written in tf2.0 to train on my own datasets. Some BatchNormalization layers were included in the model and the "training" argument were set to True during training time and False during validation time as described in many tutorials.
The train_loss decreased to a certain level during training as expected. However, the val_loss behaves really strange. I checked out the output of the model after training and found out that, if I set the training argument to True, the output is quite correct, but if I set it to False, the result is incorrect at all.
According to the tutorials in tensorflow website, when training is set to False , the model will normalize its inputs using the mean and variance of its moving statistics learned during training but it doesn't seem so. Am I missing something?
I've provided the trainning and validation code in the below.
def train():
logging.basicConfig(level=logging.INFO)
tdataset = tf.data.Dataset.from_tensor_slices((train_img_list[:200], train_label_list[:200]))
tdataset = tdataset.map(parse_function, 3).shuffle(buffer_size=200).batch(batch_size).repeat(repeat_times)
vdataset = tf.data.Dataset.from_tensor_slices((val_img_list[:100], val_label_list[:100]))
vdataset = vdataset.map(parse_function, 3).batch(batch_size)
### Vgg model
model = VGG_PR(num_classes=num_label)
logging.info('Model loaded')
start_epoch = 0
latest_ckpt = tf.train.latest_checkpoint(os.path.dirname(ckpt_path))
if latest_ckpt:
start_epoch = int(latest_ckpt.split('-')[1].split('.')[0])
model.load_weights(latest_ckpt)
logging.info('model resumed from: {}, start at epoch: {}'.format(latest_ckpt, start_epoch))
else:
logging.info('training from scratch since weights no there')
######## training loop ########
loss_object = tf.keras.losses.MeanSquaredError()
val_loss_object = tf.keras.losses.MeanSquaredError()
optimizer = tf.keras.optimizers.Adam(learning_rate=initial_lr)
train_loss = tf.metrics.Mean(name='train_loss')
val_loss = tf.metrics.Mean(name='val_loss')
writer = tf.summary.create_file_writer(log_path.format(case_num))
with writer.as_default():
for epoch in range(start_epoch, total_epoch):
print('start training')
try:
for batch, data in enumerate(tdataset):
images, labels = data
with tf.GradientTape() as tape:
pred = model(images, training=True)
if len(pred.shape) == 2:
pred = tf.reshape(pred,[-1, 1, 1, num_label])
loss = loss_object(pred, labels)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
if batch % 20 ==0:
logging.info('Epoch: {}, iter: {}, loss:{}'.format(epoch, batch, loss.numpy()))
tf.summary.scalar('train_loss', loss.numpy(), step=epoch*1250*repeat_times+batch) # the tdataset has been repeated 5 times..
tf.summary.text('Zernike_coe_pred', tf.as_string(tf.squeeze(pred)), step=epoch*1250*repeat_times+batch)
tf.summary.text('Zernike_coe_gt', tf.as_string(tf.squeeze(labels)), step=epoch*1250*repeat_times+batch)
writer.flush()
train_loss(loss)
model.save_weights(ckpt_path.format(epoch=epoch))
except KeyboardInterrupt:
logging.info('interrupted.')
model.save_weights(ckpt_path.format(epoch=epoch))
logging.info('model saved into {}'.format(ckpt_path.format(epoch=epoch)))
exit(0)
# validation step
for batch, data in enumerate(vdataset):
images, labels = data
val_pred = model(images, training=False)
if len(val_pred.shape) == 2:
val_pred = tf.reshape(val_pred,[-1, 1, 1, num_label])
v_loss = val_loss_object(val_pred, labels)
val_loss(v_loss)
logging.info('Epoch: {}, average train_loss:{}, val_loss: {}'.format(epoch, train_loss.result(), val_loss.result()))
tf.summary.scalar('val_loss', val_loss.result(), step = epoch)
writer.flush()
train_loss.reset_states()
val_loss.reset_states()
model.save_weights(ckpt_path.format(epoch=epoch))
The train losss reduced to a very small value like the groundtruth label are in the range of [0, 1] and the average train loss can be 0.007, but the val loss is much higher than this. The output of the model tends to be close to 0 if I set training to False.
updated on Nov. 6th:
I have found an interesting thing that if I use tf.function to decorate my model in its call method, the val loss will turn to be correct, but I'm not sure what has happened?
Mentioning the Answer for the benefit of the community.
Issue is resolved, i.e., val loss will turn to be correct if tf.function is used to decorate the model in its call method.
Trying to work with the framework provided in the course Stanford cs231n, given the code below.
I can see the accuracy getting better and the net is trained however after the training process and checking the results on the validation set, how would I go to input one image into the model and see its prediction?
I have searched around and couldn't find some built in predict function in tensorflow as there is in keras.
Initializing the net and its parameters
# clear old variables
tf.reset_default_graph()
# setup input (e.g. the data that changes every batch)
# The first dim is None, and gets sets automatically based on batch size fed in
X = tf.placeholder(tf.float32, [None, 30, 30, 1])
y = tf.placeholder(tf.int64, [None])
is_training = tf.placeholder(tf.bool)
def simple_model(X,y):
# define our weights (e.g. init_two_layer_convnet)
# setup variables
Wconv1 = tf.get_variable("Wconv1", shape=[7, 7, 1, 32]) # Filter of size 7x7 with depth of 3. No. of filters is 32
bconv1 = tf.get_variable("bconv1", shape=[32])
W1 = tf.get_variable("W1", shape=[4608, 360]) # 5408 is 13x13x32 where 13x13 is the output of 7x7 filter on 32x32 image with padding of 2.
b1 = tf.get_variable("b1", shape=[360])
# define our graph (e.g. two_layer_convnet)
a1 = tf.nn.conv2d(X, Wconv1, strides=[1,2,2,1], padding='VALID') + bconv1
h1 = tf.nn.relu(a1)
h1_flat = tf.reshape(h1,[-1,4608])
y_out = tf.matmul(h1_flat,W1) + b1
return y_out
y_out = simple_model(X,y)
# define our loss
total_loss = tf.losses.hinge_loss(tf.one_hot(y,360),logits=y_out)
mean_loss = tf.reduce_mean(total_loss)
# define our optimizer
optimizer = tf.train.AdamOptimizer(5e-4) # select optimizer and set learning rate
train_step = optimizer.minimize(mean_loss)
Function for evaluating the model whether for training or validation and plots the results:
def run_model(session, predict, loss_val, Xd, yd,
epochs=1, batch_size=64, print_every=100,
training=None, plot_losses=False):
# Have tensorflow compute accuracy
correct_prediction = tf.equal(tf.argmax(predict,1), y)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# shuffle indicies
train_indicies = np.arange(Xd.shape[0])
np.random.shuffle(train_indicies)
training_now = training is not None
# setting up variables we want to compute and optimize
# if we have a training function, add that to things we compute
variables = [mean_loss,correct_prediction,accuracy]
if training_now:
variables[-1] = training
# counter
iter_cnt = 0
for e in range(epochs):
# keep track of losses and accuracy
correct = 0
losses = []
# make sure we iterate over the dataset once
for i in range(int(math.ceil(Xd.shape[0]/batch_size))):
# generate indicies for the batch
start_idx = (i*batch_size)%Xd.shape[0]
idx = train_indicies[start_idx:start_idx+batch_size]
# create a feed dictionary for this batch
feed_dict = {X: Xd[idx,:],
y: yd[idx],
is_training: training_now }
# get batch size
actual_batch_size = yd[idx].shape[0]
# have tensorflow compute loss and correct predictions
# and (if given) perform a training step
loss, corr, _ = session.run(variables,feed_dict=feed_dict)
# aggregate performance stats
losses.append(loss*actual_batch_size)
correct += np.sum(corr)
# print every now and then
if training_now and (iter_cnt % print_every) == 0:
print("Iteration {0}: with minibatch training loss = {1:.3g} and accuracy of {2:.2g}"\
.format(iter_cnt,loss,np.sum(corr)/actual_batch_size))
iter_cnt += 1
total_correct = correct/Xd.shape[0]
total_loss = np.sum(losses)/Xd.shape[0]
print("Epoch {2}, Overall loss = {0:.3g} and accuracy of {1:.3g}"\
.format(total_loss,total_correct,e+1))
if plot_losses:
plt.plot(losses)
plt.grid(True)
plt.title('Epoch {} Loss'.format(e+1))
plt.xlabel('minibatch number')
plt.ylabel('minibatch loss')
plt.show()
return total_loss,total_correct
The functions calls that trains the model
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
print('Training')
run_model(sess,y_out,mean_loss,x_train,y_train,1,64,100,train_step,True)
print('Validation')
run_model(sess,y_out,mean_loss,x_val,y_val,1,64)
You do not need to go far, you simply pass your new (test) feature matrix X_test into your network and perform a forward pass - the output layer is the prediction. So the code is something like this
session.run(y_out, feed_dict={X: X_test})
I have linear regression model that seems to be working fine, but I want to display the accuracy of the model.
First, I initialize the variables and placeholders...
X_train, X_test, Y_train, Y_test = train_test_split(
X_data,
Y_data,
test_size=0.2
)
n_rows = X_train.shape[0]
X = tf.placeholder(tf.float32, [None, 89])
Y = tf.placeholder(tf.float32, [None, 1])
W_shape = tf.TensorShape([89, 1])
b_shape = tf.TensorShape([1])
W = tf.Variable(tf.random_normal(W_shape))
b = tf.Variable(tf.random_normal(b_shape))
pred = tf.add(tf.matmul(X, W), b)
cost = tf.reduce_sum(tf.pow(pred-Y, 2)/(2*n_rows-1))
optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate).minimize(cost)
X_train has shape (6702, 89) and Y_train has shape (6702, 1). Next I run the session and I display the cost per epoch as well as the total MSE...
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for epoch in range(FLAGS.training_epochs):
avg_cost = 0
for (x, y) in zip(X_train, Y_train):
x = np.reshape(x, (1, 89))
y = np.reshape(y, (1,1))
sess.run(optimizer, feed_dict={X:x, Y:y})
# display logs per epoch step
if (epoch + 1) % FLAGS.display_step == 0:
c = sess.run(
cost,
feed_dict={X:X_train, Y:Y_train}
)
y_pred = sess.run(pred, feed_dict={X:X_test})
test_error = r2_score(Y_test, y_pred)
print(test_error)
print("Epoch:", '%04d' % (epoch + 1), "cost=", "{:.9f}".format(c))
print("Optimization Finished!")
pred_y = sess.run(pred, feed_dict={X:X_test})
mse = tf.reduce_mean(tf.square(pred_y - Y_test))
print("MSE: %4f" % sess.run(mse))
This all seems to work correctly. However, now I want to see the accuracy of my model, so I want to implement tf.metrics.accuracy. The documentation says it has 2 arguments, labels and predictions. I added the following next...
accuracy, accuracy_op = tf.metrics.accuracy(labels=Y_test, predictions=pred)
init_local = tf.local_variables_initializer()
sess.run(init_local)
print(sess.run(accuracy))
Apparently I need to initialize local variales, however I think I am doing something wrong because the accuracy result that gets printed out is 0.0.
I searched everywhere for a working example but I cannot get it to work for my model, what is the proper way to implement it?
I think you are learning a regression model. The tf.metrics.accuracy is supposed to run for a classification model.
When your model predicts 1.2 but your target value is 1.15, it does not make sense to use accuracy to measure whether this is a correct prediction. accuracy is for classification problems (e.g., mnist), when your model predicts a digit to be '9' and your target image is also '9': this is a correct prediction and you get full credit; Or when your model predicts a digit to be '9' but your target image is '6': this is a wrong prediction and you get no credit.
For your regression problem, we measure the difference between prediction and target value either by absolute error - |target - prediction| or mean squared error - the one you used in your MSE calculation. Thus tf.metrics.mean_squared_error or tf.metrics.mean_absolute_error is the one you should use to measure the prediction error for regression models.