Issues calculating accuracy for custom BERT model - python-3.x

Question

I'm having some issues trying to calculate the accuracy of a custom BERT model which also uses the pretrained model from Huggingface. This is the code that I have :
import numpy as np
import pandas as pd
from sklearn import metrics, linear_model
import torch
from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
from transformers import BertTokenizer, BertModel
from torch import cuda
import re
import torch.nn as nn
device = 'cuda' if cuda.is_available() else 'cpu'
MAX_LEN = 200
TRAIN_BATCH_SIZE = 8 # 12, 64
VALID_BATCH_SIZE = 4
EPOCHS = 1
LEARNING_RATE = 1e-4 #3e-4, 1e-4, 5e-5, 3e-5
tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased')
file1 = open('test.txt', 'r')
list_com = []
list_label = []
for line in file1:
possible_labels = 'positive|negative'
label = re.findall(possible_labels, line)
line = re.sub(possible_labels, ' ', line)
line = re.sub('\n', ' ', line)
list_com.append(line)
list_label.append(label[0])
list_tuples = list(zip(list_com, list_label))
file1.close()
labels = ['positive', 'negative']
df = pd.DataFrame(list_tuples, columns=['review', 'sentiment'])
df['sentiment'] = df['sentiment'].map({'positive': 1, 'negative': 0})
for i in range(0,len(df['sentiment'])):
list_label[i] = df['sentiment'][i]
#print(df)
class CustomDataset(Dataset):
def __init__(self, dataframe, tokenizer, max_len):
self.tokenizer = tokenizer
self.data = dataframe
self.comment_text = dataframe.review
self.targets = self.data.sentiment
self.max_len = max_len
def __len__(self):
return len(self.comment_text)
def __getitem__(self, index):
comment_text = str(self.comment_text[index])
comment_text = " ".join(comment_text.split())
inputs = self.tokenizer.encode_plus(comment_text,None,add_special_tokens=True,max_length=self.max_len,
pad_to_max_length=True,return_token_type_ids=False,truncation=True)
ids = inputs['input_ids']
mask = inputs['attention_mask']
return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'targets': torch.tensor(self.targets[index], dtype=torch.float)
}
train_size = 0.8
train_dataset=df.sample(frac=train_size,random_state=200)
test_dataset=df.drop(train_dataset.index).reset_index(drop=True)
train_dataset = train_dataset.reset_index(drop=True)
print("FULL Dataset: {}".format(df.shape))
print("TRAIN Dataset: {}".format(train_dataset.shape))
print("TEST Dataset: {}".format(test_dataset.shape))
training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN)
testing_set = CustomDataset(test_dataset, tokenizer, MAX_LEN)
train_params = {'batch_size': TRAIN_BATCH_SIZE,'shuffle': True,'num_workers': 0}
test_params = {'batch_size': VALID_BATCH_SIZE,'shuffle': True,'num_workers': 0}
training_loader = DataLoader(training_set, **train_params)
testing_loader = DataLoader(testing_set, **test_params)
class BERTClass(torch.nn.Module):
def __init__(self):
super(BERTClass, self).__init__()
self.bert = BertModel.from_pretrained('bert-base-multilingual-uncased',return_dict=False,num_labels = 2)
self.lstm = nn.LSTM(768, 256, batch_first=True, bidirectional=True)
self.linear = nn.Linear(256*2,2)
def forward(self, ids , mask):
sequence_output, pooled_output = self.bert(ids, attention_mask=mask )
lstm_output, (h, c) = self.lstm(sequence_output) ## extract the 1st token's embeddings
hidden = torch.cat((lstm_output[:, -1, :256], lstm_output[:, 0, 256:]), dim=-1)
linear_output = self.linear(lstm_output[:, -1].view(-1, 256 * 2))
return linear_output
model = BERTClass()
model.to(device)
#print(model)
def loss_fn(outputs, targets):
return torch.nn.CrossEntropyLoss()(outputs, targets)
optimizer = torch.optim.Adam(params = model.parameters(), lr=LEARNING_RATE)
def train(epoch):
model.train()
for _, data in enumerate(training_loader, 0):
ids = data['ids'].to(device, dtype=torch.long)
mask = data['mask'].to(device, dtype=torch.long)
targets = data['targets'].to(device, dtype=torch.long)
outputs = model(ids, mask)
optimizer.zero_grad()
loss = loss_fn(outputs, targets)
if _ % 1000 == 0:
print(f'Epoch: {epoch}, Loss: {loss.item()}')
optimizer.zero_grad()
loss.backward()
optimizer.step()
for epoch in range(EPOCHS):
train(epoch)
def validation(epoch):
model.eval()
fin_targets = []
fin_outputs = []
with torch.no_grad():
for _, data in enumerate(testing_loader, 0):
ids = data['ids'].to(device, dtype=torch.long)
mask = data['mask'].to(device, dtype=torch.long)
targets = data['targets'].to(device, dtype=torch.float)
outputs = model(ids, mask)
fin_targets.extend(targets.cpu().detach().numpy().tolist())
fin_outputs.extend(torch.sigmoid(outputs).cpu().detach().numpy().tolist())
return fin_outputs, fin_targets
for epoch in range(EPOCHS):
outputs, targets = validation(epoch)
outputs = np.array(outputs) >= 0.5
accuracy = metrics.accuracy_score(targets, outputs)
print(f"Accuracy Score = {accuracy}")
torch.save(model.state_dict, 'model.pt')
print(f'Model saved!')
It should be a binary classification, positive(1) or negative(0), but when i try to compute the accuracy i get the error ValueError: Classification metrics can't handle a mix of binary and multilabel-indicator targets oh this line accuracy = metrics.accuracy_score(targets, outputs) .The outputs look like this:
[[ True False]
[False False]
[ True False]
[ True False]
[ True False]
[ True False]
[ True False]
[False True]
[ True False]
[ True False]
[False True]]
Can someone advise what would be the fix to this? Or if there something else that can improve this? Also, I saved the model and I want to know how can I use the saved model to classify user input in another .py file?(assuming that we enter a sentence from keyboard and we want the model to classify it).