KeyError when trying to fine tuning Bert for text classification - pytorch

Question

I am trying to fine tune Bert for text classification on my dataset and I am getting the following error:
KeyError: 'Indexing with integers (to access backend Encoding for a given batch index) is not available when using Python based tokenizers'
Here is the full error:
1/1 * Epoch (train): 0% 0/613 [00:00<?, ?it/s]Traceback (most recent call last):
File "train.py", line 47, in <module>
runner.train(
File "/usr/local/lib/python3.8/dist-packages/catalyst/runners/runner.py", line 377, in train
self.run()
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 422, in run
self._run_event("on_exception")
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 365, in _run_event
getattr(self, event)(self)
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 357, in on_exception
raise self.exception
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 419, in run
self._run()
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 410, in _run
self.engine.spawn(self._run_local)
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/engine.py", line 59, in spawn
return fn(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 405, in _run_local
self._run_experiment()
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 399, in _run_experiment
self._run_epoch()
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 391, in _run_epoch
self._run_loader()
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 384, in _run_loader
self._run_event("on_batch_start")
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 361, in _run_event
getattr(self, event)(self)
File "/usr/local/lib/python3.8/dist-packages/catalyst/runners/supervised.py", line 150, in on_batch_start
super().on_batch_start(runner)
File "/usr/local/lib/python3.8/dist-packages/catalyst/core/runner.py", line 321, in on_batch_start
self.batch_size = len(self.batch[0])
File "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py", line 241, in __getitem__
raise KeyError(
KeyError: 'Indexing with integers (to access backend Encoding for a given batch index) is not available when using Python based tokenizers'
The code I am using for data preparation:
import logging
from pathlib import Path
from typing import List, Mapping, Tuple
import pandas as pd
import torch
from catalyst.utils import set_global_seed
from torch.utils.data import DataLoader, Dataset
from transformers import AutoTokenizer
class TextClassificationDataset(Dataset):
"""
Wrapper around Torch Dataset to perform text classification
"""
def __init__(
self,
texts: List[str],
labels: List[str] = None,
label_dict: Mapping[str, int] = None,
max_seq_length: int = 512,
model_name: str = "GroNLP/hateBERT",
):
"""
Args:
texts (List[str]): a list with texts to classify or to train the
classifier on
labels List[str]: a list with classification labels (optional)
label_dict (dict): a dictionary mapping class names to class ids,
to be passed to the validation data (optional)
max_seq_length (int): maximal sequence length in tokens,
texts will be stripped to this length
model_name (str): transformer model name, needed to perform
appropriate tokenization
"""
self.texts = texts
self.labels = labels
self.label_dict = label_dict
self.max_seq_length = max_seq_length
if self.label_dict is None and labels is not None:
# {'class1': 0, 'class2': 1, 'class3': 2, ...}
# no easily handle unknown target values
self.label_dict = dict(zip(sorted(set(labels)), range(len(set(labels)))))
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
# suppresses tokenizer warnings
logging.getLogger("transformers.tokenization_utils").setLevel(logging.FATAL)
def __len__(self) -> int:
"""
Returns:
int: length of the dataset
"""
return len(self.texts)
def __getitem__(self, index) -> Mapping[str, torch.Tensor]:
"""Gets element of the dataset
Args:
index (int): index of the element in the dataset
Returns:
Single element by index
"""
# encoding the text
x = self.texts[index]
# a dictionary with `input_ids` and `attention_mask` as keys
output_dict = self.tokenizer.encode_plus(
x,
add_special_tokens=True,
padding="max_length",
max_length=self.max_seq_length,
return_tensors="pt",
truncation=True,
return_attention_mask=True,
)
# for Catalyst, there needs to be a key called features
output_dict["features"] = output_dict["input_ids"].squeeze(0)
del output_dict["input_ids"]
# encoding target
if self.labels is not None:
y = self.labels[index]
y_encoded = torch.Tensor([self.label_dict.get(y, -1)]).long().squeeze(0)
output_dict["targets"] = y_encoded
return output_dict
What is the problem?
I know questions about similar error have been already asked but they were not of much help in solving this problem.