DataLoader fails with T5 tokenizer - nlp

Question

I am trying to train a T5 model for NER problem, but whenever I try to load the data it fails.
Here is how I load the tokenizer:
tokenizer = AutoTokenizer.from_pretrained("t5-base")
training_set = dataset(train_dataset, tokenizer, MAX_LEN)
testing_set = dataset(test_dataset, tokenizer, MAX_LEN)
Then I created the DataLoad as follows:
train_params = {'batch_size': TRAIN_BATCH_SIZE,
'shuffle': True,
'num_workers': 0,
}
test_params = {'batch_size': VALID_BATCH_SIZE,
'shuffle': True,
'num_workers': 0
}
training_loader = DataLoader(training_set, **train_params)
testing_loader = DataLoader(testing_set, **test_params)
I was facing issues training the model so I tried to debug the loading as:
for idx, batch in enumerate(training_loader):
print(idx)
break
But I am getting this error:
<ipython-input-8-fa9e34b10c06> in __getitem__(self, index)
34 if mapping[0] == 0 and mapping[1] != 0:
35 # overwrite label
---> 36 encoded_labels[idx] = labels[i]
37 i += 1
38
IndexError: list index out of range
I checked the dataset, it is not empty and there is data, not sure what is wrong.
I tried the same code with BERT by changing this line:
tokenizer = AutoTokenizer.from_pretrained("t5-base")
To:
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
& it worked perfectly, so the issue is with working with T5
Here is the Dataset class:
class dataset(Dataset):
def __init__(self, dataframe, tokenizer, max_len):
self.len = len(dataframe)
self.data = dataframe
self.tokenizer = tokenizer
self.max_len = max_len
def __getitem__(self, index):
# step 1: get the sentence and word labels
sentence = self.data.sentence[index].strip().split()
word_labels = self.data.word_labels[index].split(",")
# step 2: use tokenizer to encode sentence (includes padding/truncation up to max length)
# BertTokenizerFast provides a handy "return_offsets_mapping" functionality for individual tokens
encoding = self.tokenizer(sentence,
is_split_into_words=True,
return_offsets_mapping=True,
truncation=True,
padding='max_length',
max_length=self.max_len)
#return_offsets_mapping=True,
#padding='max_length',
#truncation=True,)
# step 3: create token labels only for first word pieces of each tokenized word
labels = [labels_to_ids[label] for label in word_labels]
# code based on https://huggingface.co/transformers/custom_datasets.html#tok-ner
# create an empty array of -100 of length max_length
encoded_labels = np.ones(len(encoding["offset_mapping"]), dtype=int) * -100
# set only labels whose first offset position is 0 and the second is not 0
i = 0
for idx, mapping in enumerate(encoding["offset_mapping"]):
if mapping[0] == 0 and mapping[1] != 0:
# overwrite label
encoded_labels[idx] = labels[i]
i += 1
# step 4: turn everything into PyTorch tensors
item = {key: torch.as_tensor(val) for key, val in encoding.items()}
item['labels'] = torch.as_tensor(encoded_labels)
return item
def __len__(self):
return self.len