RNN prediction is zero - nlp

Question

I am trying to build a recurrent neural network to predict cities.
The data I am using is a compilation of 298 address that has two columns.
Column 1 is the full address which includes the following components:
Street Number, Street Name, City, State and Zip Code
Column 2 is the City.
An example of the dataset I am working with can be seen below:
Address
City
34416 Mike Station Davidmouth WY 15806
Davidmouth
7067 Nguyen Well Stevenmouth NE 92527
Stevenmouth
14420 Erika Springs Port Kristenborough MP 81670
Port Kristenborough
704 Steven Corners South George ND 44349
South George
168 Ashley Views Port Brianberg WA 2232
Port Brianberg
Below is the code I ran in google colab:
# Import libraries
import collections
import helper
import numpy as np
import pandas as pd
from keras.preprocessing.text import Tokenizer
from keras.utils import pad_sequences
from keras.models import Model
from keras.layers import GRU, Input, Dense, TimeDistributed, Activation, RepeatVector, Bidirectional
from keras.layers import Embedding
from keras.optimizers import Adam
from keras.losses import sparse_categorical_crossentropy
df = pd.read_excel("rnn_mt_addresses.xlsx")
df.head()
# select columns for addresses and cities
addresses_df = df.iloc[:, :-1]
cities_list = df.iloc[:, -1]
# change cities list into datafrmae
cities_df = pd.DataFrame(cities_list, columns=['City'])
addresses = [str(add) for add in addresses_df['Address']]
cities = [str(cit) for cit in cities_df['City']]
# count words in addresses and cities
addresses_words_counter = collections.Counter([word for address in addresses for word in address.split()])
cities_word_counter = collections.Counter([word for city in cities for word in city.split()])
# tokenize addresses
addresses_tokenizer = Tokenizer()
cities_tokenizer = Tokenizer()
addresses_tokenizer.fit_on_texts(addresses)
# addresses vocab size
len(addresses_tokenizer.word_index)
# tokenize cities
cities_tokenizer.fit_on_texts(cities)
# cities vocab size
len(cities_tokenizer.word_index)
# apply sequencing
addresses_sequences = addresses_tokenizer.texts_to_sequences(addresses)
cities_sequences = cities_tokenizer.texts_to_sequences(cities)
# calculate addresses vocabulary size
addresses_vocab_size = len(addresses_tokenizer.word_index)
print(f"The addresses vocabulary size is {len(addresses_tokenizer.word_index)}")
The addresses vocabulary size is 1266
# calculate cities vocabulary size
cities_vocab_size = len(cities_tokenizer.word_index)
print(f"The cities vocabulary size is {len(cities_tokenizer.word_index)}")
The cities vocabulary size is 286
# calculate addresses and cities max length
max_addresses_sequence_length = max([len(address) for address in addresses_sequences])
max_addresses_sequence_length
7
max_cities_sequence_length = max([len(city) for city in cities_sequences])
max_cities_sequence_length
2
# pad cities and addresses
addresses_sequences_pd = pad_sequences(addresses_sequences,maxlen = max_addresses_sequence_length, padding = 'post')
cities_sequences_pd = pad_sequences(cities_sequences, maxlen = max_addresses_sequence_length, padding = 'post')
# reshape addresses_sequences_pd and cities_sequences_pd to 3D tensors
addresses_sequences_pd = addresses_sequences_pd.reshape(addresses_sequences_pd.shape[0], addresses_sequences_pd.shape[1],1)
cities_sequences_pd = cities_sequences_pd.reshape(cities_sequences_pd.shape[0], cities_sequences_pd.shape[1],1)
addresses_sequences_pd.shape
(298, 7, 1)
cities_sequences_pd.shape
(298, 7, 1)
# define logits function
def logits_to_text(logits, tokenizer):
"""
Turn logits from a neural network into text using the tokenizer
:param logits: Logits from a neural network
:param tokenizer: Keras Tokenizer fit on the labels
:return: String that represents the text of the logits
"""
index_to_words = {id: word for word, id in tokenizer.word_index.items()}
index_to_words[0] = '<PAD>'
return ' '.join([index_to_words[prediction] for prediction in np.argmax(logits, 1)])
from keras.layers import GRU, Input, Dense, TimeDistributed, Dropout, LSTM
from keras.models import Model, Sequential
from keras.layers import Activation
from keras.optimizers import Adam
from keras.losses import sparse_categorical_crossentropy
# TODO: Build the layers
model = Sequential()
model.add(GRU(128, input_shape=addresses_sequences_pd.shape[1:], return_sequences=True))
model.add(Dropout(0.5))
model.add(GRU(128, return_sequences=True))
model.add(Dropout(0.5))
model.add(TimeDistributed(Dense(256, activation='relu')))
model.add(Dropout(0.5))
model.add(TimeDistributed(Dense(addresses_vocab_size, activation='softmax')))
model.compile(loss=sparse_categorical_crossentropy,optimizer=Adam(),metrics=['accuracy'])
tmp_x = addresses_sequences_pd
# tmp_x = tmp_x.reshape((-1, preproc_cities.shape[-2], 1))
# Fit the model
model.fit(tmp_x, cities_sequences_pd, batch_size=300, epochs=10, validation_split=0.2)
# # Print prediction(s)
print(logits_to_text(model.predict(tmp_x[:1])[0], cities_tokenizer))
when I try to predict the first city it returns this as the predicton:
<PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD>
which is basically zero as is hardcoded in the logits function.
The expected prediction is Davidmouth.
I have tried changing the shape of the inputs and the labels i.e (addresses_sequences_pd and cities_sequences_pd) respectfully, but that hasn't worked.
I have also tried to change the batch_size and epochs, but that didnd't change anything either.
what am I doing wrong?