Why is there no improvement in a categorical data time series model? - keras

Question

I built a simple categorical time series model to predict the next number of a random sequence, but the accuracy hardly moved even I trained it for 10000 epochs. The validation loss started to take off after a few hundred epochs. Could anyone make suggestions for improvement? Here's the model:
import os
import sys
import pandas as pd
import numpy as np
from sklearn.preprocessing import OneHotEncoder
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM
DEVICE = 'CPU'
if DEVICE == 'CPU':
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
else:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
print(tf.test.gpu_device_name())
TOTAL_CATALOG=4
POSSIBLE_OUTCOME_COL=4
LOOK_BACK_WINDOW=1
TRAINING_DATA_RATIO=0.8
TRAINING_EPOCHS=10000
sys.path.insert(0, '/DataScience/MyModules')
from m6data import getDrawData, series_to_supervised, Split_data, get_all_categories
def get_all_categories_local(last_combination):
all_category = np.arange(1, last_combination+1)
return all_category.reshape(1,all_category.shape[0])
All_categories=get_all_categories_local(TOTAL_CATALOG)
data_sequence = [1,1,2,4,2,3,1,2,3,3,4,1,2,3,4,2,2,3,1,3]
raw_df = pd.DataFrame(data_sequence, columns=['NE'])
values = raw_df.values
# 05-Apr-2022: One-Hot Encoding
oh_encoder = OneHotEncoder(categories=All_categories, sparse=False)
encoded_input = oh_encoder.fit_transform(values)
FEATURES = encoded_input.shape[1]
POSSIBLE_OUTCOME_COL = FEATURES
draw_reframe = series_to_supervised(encoded_input, LOOK_BACK_WINDOW,1)
train, test = Split_data(draw_reframe, TRAINING_DATA_RATIO)
# Total input = all possible One-Hot Encoding outcome * number of look-back samples.
ALL_INPUT = POSSIBLE_OUTCOME_COL * LOOK_BACK_WINDOW
# split into input and outputs
train_X, train_y = train.iloc[:,:ALL_INPUT], train.iloc[:,ALL_INPUT:]
test_X, test_y = test.iloc[:,:ALL_INPUT], test.iloc[:,ALL_INPUT:]
train_X = train_X.values.reshape((train_X.shape[0], LOOK_BACK_WINDOW , FEATURES))
test_X = test_X.values.reshape((test_X.shape[0], LOOK_BACK_WINDOW, FEATURES))
print(train_X.shape, train_y.shape)
print(test_X.shape, test_y.shape)
def create_model():
model = Sequential()
model.add(LSTM(10,
return_sequences=False,
input_shape=(train_X.shape[1], train_X.shape[2]),
activation='relu'
)
)
#model.add(LSTM(20))
model.add(Dense(units=train_y.shape[1], activation='softmax'))
model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=0.00005),
loss = 'categorical_crossentropy',
metrics=['accuracy'])
return model
model=create_model()
history = model.fit(
train_X, train_y,
epochs=TRAINING_EPOCHS,
batch_size=8,
validation_data=(test_X, test_y),
verbose=1,
)
Here are the plots of accuracies and losses (red=training, blue=validation).
Accuracies
Losses
Thank you in advance for any suggestions.
Update (13-Jun-2022)
I changed my model to the following
def create_model():
model = Sequential()
model.add(LSTM(50,
return_sequences=True,
input_shape=(train_X.shape[1], train_X.shape[2]),
activation='relu'
)
)
model.add(LSTM(units=1000, kernel_regularizer=regularizers.l1(0.05), return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(units=1000, kernel_regularizer=regularizers.l1(0.05), return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(units=1000, kernel_regularizer=regularizers.l1(0.05), return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(units=1000, kernel_regularizer=regularizers.l1(0.05), activation='relu'))
model.add(Dropout(0.3))
model.add(BatchNormalization())
model.add(Dense(1000))
model.add(Dense(units=train_y.shape[1], activation='softmax'))
model.compile(optimizer = tf.keras.optimizers.SGD(learning_rate=1e-2, nesterov=True),
#tf.keras.optimizers.Adam(learning_rate=0.001),
loss = 'categorical_crossentropy',
metrics=['accuracy'])
return model
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=20,min_lr=1e-10)
early_stop = EarlyStopping(monitor='loss', patience=100)
history = model.fit(
train_X, train_y,
epochs=TRAINING_EPOCHS,
batch_size=16,
validation_split=0.1,
validation_data=(test_X, test_y),
verbose=1,
shuffle=False,
callbacks=([reduce_lr], [early_stop])
Accuracy was bouncing around and Val_accuracy was zero all the way. The loss and val_loss were almost the same and dropping together.
Can anyone advise what I can do in this scenario?