Discrepancy between tensorflow model accuracy and actual accuracy - python-3.x

Question

I am trying to do categorical classification of data. The data consists of 3 text variables, and one real value. I split the data into three sets - training, validation and testing. I am using tensorflow and python. For the test data I get the following stats: test data statistics
A categorical accuracy of 0.9919, however when I perform a prediction on the same test data, and evaluate the accuracy with the sci-kit classification_report function, I get an accuracy of 0.60, as seen here: classification report.
df.info() looks like this ('category' being the Y value): info. I don't think that the sci-kit learn accuracy statistic is misrepresenting the prediction, since the confusion matrix looks like this.
I have re-built the model multiple times, and tried balancing classes with class weights just in case, however this still wouldn't explain the discrepancy between tensorflows accuracy, and the sci-kit one (gained by prediction).
The code:
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_text
import tensorflow_addons as tfa
import tensorflow_datasets as tfds
df = pd.read_csv("procData_nosub.csv")
df = df.sample(frac=1).reset_index(drop=True) # Shuffling
df = df[(df.category != 'Pictures')] # Removing small categories
df = df[(df.category != 'Software')]
df = df.drop("fileAmount", axis=1)
df = df.drop("more100Files", axis=1)
train, val, test = np.split(df.sample(frac=1), [int(0.8 * len(df)), int(0.9 * len(df))])
# Function to convert dataframe to dataset (from https://www.tensorflow.org/tutorials/structured_data/preprocessing_layers)
def df_to_dataset(dataframe, shuffle=True, batch_size=64):
df = dataframe.copy()
labels = df.pop('category')
df = {key: value[:,tf.newaxis] for key, value in df.items()}
ds = tf.data.Dataset.from_tensor_slices((dict(df), labels))
if shuffle:
ds = ds.shuffle(buffer_size=len(df))
ds = ds.batch(batch_size)
ds = ds.prefetch(batch_size)
return ds
train_data = df_to_dataset(train)
test_data = df_to_dataset(test)
validation_data = df_to_dataset(val)
# Function to convert text (for the Y) to one-hot-encoding
catVals = np.unique(df['category'])
table = tf.lookup.StaticHashTable(
initializer = tf.lookup.KeyValueTensorInitializer(
keys = tf.constant(catVals),
values = list(range(len(catVals)))
),
default_value = -1,
name = "target_encoding"
)
#tf.function
def target(x):
return table.lookup(x)
def fetch(features, labels):
return features, tf.one_hot(target(labels), len(catVals))
# Applying the text (Y) -> one-hot-encoding
train_data_f = train_data.map(fetch)
test_data_f = test_data.map(fetch)
validation_data_f = validation_data.map(fetch)
# Using an encoder
embedding = "https://tfhub.dev/google/universal-sentence-encoder-multilingual/3"
hub_layer = hub.KerasLayer(embedding, output_shape = 512, input_shape = [], dtype=tf.string, trainable=True)
# Normalizing real values (from https://www.tensorflow.org/tutorials/structured_data/preprocessing_layers)
def get_normalization_layer(name, dataset):
normalizer = tf.keras.layers.Normalization(axis=None)
feature_ds = dataset.map(lambda x, y: x[name])
normalizer.adapt(feature_ds)
return normalizer
all_inputs = []
encoded_features = []
# Adding filesize to features
for header in ['fileSize']: # fileAmount
numeric_col = tf.keras.Input(shape=(1,), name=header)
normalization_layer = get_normalization_layer(header, train_data_f)
encoded_numeric_col = normalization_layer(numeric_col)
all_inputs.append(numeric_col)
encoded_features.append(encoded_numeric_col)
# Adding title, description, files to features
for header in ['title', 'description', 'files']:
text_col = tf.keras.Input(shape=(), name=header, dtype='string')
encoded_text_col = hub_layer(text_col)
all_inputs.append(text_col)
encoded_features.append(encoded_text_col)
# Describing the model
all_features = tf.keras.layers.concatenate(encoded_features)
x = all_features
for i in range(3):
x = tf.keras.layers.Dense(16, activation='relu')(x)
# x = tf.keras.layers.Dropout(0.2)(x)
output = tf.keras.layers.Dense(len(catVals), activation='softmax')(x)
model = tf.keras.Model(all_inputs, output)
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),
loss=tf.keras.losses.CategoricalCrossentropy(), # deleted from_logits=True
metrics=["categorical_accuracy",
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall(),
tfa.metrics.F1Score(num_classes=len(catVals),
average='macro',
threshold=0.5)])
history = model.fit(train_data_f, epochs=5, validation_data=validation_data_f) # removed class weights
model.evaluate(test_data_f)
model.evaluate(validation_data_f)
# Taking x and y from the test data
test_x = test_data_f.unbatch().map(lambda x, y: x)
test_y = test_data_f.unbatch().map(lambda x, y: y)
test_predicted = model.predict(test_x)
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
# converting test_y to a numpy array (from a tensorflow Dataset)
test_y = np.array([x for x in test_y])
print(classification_report(test_y.argmax(1), test_predicted.argmax(1)))
print(tf.math.confusion_matrix(test_y.argmax(1), test_predicted.argmax(1)))
As a TLDR: I am using universal-sentence-encoder-multilingual/3 from tensorflow hub, and the model looks like this: model build, fit history. Model summary:
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
fileSize (InputLayer) [(None, 1)] 0 []
title (InputLayer) [(None,)] 0 []
description (InputLayer) [(None,)] 0 []
files (InputLayer) [(None,)] 0 []
normalization (Normalization) (None, 1) 3 ['fileSize[0][0]']
keras_layer (KerasLayer) (None, 512) 68927232 ['title[0][0]',
'description[0][0]',
'files[0][0]']
concatenate (Concatenate) (None, 1537) 0 ['normalization[0][0]',
'keras_layer[0][0]',
'keras_layer[1][0]',
'keras_layer[2][0]']
dense (Dense) (None, 16) 24608 ['concatenate[0][0]']
dense_1 (Dense) (None, 16) 272 ['dense[0][0]']
dense_2 (Dense) (None, 16) 272 ['dense_1[0][0]']
dense_3 (Dense) (None, 4) 68 ['dense_2[0][0]']
==================================================================================================
Total params: 68,952,455
Trainable params: 68,952,452
Non-trainable params: 3
__________________________________________________________________________________________________