Training and evaluating accuracies different in keras LSTM model - python-3.x

Question

I am training a LSTM model using keras. Training the training set gives accuracy:83% in the last epoch, but evaluating or predicting the model with the same training set gives accuracy:47%. I cant figure out the problem.
Also, I have commented out the training part in the following code, since i save the model & weights on disk, but I used it before to train i.e. model.fit(...) etc.
import pandas as pd
import Preprocessing as pre
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.utils import shuffle
import pickle
import numpy as np
import sys
from keras.models import Sequential, load_model
from keras.layers import Dense, Dropout, Activation
from keras.layers import Embedding
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
from keras.layers import LSTM
from keras.preprocessing.sequence import pad_sequences
from keras.models import model_from_json
from keras.preprocessing.text import Tokenizer
df = pd.read_csv('tweets.csv',header=None,encoding = "ISO-8859-1")
df=shuffle(df)
length=df.size
train=[]
test=[]
Y=[]
Y2=[]
count=450000
for a in range(450000): #loading data
# b=pre.preprocess_tweet(df[1][a])
label=df[0][a]
# train.append(b)
Y.append(label)
count-=1
print("Loading training data...", count)
# with open('training_data_no_stopwords.obj', 'wb') as fp:
# pickle.dump(train, fp)
with open ('training_data.obj', 'rb') as fp:
train = pickle.load(fp)
count=156884
for a in range(450000,606884): #loading testin data
# b = pre.preprocess_tweet(df[1][a])
label=df[0][a]
# test.append(b)
Y2.append(label)
count-=1
print("Loading testing data...", count)
# with open('testing_data_no_stopwords.obj', 'wb') as fp:
# pickle.dump(test, fp)
with open ('testing_data.obj', 'rb') as fp:
test = pickle.load(fp)
# vectorizer = CountVectorizer(analyzer = "word",tokenizer = None, preprocessor = None, stop_words = None, max_features = 3500)
# # # fit_transform() does two functions: First, it fits the model
# # # and learns the vocabulary; second, it transforms our training data
# # # into feature vectors. The input to fit_transform should be a list of
# # # strings.
#
# train = vectorizer.fit_transform(train)
# test = vectorizer.transform(test)
tokenizer = Tokenizer(split=' ')
tokenizer.fit_on_texts(train)
train = tokenizer.texts_to_sequences(train)
max_words = 134
train = pad_sequences(train, maxlen=max_words)
tokenizer.fit_on_texts(test)
test = tokenizer.texts_to_sequences(test)
test = pad_sequences(test, maxlen=max_words)
print ('Extracting features & training batches')
print("Training...")
embedding_size=32
model = Sequential()
model.add(Embedding(606884, 32, input_length=134))
model.add(Dropout(0.4))
model.add(LSTM(128))
model.add(Dense(64))
model.add(Dropout(0.5))
model.add(Activation('relu'))
model.add(Dense(1))
model.add(Activation('sigmoid'))
print(model.summary())
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# batch_size = 501
# num_epochs = 3
#
# model.fit(train, np.array(Y), batch_size=batch_size, epochs=num_epochs ,validation_split=0.2,shuffle=True)
# # Save the weights
# model.save_weights('LSTM_model_weights.h5')
#
# # Save the model architecture
# with open('LSTM_model.json', 'w') as f:
# f.write(model.to_json())
# Model reconstruction from JSON file
with open('LSTM_model.json', 'r') as f:
model = model_from_json(f.read())
# Load weights into the new model
model.load_weights('LSTM_model_weights.h5')
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
scores = model.evaluate(train, np.array(Y))
print('Test accuracy:', scores[1])

Answer 1

Maybe try training with Stratified K-Fold. This probably happens because your test data is very different from training data. Also, you can try shuffling the data before train-test splitting.