I have multiple features to be used in LSTM with timesteps = 5 (n), when i run the model
def create_dataset(dataset, time_stamp=5):
dataX, dataY = [], []
print(len(dataset) - time_stamp - 1)
for i in range(len(dataset) - time_stamp - 1):
a = dataset[i:(i + time_stamp), :]
dataX.append(a)
dataY.append(dataset[i + time_stamp, 0])
return np.array(dataX), np.array(dataY)
model = Sequential()
model.add(LSTM(16,activation="relu",input_shape=(timeFrame,2), return_sequences=True))
model.add(Dense(1,activation='relu'))
model.compile(optimizer="adam",loss='mean_squared_error')
model.summary()
model.fit(X_train, Y_train, epochs=30, batch_size=1, verbose=1)
X_train.shape = (8,5,2) // The data sets have 8 different datasets with 5 timesteps and 2 features
Y_train.shape = (8) //
train_predict = model.predict(X_train)
train_predict.shape = (8,5,1)
when i try to do So this works fine ... now the problem is that when i run the following
scaler.inverse_transform(train_predict) //{ValueError}Found array with dim 3. Estimator expected <= 2.
scaler.inverse_transform(Y_train) //geting error
the issue is both have different dimension as two input feature is getting mapped with 1 feature. So inverse_transform does not work properly. How to fix this problem
goal is to find mse
[math.sqrt(mean_squared_error(Y_train, train_predict))]
Related
I run a sequential model in Google Colab. I have tried all suggestions I could find about setting seed so I can get reproducible results but nothing works. train and validation accuracy differ every time, as well as the train and validation losses.
More info about my project:
Tensorflow version: 2.9.2
I have paid for the ColabPro version if this matters.
Since I am preprocessing text data I do the following:
# Pad tokens (X var)
tokens = data_sent['word_idx'].tolist()
maxlen = config.MAX_LEN #max([len(s) for s in tokens])
pad_tokens = pad_sequences(tokens, maxlen=maxlen, dtype='int32', padding='post')
This is the function I use to split my data:
def data_split(features, labels, train_frac, random_state=config.SEED):
assert train_frac >= 0 and train_frac <= 1, "Invalid training set fraction"
X_train, X_test, Y_train, Y_test = train_test_split(
features, labels, train_size=train_frac, random_state=random_state)
print(
'train_tokens length:', len(X_train),
'\ntest_tokens length:', len(X_test),
'\ntrain_tags:', len(Y_train),
'\ntest_tags:', len(Y_test),
)
return X_train, X_test, Y_train, Y_test
Model:
model = tf.keras.models.Sequential([
tf.keras.layers.Embedding(input_dim=config.MAX_TOKENS, output_dim=config.EMBEDDING_DIM, input_length=config.MAX_LEN, mask_zero=True),
tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(units=config.EMBEDDING_DIM, activation='tanh', return_sequences=True, dropout=0.2, kernel_regularizer=regularizers.L2(l2_regularizer_start), kernel_initializer=tf.keras.initializers.glorot_uniform(seed=config.SEED))),
tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(units=config.EMBEDDING_DIM, activation='tanh', return_sequences=True, dropout=0.2, kernel_regularizer=regularizers.L2(l2_regularizer_start), kernel_initializer=tf.keras.initializers.glorot_uniform(seed=config.SEED))),
tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(units=config.EMBEDDING_DIM, activation='tanh', return_sequences=True, dropout=0.2, kernel_regularizer=regularizers.L2(l2_regularizer_start), kernel_initializer=tf.keras.initializers.glorot_uniform(seed=config.SEED))),
tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(n_tags, activation='softmax', kernel_initializer=tf.keras.initializers.glorot_uniform(seed=config.SEED))),
])
What I have tried:
1.Set seed for the initializer in each layer:
kernel_initializer=tf.keras.initializers.glorot_uniform(seed=config.SEED)
2.At the beginning of the code v0
np.random.seed(config.SEED)
random.seed(config.SEED)
tf.random.set_seed(config.SEED)
3.At the beginning of the code v1
random.seed(SEED)
np.random.seed(SEED)
tf.random.set_seed(SEED)
os.environ['PYTHONHASHSEED']=str(SEED)
os.environ['TF_DETERMINISTIC_OPS'] = '1'
4.At the beginning of the code v2
tf.keras.utils.set_random_seed(config.SEED)
tf.config.experimental.enable_op_determinism()
5.At the beginning of the code v3
def set_seeds(seed=config.SEED):
os.environ['PYTHONHASHSEED'] = str(seed)
random.seed(seed)
tf.random.set_seed(seed)
np.random.seed(seed)
def set_global_determinism(seed=config.SEED):
set_seeds(seed=seed)
os.environ['TF_DETERMINISTIC_OPS'] = '1'
os.environ['TF_CUDNN_DETERMINISTIC'] = '1'
tf.config.threading.set_inter_op_parallelism_threads(1)
tf.config.threading.set_intra_op_parallelism_threads(1)
# Call the above function with seed value
set_global_determinism(seed=config.SEED)
Update:
If I run the notebook multiple times without restarting the runtime I get the same results.
If I restart the Colab Runtime the results (accuracy and loss) are always different despite the random seed being set and the same each time.
Update 1
Using !nvidia-smi shows that each time I restart a session in Colab the GPU I am attached to is changing.
So, it appears the Colab creates the issue but I cannot figure out how to fix it.
From the question, you can use the random generator.
Sample: Random uniform values. Expecting no sequencing output from determining input, accuracy but varies and none determining with gardients.
for image in list_file:
g = tf.random.Generator.from_seed(1234)
g.reset_from_seed( 1235 + icount )
temp = tf.random.uniform( shape=(4, 1), minval=0, maxval=6, dtype=tf.dtypes.int64,seed=1235 + icount,name="random" )
arg = tf.math.argmax( temp ).numpy()[0]
result = temp[arg].numpy()[0]
icount = icount + 1
Output:
3
4
3
3
4
3
5
4
5
4
I have created an LSTM sales prediction model that works really well on the train and test sets. I would now like to predict beyond the dates in the entire dataset.
I have tried following this answer how to use the Keras model to forecast for future dates or events? but I really can't figure out how to adjust my code to do future predictions.
Also, I changed my code from
X_train, y_train = train_set_scaled[:, 1:], train_set_scaled[:, 0:1]
X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1])
X_test, y_test = test_set_scaled[:, 1:], test_set_scaled[:, 0:1]
X_test = X_test.reshape(X_test.shape[0], 1, X_test.shape[1])
to
X_train, y_train = train_set_scaled[:, 1:], train_set_scaled[:, 1:8]
X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1])
X_test, y_test = test_set_scaled[:, 1:], test_set_scaled[:, 1:8]
X_test = X_test.reshape(X_test.shape[0], 1, X_test.shape[1])
after trying the solution in Keras time series can I predict next 6 month in one time
Here is the code where training and modelling happens:
# changed to initial
for df in m:
train_set, test_set = m[df][0:-6].values, m[df][-6:].values
#apply Min Max Scaler
scaler = MinMaxScaler(feature_range=(-1, 1))
scaler = scaler.fit(train_set)
# reshape training set
train_set = train_set.reshape(train_set.shape[0], train_set.shape[1])
train_set_scaled = scaler.transform(train_set)
# reshape test set
test_set = test_set.reshape(test_set.shape[0], test_set.shape[1])
test_set_scaled = scaler.transform(test_set)
#build the LSTM Model
X_train, y_train = train_set_scaled[:, 1:], train_set_scaled[:, 0:1]
X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1])
X_test, y_test = test_set_scaled[:, 1:], test_set_scaled[:, 0:1]
X_test = X_test.reshape(X_test.shape[0], 1, X_test.shape[1])
print('Fitting model for: {}'.format(df))
#fit our LSTM Model
model = Sequential()
model.add(LSTM(4, batch_input_shape=(1, X_train.shape[1], X_train.shape[2]), stateful=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(X_train, y_train, nb_epoch=500, batch_size=1, verbose=1, shuffle=False)
# model.save('lstm_model.h5')
print('Predictions for: {}'.format(df))
#check prediction
y_pred = model.predict(X_test,batch_size=1)
print('Inverse Transform for: {}'.format(df))
#inverse transformation to see actual sales
#reshape y_pred
y_pred = y_pred.reshape(y_pred.shape[0], 1, y_pred.shape[1])
#rebuild test set for inverse transform
pred_test_set = []
for index in range(0,len(y_pred)):
print (np.concatenate([y_pred[index],X_test[index]],axis=1))
pred_test_set.append(np.concatenate([y_pred[index],X_test[index]],axis=1))
#reshape pred_test_set
pred_test_set = np.array(pred_test_set)
pred_test_set = pred_test_set.reshape(pred_test_set.shape[0], pred_test_set.shape[2])
#inverse transform
pred_test_set_inverted = scaler.inverse_transform(pred_test_set)
I would like the predictions to go beyond the data in the dataset.
UPDATE: I trained the model and took its predictions on the test set. Use these as input for another LSTM model to fit and predict for 12 months. It worked for me. Also changed my last Dense layer (above) to predict 1 point at a time instead of 7 as I had before.
Below is the code:
from numpy import array
for df in d:
if df in list_df:
# df_ADIDAS DYN PUL DEO 150 FCA5421
#KEEP
result_list = []
sales_dates = list(d["{}".format(df)][-7:].Month)
act_sales = list(d["{}".format(df)][-7:].Sale)
for index in range(0,len(pred_test_set_inverted)):
result_dict = {}
result_dict['pred_value'] = int(pred_test_set_inverted[index][0] + act_sales[index]) #change to 0 ffrom act_sales[index]
result_dict['date'] = sales_dates[index] #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>REVIEW
result_list.append(result_dict)
df_result = pd.DataFrame(result_list)
predictions = list(df_result['pred_value'])
forecasts = []
result_list
for i in range(len(result_list)):
forecasts.append(result_list[i]['pred_value'])
def split_sequence(sequence, n_steps):
X, y = list(), list()
for i in range(len(sequence)):
# find the end of this pattern
end_ix = i + n_steps
# check if we are beyond the sequence
if end_ix > len(sequence)-1:
break
# gather input and output parts of the pattern
seq_x, seq_y = sequence[i:end_ix], sequence[end_ix]
X.append(seq_x)
y.append(seq_y)
return array(X), array(y)
# choose a number of time steps
n_steps = 4
# split into samples
X, y = split_sequence(forecasts, n_steps)
# summarize the data
# for i in range(len(X)):
# print(X[i], y[i])
n_features = 1
X = X.reshape((X.shape[0], X.shape[1], n_features))
# define model
model = Sequential()
model.add(LSTM(50, activation='relu', input_shape=(n_steps, n_features)))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')
# fit model
model.fit(X, y, epochs=200, verbose=0)
# demonstrate prediction
x_input = array(predictions[-4:])
x_input = x_input.reshape((1, n_steps, n_features))
yhat = model.predict(x_input, verbose=0)
#print(yhat)
currentStep = yhat[:, -1:]
print('Twelve Month Prediction for {}'.format(df))
for i in range(12):
if i == 0:
x_input = x_input.reshape((1, n_steps, n_features))
yhat = model.predict(x_input, verbose=0)
print(yhat)
else:
x0_input = np.append(x_input, [currentStep[i-1]])
x0_input = x0_input.reshape((1, n_steps+1, n_features))
x_input = x0_input[:,1:]
yhat = model.predict(x_input)
currentStep = np.append(currentStep, yhat[:,-1:])
print(yhat)
Your last Dense layer says that you are predicting 7 points at a time. Save those predictions and feed them to the model again to predict next 7. That makes it 14 predictions simultaneously. And so on. Or change the number of nodes and shape of y from 7 to corresponding number and train again.
I need help to understand Keras prediction class labeling
I am building the binary classification model with Keras. I use 3 'relu' layers plus sigmoid output, the loss function is binary_crossentropy.
The point is that I am not able to understand simple thing:
- for my y_train/y_test I use two values: False / True. When I predict on the X_test I am getting he probability or class predicted, fine. But unfortunately the model has in 50% of cases the accuracy of 0,1 and in other 50% of cases 0,9. This comes from the fact that the distribution of the False / True is uneven - 90% of False and 10% of True. ... and then, what happens is that it looks like the class label for False is 0 in half of the cases and 1 in other half of cases. I am unable to figure out why.
I found some posts saying Keras would number the classes in alphabetical order, so I assumed False would always get 0 and True 1, but this does not seem to be the case.
Please find my code below and advise what I do wrong.
# ####################### Fetch data #################################
#df = pd.read_csv(os.path.join(data_folder, data_file))
df = pd.read_csv('\\\...\\train_data_with_anomalous_column.csv', delimiter=',')
x = df.drop('ANOMALOUS', axis=1)
# x = x.drop('TIMESTAMP', axis=1)
y = df['ANOMALOUS'].copy()
x = x.as_matrix()
y = y.as_matrix()
# y.astype(int)
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.1)
# ####################### get rid of TIMESTAMP columns ###############
# ###### first, copy the TIMESTAMP column into ananother np-array
# ###### in order to be able to join it later
X_test_TST = X_test[:, [0]]
X_train = np.delete(X_train, 0, 1)
X_test = np.delete(X_test, 0, 1)
class_weight = {False: 1.,
True: 1.
}
training_set_size = X_train.shape[0]
n_inputs = 22
n_h1 = args.n_hidden_1
n_h2 = args.n_hidden_2
n_h3 = args.n_hidden_3
n_outputs = 1
n_epochs = 2
model_validation_split = 0.1
batch_size = args.batch_size
learning_rate = args.learning_rate
print(X_train.shape, y_train.shape, X_test.shape, y_test.shape, sep='\n')
# Build a simple MLP model
model = Sequential()
# first hidden layer
model.add(Dense(n_h1, activation='relu', input_shape=(n_inputs,)))
# second hidden layer
model.add(Dense(n_h2, activation='relu'))
# second hidden layer
model.add(Dense(n_h3, activation='relu'))
# output layer
model.add(Dense(n_outputs, activation='sigmoid'))
model.summary()
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy']
)
# start an Azure ML run
run = Run.get_context()
class LogRunMetrics(Callback):
# callback at the end of every epoch
def on_epoch_end(self, epoch, log):
# log a value repeated which creates a list
run.log('Loss', log['loss'])
run.log('Accuracy', log['acc'])
history = model.fit(X_train, y_train,
batch_size=batch_size,
epochs=n_epochs,
verbose=2,
class_weight=class_weight,
callbacks=[LogRunMetrics()])
score = model.evaluate(X_test, y_test, verbose=1)
# #### get accuracy, F1, precision and recall
# Get the class predictions
y_classes = model.predict_classes(X_test, batch_size=batch_size, verbose=1)
print(y_classes
print(classification_report(y_test, y_classes))
...
I am at the step of train the model. However, when I apply the code from a tutorial: batch_x, batch_y = mnist.train.next_batch(50). It shows that there is no attribute 'train' in the TensorFlow model. I know it is outdated code, and I tried to convert to new version of TensorFlow. However, i couldn't find a matching code that can do the same thing as the above line of code do. I bet there is a way but I couldn't come up with one solution.
I found a method that asked me to use tf.data.Dataset.batch(batch_size).
I tried the following way, but none of them works.
a. batch_x, batch_y = mnist.train.next_batch(50)
b. batch_x, batch_y = tf.data.Dataset.batch(batch_size)
c. batch_x, batch_y = tf.data.Dataset.batch(50)
d. batch_x, batch_y = mnist.batch(50)
with tf.Session() as sess:
#FIrst, run vars_initializer to initialize all variables
sess.run(vars_initializer)
for i in range(steps):
#Each batch: 50 images
batch_x, batch_y = mnist.train.next_batch(50)
#Train the model
#Dropout keep_prob (% to keep): 0.5 --> 50% will be dropped out
sess.run(cnn_trainer, feed_dict={x: batch_x, y_true: batch_y, hold_prob: 0.5})
#Test the model: at each 100th step
#Run this block of code for each 100 times of training, each time run a batch
if i % 100 == 0:
print('ON STEP: {}'.format(i))
print('ACCURACY: ')
#Compare to find matches of y_pred and y_true
matches = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))
#Cast the matches from integers to tf.float32
#Calculate the accuracy using the mean of matches
acc = tf.reduce_mean(tf.cast(matches, tf.float32))
#Test the model at each 100th step
#Using test dataset
#Dropout: NONE because of test, not training.
test_accuracy = sess.run(acc, feed_dict = {x:mnist.test.images, y_true:mnist.test.labels, hold_prob:1.0})
print(test_accuracy)
print('\n')
You can use tf.keras.datasets.mnist.load_data. It returns a tuple of Numpy arrays: (x_train, y_train), (x_test, y_test).
After that, you need to create dataset object using Dataset API. This will create training dataset. Test dataset could be created in the same fashion.
train, test = tf.keras.datasets.mnist.load_data()
dataset = tf.data.Dataset.from_tensor_slices((train[0], train[1]))
Then, to create batch, you need to apply batch function to it
dataset = dataset.batch(1)
To output it contents or use it in training you need to create iterator. Code below creates most common iterator and outputs element of batch_size in this case 1.
iterator = dataset.make_one_shot_iterator()
with tf.Session() as sess:
print(sess.run(iterator.get_next())
Please read https://www.tensorflow.org/guide/datasets
This uses TensorFlow 1.11.0 and Keras and intended to show how to use the batch. You have to adapt it to your need.
import tensorflow as tf
from tensorflow import keras as k
(x_train, y_train), (X_test, Y_test) = tf.keras.datasets.mnist.load_data()
X_train = x_train.reshape(x_train.shape[0], 28, 28,1)
y_train = tf.keras.utils.to_categorical(y_train,10)
X_test = X_test.reshape(X_test.shape[0], 28, 28,1)
Y_test = tf.keras.utils.to_categorical(Y_test,10)
train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))
train_dataset = train_dataset.batch(32)
test_dataset = tf.data.Dataset.from_tensor_slices((X_test, Y_test))
test_dataset = test_dataset.batch(32)
model = tf.keras.models.Sequential([
tf.keras.layers.Convolution2D(32, (2, 2), activation='relu', input_shape=(28, 28,1)),
tf.keras.layers.MaxPool2D(pool_size=2),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(128),
tf.keras.layers.Activation('relu'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(10, activation='softmax')
])
tbCallback = [
k.callbacks.TensorBoard(
log_dir="D:/TensorBoard", histogram_freq=1, write_graph=True, write_images=True
)
]
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(train_dataset, epochs = 10, steps_per_epoch = 30,validation_data=test_dataset,validation_steps=1, callbacks=tbCallback)
I’m working with Keras on a sentence similarity task (using the STS dataset) and am having problems merging the layers. The data consists of 1184 sentence pairs each scored between 0 and 5. Below are the shapes of my numpy arrays. I’ve padded each of the sentences to 50 words and run them through and embedding layer, using the glove embedding’s with 100 dimensions. When merging the two networks I'm getting an error..
Exception: Error when checking model input: the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 1 arrays but instead got the following list of 2 arrays:
Here is what my code looks like
total training data = 1184
X1.shape = (1184, 50)
X2.shape = (1184, 50)
Y.shape = (1184, 1)
embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=50,
trainable=False)
s1rnn = Sequential()
s1rnn.add(embedding_layer)
s1rnn.add(LSTM(128, input_shape=(100, 1)))
s1rnn.add(Dense(1))
s2rnn = Sequential()
s2rnn.add(embedding_layer)
s2rnn.add(LSTM(128, input_shape=(100, 1)))
s2rnn.add(Dense(1))
model = Sequential()
model.add(Merge([s1rnn,s2rnn],mode='concat'))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='RMSprop', metrics=['accuracy'])
model.fit([X1,X2], Y,batch_size=32, nb_epoch=100, validation_split=0.05)
The problem is not with the merge layer. You need to create two embedding layers to feed in 2 different inputs.
The following modifications should work:
embedding_layer_1 = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=50,
trainable=False)
embedding_layer_2 = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=50,
trainable=False)
s1rnn = Sequential()
s1rnn.add(embedding_layer_1)
s1rnn.add(LSTM(128, input_shape=(100, 1)))
s1rnn.add(Dense(1))
s2rnn = Sequential()
s2rnn.add(embedding_layer_2)
s2rnn.add(LSTM(128, input_shape=(100, 1)))
s2rnn.add(Dense(1))