How do I create a variable-length input LSTM in Keras? - python-3.x

I am trying to do some vanilla pattern recognition with an LSTM using Keras to predict the next element in a sequence.
My data look like this:
where the label of the training sequence is the last element in the list: X_train['Sequence'][n][-1].
Because my Sequence column can have a variable number of elements in the sequence, I believe an RNN to be the best model to use. Below is my attempt to build an LSTM in Keras:
# Build the model
# A few arbitrary constants...
max_features = 20000
out_size = 128
# The max length should be the length of the longest sequence (minus one to account for the label)
max_length = X_train['Sequence'].apply(len).max() - 1
# Normal LSTM model construction with sigmoid activation
model = Sequential()
model.add(Embedding(max_features, out_size, input_length=max_length, dropout=0.2))
model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2))
model.add(Dense(1))
model.add(Activation('sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
And here's how I attempt to train my model:
# Train the model
for seq in X_train['Sequence']:
print("Length of training is {0}".format(len(seq[:-1])))
print("Training set is {0}".format(seq[:-1]))
model.fit(np.array([seq[:-1]]), [seq[-1]])
My output is this:
Length of training is 13
Training set is [1, 3, 13, 87, 1053, 28576, 2141733, 508147108, 402135275365, 1073376057490373, 9700385489355970183, 298434346895322960005291, 31479360095907908092817694945]
However, I get the following error:
Exception: Error when checking model input: expected embedding_input_1 to have shape (None, 347) but got array with shape (1, 13)
I believe my training step is correctly setup, so my model construction must be wrong. Note that 347 is max_length.
How can I correctly build a variable-length input LSTM in Keras? I'd prefer not to pad the data. Not sure if it's relevant, but I'm using the Theano backend.

I am not clear about the embedding procedure. But still here is a way to implement a variable-length input LSTM. Just do not specify the timespan dimension when building LSTM.
import keras.backend as K
from keras.layers import LSTM, Input
I = Input(shape=(None, 200)) # unknown timespan, fixed feature size
lstm = LSTM(20)
f = K.function(inputs=[I], outputs=[lstm(I)])
import numpy as np
data1 = np.random.random(size=(1, 100, 200)) # batch_size = 1, timespan = 100
print f([data1])[0].shape
# (1, 20)
data2 = np.random.random(size=(1, 314, 200)) # batch_size = 1, timespan = 314
print f([data2])[0].shape
# (1, 20)

The trick to training and classifying sequences is training with masking and classifying using a stateful network. Here's an example that I made that classifies whether a sequence of variable length starts with zero or not.
import numpy as np
np.random.seed(1)
import tensorflow as tf
tf.set_random_seed(1)
from keras import models
from keras.layers import Dense, Masking, LSTM
import matplotlib.pyplot as plt
def stateful_model():
hidden_units = 256
model = models.Sequential()
model.add(LSTM(hidden_units, batch_input_shape=(1, 1, 1), return_sequences=False, stateful=True))
model.add(Dense(1, activation='relu', name='output'))
model.compile(loss='binary_crossentropy', optimizer='rmsprop')
return model
def train_rnn(x_train, y_train, max_len, mask):
epochs = 10
batch_size = 200
vec_dims = 1
hidden_units = 256
in_shape = (max_len, vec_dims)
model = models.Sequential()
model.add(Masking(mask, name="in_layer", input_shape=in_shape,))
model.add(LSTM(hidden_units, return_sequences=False))
model.add(Dense(1, activation='relu', name='output'))
model.compile(loss='binary_crossentropy', optimizer='rmsprop')
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs,
validation_split=0.05)
return model
def gen_train_sig_cls_pair(t_stops, num_examples, mask):
x = []
y = []
max_t = int(np.max(t_stops))
for t_stop in t_stops:
one_indices = np.random.choice(a=num_examples, size=num_examples // 2, replace=False)
sig = np.zeros((num_examples, max_t), dtype=np.int8)
sig[one_indices, 0] = 1
sig[:, t_stop:] = mask
x.append(sig)
cls = np.zeros(num_examples, dtype=np.bool)
cls[one_indices] = 1
y.append(cls)
return np.concatenate(x, axis=0), np.concatenate(y, axis=0)
def gen_test_sig_cls_pair(t_stops, num_examples):
x = []
y = []
for t_stop in t_stops:
one_indices = np.random.choice(a=num_examples, size=num_examples // 2, replace=False)
sig = np.zeros((num_examples, t_stop), dtype=np.bool)
sig[one_indices, 0] = 1
x.extend(list(sig))
cls = np.zeros((num_examples, t_stop), dtype=np.bool)
cls[one_indices] = 1
y.extend(list(cls))
return x, y
if __name__ == '__main__':
noise_mag = 0.01
mask_val = -10
signal_lengths = (10, 15, 20)
x_in, y_in = gen_train_sig_cls_pair(signal_lengths, 10, mask_val)
mod = train_rnn(x_in[:, :, None], y_in, int(np.max(signal_lengths)), mask_val)
testing_dat, expected = gen_test_sig_cls_pair(signal_lengths, 3)
state_mod = stateful_model()
state_mod.set_weights(mod.get_weights())
res = []
for s_i in range(len(testing_dat)):
seq_in = list(testing_dat[s_i])
seq_len = len(seq_in)
for t_i in range(seq_len):
res.extend(state_mod.predict(np.array([[[seq_in[t_i]]]])))
state_mod.reset_states()
fig, axes = plt.subplots(2)
axes[0].plot(np.concatenate(testing_dat), label="input")
axes[1].plot(res, "ro", label="result", alpha=0.2)
axes[1].plot(np.concatenate(expected, axis=0), "bo", label="expected", alpha=0.2)
axes[1].legend(bbox_to_anchor=(1.1, 1))
plt.show()

Not sure how applicable recurrent networks are for your sequences, ie how strongly dependent each element is on its preceding sequence as opposed to other factors. That being said (which doesn't help you one bit of course), if you don't want to pad your input with some bad value, a stateful model that processes a single timestep at once is the only alternative for variable length sequences IMHO. If you don't mind taking an alternative approach to encoding:
import numpy as np
import keras.models as kem
import keras.layers as kel
import keras.callbacks as kec
import sklearn.preprocessing as skprep
X_train, max_features = {'Sequence': [[1, 2, 4, 5, 8, 10, 16], [1, 2, 1, 5, 5, 1, 11, 16, 7]]}, 16
num_mem_units = 64
size_batch = 1
num_timesteps = 1
num_features = 1
num_targets = 1
num_epochs = 1500
model = kem.Sequential()
model.add(kel.LSTM(num_mem_units, stateful=True, batch_input_shape=(size_batch, num_timesteps, num_features),
return_sequences=True))
model.add(kel.Dense(num_targets, activation='sigmoid'))
model.summary()
model.compile(loss='binary_crossentropy', optimizer='adam')
range_act = (0, 1) # sigmoid
range_features = np.array([0, max_features]).reshape(-1, 1)
normalizer = skprep.MinMaxScaler(feature_range=range_act)
normalizer.fit(range_features)
reset_state = kec.LambdaCallback(on_epoch_end=lambda *_ : model.reset_states())
# training
for seq in X_train['Sequence']:
X = seq[:-1]
y = seq[1:] # predict next element
X_norm = normalizer.transform(np.array(X).reshape(-1, 1)).reshape(-1, num_timesteps, num_features)
y_norm = normalizer.transform(np.array(y).reshape(-1, 1)).reshape(-1, num_timesteps, num_targets)
model.fit(X_norm, y_norm, epochs=num_epochs, batch_size=size_batch, shuffle=False,
callbacks=[reset_state])
# prediction
for seq in X_train['Sequence']:
model.reset_states()
for istep in range(len(seq)-1): # input up to not incl last
val = seq[istep]
X = np.array([val]).reshape(-1, 1)
X_norm = normalizer.transform(X).reshape(-1, num_timesteps, num_features)
y_norm = model.predict(X_norm)
yhat = int(normalizer.inverse_transform(y_norm[0])[0, 0])
y = seq[-1] # last
put = '{0} predicts {1:d}, expecting {2:d}'.format(', '.join(str(val) for val in seq[:-1]), yhat, y)
print(put)
which produces sth like:
1, 2, 4, 5, 8, 10 predicts 11, expecting 16
1, 2, 1, 5, 5, 1, 11, 16 predicts 7, expecting 7
with ridiculous loss, however.

It turns out that you can do this using ragged inputs.
Firstly, you need to convert your input data to classes using the to_categorical function
from tensorflow.keras.utils import to_categorical
from tensorflow.ragged import constant
X_train = constant(list(map(lambda x: to_categorical(x, num_classes=max_features),X_train)))
Then, you need to edit your model slightly:
model = Sequential()
model.add(Input((None,max_features),ragged=True)) # use this instead of an Embedding
model.add(Embedding(max_features, out_size, input_length=max_length, dropout=0.2))
model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2))
model.add(Dense(1))
model.add(Activation('sigmoid'))
And then work from there!

Related

How to Determine the Output Shape of a Conv2DTranspose of an AutoEncoder

I am building an autoencoder for learning 28 ultrasound time signals of shape [262144,2] i.e. 262144 pairs of time and voltage points concatenated to form a [262144x2] tensor as input data to a stacked convolutional encoder. The latent space is set to produce a vector of length 16. The problem arise from the decoder, where a 'for loop' is used to stack two Conv2DTranspose layers each with a filter sizes of 64 and 32 and a kernel of 3 to the latent space output in order to reproduce the original input shape of [262144x2]. Instead, the decoder network gives a [262144x4] output tensor which does not match the validation and input data shapes of [262144x2]. What model parameters (filter, kernel, strides and padding) should I use to get appropriate tensor dimensions? the code and output are shown below. Your assistance is greatly appreciated!
from keras.layers import Dense, Input
from keras.layers import Conv2D, Flatten
from keras.layers import Reshape, Conv2DTranspose
from keras.models import Model
from keras.datasets import mnist
from keras.utils import plot_model
from keras import backend as K
import numpy as np
import matplotlib.pyplot as plt
x_Train = Signals
x_Test = Signals1
Sig_size1 = x_Train.shape[1]
Sig_size2 = x_Train.shape[2]
Sig_size11 = x_Test.shape[1]
Sig_size22 = x_Test.shape[2]
x_Train = np.reshape(x_Train,[-1, Sig_size1, Sig_size2, 1])
x_Train = x_Train.astype('float32') / np.max(x_Train)
x_Test = np.reshape(x_Test,[-1, Sig_size11, Sig_size22, 1])
x_Test = x_Test.astype('float32') / np.max(x_Test)
# network parameters# encoder/decoder number of filters per CNN layer
input_shape = (Sig_size1, Sig_size2, 1)
batch_size = 32 # Was 32
kernel_size = 1 # Was 3
latent_dim = 16 # Was 16
# encoder/decoder number of filters per CNN layer
layer_filters = [2, 6]
# build the autoencoder model
# first build the encoder model
inputs = Input(shape=input_shape, name='encoder_input')
x = inputs
# stack of Conv2D(32)-Conv2D(64)
for filters in layer_filters:
x = Conv2D(filters=filters,
kernel_size=kernel_size,
strides=2,
activation='relu',
padding='same')(x)
shape = K.int_shape(x)
# generate latent vector
x = Flatten()(x)
latent = Dense(latent_dim, name='latent_vector')(x)
# instantiate encoder model
encoder = Model(inputs, latent, name='encoder')
encoder.summary()
plot_model(encoder, to_file='encoder.png', show_shapes=True)
# build the decoder model
latent_inputs = Input(shape=(latent_dim,), name='decoder_input')
# use the shape (7, 7, 64) that was earlier saved
x = Dense(shape[1] * shape[2] * shape[3])(latent_inputs)
# from vector to suitable shape for transposed conv
x = Reshape((shape[1], shape[2], shape[3]))(x)
# stack of Conv2DTranspose(64)-Conv2DTranspose(32)
layer_filters = [1,8] ########change
kernel_size = 3 # Was 3
# for filters in layer_filters[::-1]:
for filters in layer_filters[::-1]:
x = Conv2DTranspose(filters=filters,
kernel_size=kernel_size,
strides=2,
activation='relu',
padding='same')(x)
# layer_filters = [64, 32]
kernel_size = 3 # Was 3
# reconstruct the input
outputs = Conv2DTranspose(filters=1, #Was 1
kernel_size=kernel_size,
activation='sigmoid',
padding='same',
name='decoder_output')(x)
# instantiate decoder model
decoder = Model(latent_inputs, outputs, name='decoder')
decoder.summary()
plot_model(decoder, to_file='decoder.png', show_shapes=True)
# autoencoder = encoder + decoder
# instantiate autoencoder model
autoencoder = Model(inputs,
decoder(encoder(inputs)),
name='autoencoder')
autoencoder.summary()
plot_model(autoencoder,
to_file='autoencoder.png',
show_shapes=True)
# Mean Square Error (MSE) loss funtion, Adam optimizer
autoencoder.compile(loss='mse', optimizer='adam')
# train the autoencoder
autoencoder.fit(x_Train,
x_Train,
validation_data=(x_Test, x_Test),
epochs=1,
batch_size=batch_size)
# predict the autoencoder output from test data
x_decoded = autoencoder.predict(x_Test)
This code was adapted from Advanced Deep Learning with Keras by Rowel Atienza (Chapter 3 for a denoising decoder for MNIST data)

Multi-input/Multi-output : Wrong output dimension when using KerasClassifier and GridSearchCV

I have built a multi-input (100 features) multi-ouput (100 predictions) ANN model using keras and tensorflow. I have been able to train my model and reach a quite satisfying accuracy on the test set using the following code :
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import backend as K
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Dropout
def my_loss_fn(y_true, y_pred) :
d = K.sum(K.abs(y_true), axis = -1)
n = K.sum((K.tanh(100000*y_true*y_pred)/2 + 0.5)*K.abs(y_true), axis = -1)
return 1 - n/d
def my_metric_fn(y_true, y_pred) :
d = K.sum(K.abs(y_true))
n = K.sum((K.tanh(100000*y_true*y_pred)/2 + 0.5)*K.abs(y_true))
return n/d
def accuracy(y_true, y_pred) :
#print(y_true.shape, y_true)
#print(y_pred.shape, y_true)
acc = np.zeros([1, len(y_true)])
for day in range(len(y_pred)) :
d = 0
n = 0
for i in range(len(y_pred[0])) :
d = d + abs(y_true[day, i])
if np.sign(y_pred[day, i])*np.sign(y_true[day, i]) > 0 :
n = n + abs(y_true[day, i])
else :
n = n + 0
acc[0, day] = n/d
return np.mean(acc, axis = -1)[0]
#Model
classifier = Sequential()
classifier.add(Dense(units = 50, input_shape = (100, ), activation = "tanh"))
classifier.add(Dropout(0.2))
classifier.add(Dense(units=100, activation = 'tanh'))
classifier.compile(optimizer = 'rmsprop', loss = my_loss_fn, metrics = ['accuracy', my_metric_fn])
#Training
callback = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', min_delta = 0.0001, patience = 20, verbose = 0, mode = 'min')
nb_epochs = 250
history = classifier.fit(X_train, y_train, epochs = nb_epochs, batch_size = 31, callbacks = [callback], verbose = True, validation_split = 0., validation_data = (X_test, y_test), use_multiprocessing = True)
#Prediction
y_pred_train = classifier.predict(X_train)
y_pred_test = classifier.predict(X_test)
acc_test = accuracy(y_test, y_pred_test)
acc_train = accuracy(y_train, y_pred_train)
I am trying to improve the performance of my model by tuning the hyperparameters so I used KerasClassifier() and GridSearchCV(). The following code illustrates my approach for the gridsearch.
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer
from tensorflow import autograph
#Building a function to create the classifier
def build_classifier(nb_layers, nb_nodes, optimizer, dropout, activation_fn):
classifier=Sequential()
classifier.add(Dense(units = nb_nodes, input_shape = (100, ), activation = activation_fn))
for i in range(nb_layers-1) :
classifier.add(Dense(units = nb_nodes, activation = activation_fn, kernel_initializer = "uniform"))
classifier.add(Dropout(dropout))
classifier.add(Dense(units = 100, activation = 'tanh'))
classifier.compile(optimizer=optimizer, loss = tf.autograph.experimental.do_not_convert(my_loss_fn), metrics= ['accuracy', tf.autograph.experimental.do_not_convert(my_metric_fn)])
return classifier
#Creating a scorer to feed to the GridSearchCV()
my_scorer = make_scorer(accuracy, greater_is_better = True)
classifier=KerasClassifier(build_fn=build_classifier)
parameters={'batch_size':[13, 31],'epochs':[100, 150], 'optimizer':['adam', 'rmsprop'], 'dropout' : [0.2, 0.1], 'nb_layers' : [2, 3], 'nb_nodes' : [45, 50, 110, 115], 'activation_fn' : ['relu', 'tanh']}
grid_search=GridSearchCV(estimator=classifier, scoring = my_scorer, param_grid=parameters, cv=5, verbose = 1)
grid_search=grid_search.fit(X_train_, y_train_raw)
When I fit my GridSearchCV() object I get the following error at the end of the first combination of hyperparameters (when the scoring is computed) :
TypeError: object of type 'numpy.int32' has no len()
I investigated by adding print commandes inside my accuracy() function
#print(y_true.shape, y_true)
#print(y_pred.shape, y_pred)
to print both the shape and the array y_true and y_pred given as inputs for my accuracy() function used as the scoring in the GridSearchCV() object.
I found out that y_true.shape == (555, 100) but y_pred.shape == (555,). The value 555 corresponds to the number of lines of the fifth validation set because cv = 5.
However, I do not understand why the prediction of the gridsearch is not a multi-output prediction even though the number of nodes of the last layer of the classifier is (100,).
This was a regression problem so I used KerasRegressor() instead and it solved the issue. I guess that for a multi-output classification problem, KerasClassifier() expect the output to be a 2D hot encoded array.

ValueError: It seems that you are using the Keras 2 and you are passing both `kernel_size` and `strides` as integer positional arguments

I'm a computer science undergraduate student in the 4th semester, and I'm learning about Machine Learning in this lockdown.
from __future__ import print_function
from keras import backend as K
K.common.set_image_dim_ordering('th') # ensure our dimension notation matches
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.layers import Reshape
from keras.layers.core import Activation
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import UpSampling2D
from keras.layers.convolutional import Convolution2D, AveragePooling2D
from keras.layers.core import Flatten
from keras.optimizers import SGD, Adam
from keras.datasets import mnist
from keras import utils
import numpy as np
from PIL import Image, ImageOps
import argparse
import math
import os
import os.path
import glob
def generator_model():
model = Sequential()
model.add(Dense(input_dim=100, output_dim=1024))
model.add(Activation('tanh'))
model.add(Dense(128*8*8))
model.add(BatchNormalization())
model.add(Activation('tanh'))
model.add(Reshape((128, 8, 8), input_shape=(128*8*8,)))
model.add(UpSampling2D(size=(4, 4)))
model.add(Convolution2D(64, 5, 5, border_mode='same'))
model.add(Activation('tanh'))
model.add(UpSampling2D(size=(4, 4)))
model.add(Convolution2D(1, 5, 5, border_mode='same'))
model.add(Activation('tanh'))
return model
def discriminator_model():
model = Sequential()
model.add(Convolution2D(64, 5, 5, border_mode='same', input_shape=(1, 128, 128)))
model.add(Activation('tanh'))
model.add(AveragePooling2D(pool_size=(4, 4)))
model.add(Convolution2D(128, 5, 5))
model.add(Activation('tanh'))
model.add(AveragePooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(256))
model.add(Activation('tanh'))
model.add(Dense(1))
model.add(Activation('sigmoid'))
return model
def generator_containing_discriminator(generator, discriminator):
model = Sequential()
model.add(generator)
discriminator.trainable = False
model.add(discriminator)
return model
def combine_images(generated_images):
num = generated_images.shape[0]
width = int(math.sqrt(num))
height = int(math.ceil(float(num)/width))
shape = generated_images.shape[2:]
image = np.zeros((height*shape[0], width*shape[1]),
dtype=generated_images.dtype)
for index, img in enumerate(generated_images):
i = int(index/width)
j = index % width
image[i*shape[0]:(i+1)*shape[0], j*shape[1]:(j+1)*shape[1]] = \
img[0, :, :]
return image
model = generator_model()
print(model.summary())
def load_data(pixels=128, verbose=False):
print("Loading data")
X_train = []
paths = glob.glob(os.path.normpath(os.getcwd() + '/logos/*.jpg'))
for path in paths:
if verbose: print(path)
im = Image.open(path)
im = ImageOps.fit(im, (pixels, pixels), Image.ANTIALIAS)
im = ImageOps.grayscale(im)
#im.show()
im = np.asarray(im)
X_train.append(im)
print("Finished loading data")
return np.array(X_train)
def train(epochs, BATCH_SIZE, weights=False):
"""
:param epochs: Train for this many epochs
:param BATCH_SIZE: Size of minibatch
:param weights: If True, load weights from file, otherwise train the model from scratch.
Use this if you have already saved state of the network and want to train it further.
"""
X_train = load_data()
X_train = (X_train.astype(np.float32) - 127.5)/127.5
X_train = X_train.reshape((X_train.shape[0], 1) + X_train.shape[1:])
discriminator = discriminator_model()
generator = generator_model()
if weights:
generator.load_weights('goodgenerator.h5')
discriminator.load_weights('gooddiscriminator.h5')
discriminator_on_generator = \
generator_containing_discriminator(generator, discriminator)
d_optim = SGD(lr=0.0005, momentum=0.9, nesterov=True)
g_optim = SGD(lr=0.0005, momentum=0.9, nesterov=True)
generator.compile(loss='binary_crossentropy', optimizer="SGD")
discriminator_on_generator.compile(
loss='binary_crossentropy', optimizer=g_optim)
discriminator.trainable = True
discriminator.compile(loss='binary_crossentropy', optimizer=d_optim)
noise = np.zeros((BATCH_SIZE, 100))
for epoch in range(epochs):
print("Epoch is", epoch)
print("Number of batches", int(X_train.shape[0]/BATCH_SIZE))
for index in range(int(X_train.shape[0]/BATCH_SIZE)):
for i in range(BATCH_SIZE):
noise[i, :] = np.random.uniform(-1, 1, 100)
image_batch = X_train[index*BATCH_SIZE:(index+1)*BATCH_SIZE]
generated_images = generator.predict(noise, verbose=0)
#print(generated_images.shape)
if index % 20 == 0 and epoch % 10 == 0:
image = combine_images(generated_images)
image = image*127.5+127.5
destpath = os.path.normpath(os.getcwd()+ "/logo-generated-images/"+str(epoch)+"_"+str(index)+".png")
Image.fromarray(image.astype(np.uint8)).save(destpath)
X = np.concatenate((image_batch, generated_images))
y = [1] * BATCH_SIZE + [0] * BATCH_SIZE
d_loss = discriminator.train_on_batch(X, y)
print("batch %d d_loss : %f" % (index, d_loss))
for i in range(BATCH_SIZE):
noise[i, :] = np.random.uniform(-1, 1, 100)
discriminator.trainable = False
g_loss = discriminator_on_generator.train_on_batch(
noise, [1] * BATCH_SIZE)
discriminator.trainable = True
print("batch %d g_loss : %f" % (index, g_loss))
if epoch % 10 == 9:
generator.save_weights('goodgenerator.h5', True)
discriminator.save_weights('gooddiscriminator.h5', True)
def clean(image):
for i in range(1, image.shape[0] - 1):
for j in range(1, image.shape[1] - 1):
if image[i][j] + image[i+1][j] + image[i][j+1] + image[i-1][j] + image[i][j-1] > 127 * 5:
image[i][j] = 255
return image
def generate(BATCH_SIZE):
generator = generator_model()
generator.compile(loss='binary_crossentropy', optimizer="SGD")
generator.load_weights('goodgenerator.h5')
noise = np.zeros((BATCH_SIZE, 100))
a = np.random.uniform(-1, 1, 100)
b = np.random.uniform(-1, 1, 100)
grad = (b - a) / BATCH_SIZE
for i in range(BATCH_SIZE):
noise[i, :] = np.random.uniform(-1, 1, 100)
generated_images = generator.predict(noise, verbose=1)
#image = combine_images(generated_images)
print(generated_images.shape)
for image in generated_images:
image = image[0]
image = image*127.5+127.5
Image.fromarray(image.astype(np.uint8)).save("dirty.png")
Image.fromarray(image.astype(np.uint8)).show()
clean(image)
image = Image.fromarray(image.astype(np.uint8))
image.show()
image.save("clean.png")
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument("--mode", type=str)
parser.add_argument("--batch_size", type=int, default=128)
parser.add_argument("--nice", dest="nice", action="store_true")
parser.set_defaults(nice=False)
args = parser.parse_args()
return args
train(400, 10, False)
generate(1)
I was trying this GAN code from this GitHub repository, for learning about Generative Adversarial Networks, but the below error occurred. Can you please tell me where the definitions are provided in the code? Please help me!
The Troublesome Line:-
train(400, 10, False)
This is the error:-
ValueError: It seems that you are using the Keras 2 and you are passing both `kernel_size` and `strides` as integer positional arguments. For safety reasons, this is disallowed. Pass `strides` as a keyword argument instead.
The error arises from every addition of a Conv2D layer in your model. You need to change the line in your code
model.add(Convolution2D(64, 5, 5, border_mode='same'))
to something like (depending on what exactly you want)
model.add(Conv2D(64,kernel_size=5,strides=2,padding='same'))
Notice that I have explicity named the argument strides here because the error says I should pass it as a keyword argument.

Translating LSTM model from Keras to Pytorch

I am having a hard time translating a quite simple LSTM model from Keras to Pytorch. X (get it here) corresponds to 1152 samples of 90 timesteps, each timestep has only 1 dimension. y (here) is a single prediction at t = 91 for all 1152 samples.
In Keras:
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, LSTM
import numpy as np
import pandas as pd
X = pd.read_csv('X.csv', header = None).values
X.shape
y = pd.read_csv('y.csv', header = None).values
y.shape
# From Keras documentation [https://keras.io/layers/recurrent/]:
# Input shape 3D tensor with shape (batch_size, timesteps, input_dim).
X = np.reshape(X, (1152, 90, 1))
regressor = Sequential()
regressor.add(LSTM(units = 100, return_sequences = True, input_shape = (90, 1)))
regressor.add(Dropout(0.3))
regressor.add(LSTM(units = 50, return_sequences = True))
regressor.add(Dropout(0.3))
regressor.add(LSTM(units = 50, return_sequences = True))
regressor.add(Dropout(0.3))
regressor.add(LSTM(units = 50))
regressor.add(Dropout(0.3))
regressor.add(Dense(units = 1, activation = 'linear'))
regressor.compile(optimizer = 'rmsprop', loss = 'mean_squared_error', metrics = ['mean_absolute_error'])
regressor.fit(X, y, epochs = 10, batch_size = 32)
... leads me to:
# Epoch 10/10
# 1152/1152 [==============================] - 33s 29ms/sample - loss: 0.0068 - mean_absolute_error: 0.0628
Then in Pytorch:
import torch
from torch import nn, optim
from sklearn.metrics import mean_absolute_error
X = pd.read_csv('X.csv', header = None).values
y = pd.read_csv('y.csv', header = None).values
X = torch.tensor(X, dtype = torch.float32)
y = torch.tensor(y, dtype = torch.float32)
dataset = torch.utils.data.TensorDataset(X, y)
loader = torch.utils.data.DataLoader(dataset, batch_size = 32, shuffle = True)
class regressor_LSTM(nn.Module):
def __init__(self):
super().__init__()
self.lstm1 = nn.LSTM(input_size = 1, hidden_size = 100)
self.lstm2 = nn.LSTM(100, 50)
self.lstm3 = nn.LSTM(50, 50, dropout = 0.3, num_layers = 2)
self.dropout = nn.Dropout(p = 0.3)
self.linear = nn.Linear(in_features = 50, out_features = 1)
def forward(self, X):
# From the Pytorch documentation [https://pytorch.org/docs/stable/_modules/torch/nn/modules/rnn.html]:
# **input** of shape `(seq_len, batch, input_size)`
X = X.view(90, 32, 1)
# I am discarding hidden/cell states since in Keras I am using a stateless approach
# [https://keras.io/examples/lstm_stateful/]
X, _ = self.lstm1(X)
X = self.dropout(X)
X, _ = self.lstm2(X)
X = self.dropout(X)
X, _ = self.lstm3(X)
X = self.dropout(X)
X = self.linear(X)
return X
regressor = regressor_LSTM()
criterion = nn.MSELoss()
optimizer = optim.RMSprop(regressor.parameters())
for epoch in range(10):
running_loss = 0.
running_mae = 0.
for i, data in enumerate(loader):
inputs, labels = data
optimizer.zero_grad()
outputs = regressor(inputs)
outputs = outputs[-1].view(*labels.shape)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
running_loss += loss.item()
mae = mean_absolute_error(labels.detach().cpu().numpy().flatten(), outputs.detach().cpu().numpy().flatten())
running_mae += mae
print('EPOCH %3d: loss %.5f - MAE %.5f' % (epoch+1, running_loss/len(loader), running_mae/len(loader)))
... leads me to:
# EPOCH 10: loss 0.04220 - MAE 0.16762
You can notice that both loss and MAE are quite different (Pytorch's are much higher). If I use Pytorch's model to predict the values, they all return as a constant.
What am I doing wrong?
Oh I believe I made considerable progress. It seems that the way to represent y is different between Keras and Pytorch. In Keras, we should pass it as a single value representing one timestep in the future (or, at least, for the problem I am trying to solve). But in Pytorch, y must be X shifted one timestep to the future. It is like this:
time_series = [0, 1, 2, 3, 4, 5]
X = [0, 1, 2, 3, 4]
# Keras:
y = [5]
# Pytorch:
y = [1, 2, 3, 4, 5]
This way, Pytorch compares all values in the time slice when calculating loss. I believe Keras rearranges the data under the hood to conform to this approach, as the code works when fed the variables just like that. But in Pytorch, I was estimating loss based only on one value (the one I was trying to predict), not the whole series, therefore I believe it could not correctly capture the time dependency.
When taking this in consideration, I got to:
EPOCH 100: loss 0.00551 - MAE 0.058435
And, most importantly, comparing true and predicted values in a separate dataset got me to
The patterns were clearly captured by the model.
Hooray!

InvalidArgumentError logits and labels must be same size: logits_size=[3215,25] labels_size=[10,25]

I was having quite a few errors (OOM, shape problems, etc) which I had managed to fix somehow.
But I'm unable to get my head around this error. I have searched quite a bit and I have also tried the sparse cross entropy with logits method in tensorflow and the tf.squeeze function also but that also didn't help me in resolving this error. Here is the link of the code (it's a github gist with the entire stacktrace and errors).
Code Link
Here is the link for the data set(It's around 500 Mb)
Dataset Link
Here is the Code (just in Case):
from PIL import Image
import numpy as np
import glob
from numpy import array
import pandas as pd
from sklearn.preprocessing import LabelEncoder,OneHotEncoder
import h5py
import tensorflow as tf
def loading_saving_image_as_grayscale_train(img):
##combined_path='M:/PycharmProjects/AI+DL+CP/test_img'+img
loading=Image.open(img)
loading=loading.resize((28,28),Image.ANTIALIAS)
loading=loading.convert('L')
#loading.show()
conversion_to_array=np.asarray(loading,dtype=float)
train_data.append(conversion_to_array)
def loading_saving_image_as_grayscale_test(img):
#combined_path = 'M:/PycharmProjects/AI+DL+CP/train_img/' + img
#print(combined_path)
loading=Image.open(img,'r')
loading=loading.resize((28,28),Image.ANTIALIAS)
loading=loading.convert('L')
conversion_to_array=np.asarray(loading,dtype=float)
test_data.append(conversion_to_array)
import os
import requests, zipfile, io
import pandas as pd
#url = requests.get('https://he-s3.s3.amazonaws.com/media/hackathon/deep-learning-challenge-1/identify-the-objects/a0409a00-8-dataset_dp.zip')
#data = zipfile.ZipFile(io.BytesIO(url.content))
#data.extractall()
#os.listdir()
dataframe1=pd.read_csv('test.csv')
dataframe1.index=dataframe1.index+1
only_index=dataframe['image_id']
test_data=[]
train_data=[]
train=glob.glob('train_img/*.png')
test=glob.glob('test_img/*.png')
#other=loading_saving_image_as_grayscale('M:/PycharmProjects/AI+DL+CP/test_img/test_1000b.png')
#print(Image.open('M:/PycharmProjects/AI+DL+CP/test_img/test_1000b.png'))
#print(test)
#loading_sample=Image.open('M:/PycharmProjects/AI+DL+CP/test_img/test_1000b.png')
#loading_sample.show()
#print(train)
#print(test)
for data in train:
#print(data)
loading_saving_image_as_grayscale_train(data)
for item in test:
#print(item)
loading_saving_image_as_grayscale_test(item)
#print(train_data)
#print(test_data)
'''with Image.fromarray(train_data[1]) as img:
width,height=img.size
print(width,height)
'''
def OneHot(label,n_classes):
label=np.array(label).reshape(-1)
label=np.eye(n_classes)[label]
return label
dataframe=pd.read_csv('train.csv')
train_data=np.asarray(train_data)
test_data=np.asarray(test_data)
uni=dataframe['label']
dataframe1=pd.read_csv('test.csv')
dataframe1.index=dataframe1.index+1
only_index=dataframe['image_id']
label=LabelEncoder()
integer_encoding=label.fit_transform(uni)
#del uni
#del dataframe
#print(integer_encoding)
binary=OneHotEncoder(sparse=False)
integer_encoding=integer_encoding.reshape(len(integer_encoding),1)
onehot=binary.fit_transform(integer_encoding)
train_data=np.reshape(train_data,[-1,28,28,1])
test_data=np.reshape(test_data,[-1,28,28,1])
#onehot=np.reshape(onehot,[-1,10])
train_data=np.transpose(train_data,(0,2,1,3))
test_data=np.transpose(test_data,(0,2,1,3))
train_data=train_data.astype(np.float32)
test_data=test_data.astype(np.float32)
print(train_data.shape,test_data.shape,onehot.shape)
graph = tf.Graph()
with graph.as_default():
# placeholders for input data batch_size x 32 x 32 x 3 and labels batch_size x 10
data_placeholder = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
label_placeholder = tf.placeholder(tf.int32, shape=[None, 25])
# defining decaying learning rate
global_step = tf.Variable(0)
decay_rate = tf.train.exponential_decay(1e-4, global_step=global_step, decay_steps=10000, decay_rate=0.97)
layer1_weights = tf.Variable(tf.truncated_normal([3, 3, 1, 64],stddev=0.1))
layer1_biases = tf.Variable(tf.constant(0.1, shape=[64]))
layer2_weights = tf.Variable(tf.truncated_normal([3, 3, 64,32],stddev=0.1))
layer2_biases = tf.Variable(tf.constant(0.1,shape=[32]))
layer3_weights = tf.Variable(tf.truncated_normal([2, 2, 32, 20],stddev=0.1))
layer3_biases = tf.Variable(tf.constant(0.1,shape=[20]))
layer4_weights = tf.Variable(tf.truncated_normal([20,25],stddev=0.1))
layer4_biases = tf.Variable(tf.constant(0.1,shape=[25]))
layer5_weights = tf.Variable(tf.truncated_normal([25, 25], stddev=0.1))
layer5_biases = tf.Variable(tf.constant(0.1, shape=[25]))
def layer_multiplication(data_input_given):
#Convolutional Layer 1
#data_input_given=np.reshape(data_input_given,[-1,64,64,1])
CNN1=tf.nn.relu(tf.nn.conv2d(data_input_given,layer1_weights,strides=[1,1,1,1],padding='SAME')+layer1_biases)
print('CNN1 Done!!')
#Pooling Layer
Pool1=tf.nn.max_pool(CNN1,ksize=[1,4,4,1],strides=[1,4,4,1],padding='SAME')
print('Pool1 DOne')
#second Convolution layer
CNN2=tf.nn.relu(tf.nn.conv2d(Pool1,layer2_weights,strides=[1,1,1,1],padding='SAME'))+layer2_biases
print('CNN2 Done')
#Second Pooling
Pool2 = tf.nn.max_pool(CNN2, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')
print('pool2 Done')
#Third Convolutional Layer
CNN3 = tf.nn.relu(tf.nn.conv2d(Pool2, layer3_weights, strides=[1, 1, 1, 1], padding='SAME')) + layer3_biases
print('CNN3 Done')
#Third Pooling Layer
Pool3 = tf.nn.max_pool(CNN3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
print('Pool3 DOne')
#Fully Connected Layer
Pool4=tf.nn.max_pool(Pool3,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')
FullyCon=tf.reshape(Pool4,[-1,20])
FullyCon=tf.nn.relu(tf.matmul(FullyCon,layer4_weights)+layer4_biases)
print('Fullyconnected Done')
dropout = tf.nn.dropout(FullyCon, 0.4)
dropout=tf.reshape(dropout,[-1,25])
dropout=tf.matmul(dropout,layer5_weights)+layer5_biases
#print(dropout.shape)
return dropout
train_input = layer_multiplication(train_data)
print(train_input.shape)
loss = (tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=label_placeholder,logits=train_input))
+ 0.01 * tf.nn.l2_loss(layer1_weights)
+ 0.01 * tf.nn.l2_loss(layer2_weights)
+ 0.01 * tf.nn.l2_loss(layer3_weights)
+ 0.01 * tf.nn.l2_loss(layer4_weights)
)
#other=(tf.squeeze(label_placeholder))
#print(tf.shape())
optimizer = tf.train.GradientDescentOptimizer(name='Stochastic', learning_rate=decay_rate).minimize(loss,global_step=global_step)
#print(train_input.shape)
batch_size = 10
num_steps=10000
prediction=[]
with tf.Session(graph=graph) as session:
tf.global_variables_initializer().run()
print('Initialized')
for i in range(num_steps):
print("in loop")
offset = (i * batch_size) % (onehot.shape[0] - batch_size)
batch_data = train_data[offset:(offset + batch_size), :, :]
batch_labels = onehot[offset:(offset + batch_size), :]
print("training")
feed_dict = {data_placeholder: batch_data, label_placeholder: batch_labels}
_, l, predictions = session.run(
[optimizer, loss, train_input], feed_dict=feed_dict)
print(sess.run(tf.argmax(label_placeholder, 1), feed_dict={x:test_data}))
prediction.append(sess.run(tf.argmax(label_placeholder,1),feed_dict={x:test_data}))
print('Finished')
submit=pd.Dataframe({'image_id':only_index, 'label':prediction})
submit.to_csv('submit.csv',index=False)
I also had a doubt regarding predicting class labels. Can someone tell me whether the method I'm using for storing the predicted class labels will work or not?
The reshape operations do not make sense:
FullyCon=tf.reshape(Pool4,[-1,20])
this will collapse batch dimension and feature dimensions.
Why would output of Pool4 have 20 dimensions? The fact it has 20 kernels does not mean it has 20 dimensions. Dimensionality is 20 * size of the image on this level of convolutions, which will be much bigger (my guess is it will be 6430).
It should be something among the lines of
output_shape = Pool4.shape[1] * Pool4.shape[2] * Pool4.shape[3]
FullyCon=tf.reshape(Pool4, [-1, output_shape])
and then you will have to change final layer accordingly (to match shapes).
The error has been fixed after reshaping everything properly and also in the softmax with logits part,i had to send the data_placeholder for logits.After doing this the issue got cleared.

Resources