Related
I am using PyTorch version: 1.9.0+cu102 with Convolutional Autoencoder for CIFAR-10 dataset as follows:
# Define transformations for training and test sets-
transform_train = transforms.Compose(
[
# transforms.RandomCrop(32, padding = 4),
# transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
# transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
]
)
transform_test = transforms.Compose(
[
transforms.ToTensor(),
# transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
]
)
# Load dataset-
train_dataset = torchvision.datasets.CIFAR10(
root = './data', train = True,
download = True, transform = transform_train
)
test_dataset = torchvision.datasets.CIFAR10(
root = './data', train = False,
download = True, transform = transform_test
)
print(f"len(train_dataset) = {len(train_dataset)} & len(test_dataset) = {len(test_dataset)}")
# len(train_dataset) = 50000 & len(test_dataset) = 10000
batch_size = 64
# Create training and testing loaders-
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size = batch_size,
shuffle = True
)
test_loader = torch.utils.data.DataLoader(
test_dataset, batch_size = batch_size,
shuffle = False
)
print(f"len(train_loader) = {len(train_loader)} & len(test_loader) = {len(test_loader)}")
# len(train_loader) = 782 & len(test_loader) = 157
# Sanity check-
len(train_dataset) / batch_size, len(test_dataset) / batch_size
# (781.25, 156.25)
# Get some random training images-
images, labels = next(iter(train_loader))
print(f"images.shape: {images.shape} & labels.shape: {labels.shape}")
# images.shape: torch.Size([64, 3, 32, 32]) & labels.shape: torch.Size([64])
LEARNING_RATE = 0.001
num_epochs = 20
class Reshape(nn.Module):
def __init__(self, *args):
super().__init__()
self.shape = args
def forward(self, x):
return x.view(self.shape)
class Trim(nn.Module):
def __init__(self, *args):
super().__init__()
def forward(self, x):
return x[:, :, :32, :32]
encoder = nn.Sequential(
nn.Conv2d(
in_channels = 3, out_channels = 32,
kernel_size = 3, padding = 1,
stride = 1, bias = True
),
nn.LeakyReLU(negative_slope = 0.01),
nn.Conv2d(
in_channels = 32, out_channels = 64,
kernel_size = 3, padding = 1,
stride = 2, bias = True
),
nn.LeakyReLU(negative_slope = 0.01),
nn.Conv2d(
in_channels = 64, out_channels = 64,
kernel_size = 3, padding = 1,
stride = 2, bias = True
),
nn.LeakyReLU(negative_slope = 0.01),
nn.Conv2d(
in_channels = 64, out_channels = 64,
kernel_size = 3, padding = 1,
stride = 1, bias = True
),
nn.LeakyReLU(negative_slope = 0.01),
nn.Flatten(),
nn.Linear(
in_features = 4096, out_features = 1500,
bias = True
),
nn.Linear(
in_features = 1500, out_features = 500,
bias = True
),
nn.Linear(
in_features = 500, out_features = 100,
bias = True
)
)
# Sanity check-
x = torch.rand(size = (32, 3, 32, 32))
print(f"x.shape = {x.shape}")
encoder_op = encoder(x)
print(f"encoder_op.shape = {encoder_op.shape}")
# x.shape = torch.Size([32, 3, 32, 32])
# encoder_op.shape = torch.Size([32, 100])
decoder = nn.Sequential(
nn.Linear(
in_features = 100, out_features = 500,
bias = True),
nn.Linear(
in_features = 500, out_features = 1500,
bias = True),
nn.Linear(
in_features = 1500, out_features = 4096,
bias = True),
Reshape(-1, 64, 8, 8),
nn.ConvTranspose2d(
in_channels = 64, out_channels = 64,
kernel_size = 3, stride = 1,
padding = 1, bias = True),
# output: torch.Size([32, 64, 8, 8])
nn.ConvTranspose2d(
in_channels = 64, out_channels = 64,
kernel_size = 3, stride = 2,
padding = 1, bias = True),
# output: torch.Size([32, 64, 15, 15])
nn.ConvTranspose2d(
in_channels = 64, out_channels = 32,
kernel_size = 3, stride = 2,
padding = 0, bias = True),
# torch.Size([32, 32, 31, 31])
nn.ConvTranspose2d(
in_channels = 32, out_channels = 3,
kernel_size = 3, stride = 1,
padding = 0, bias = True),
# output: torch.Size([32, 3, 33, 33])
Trim(),
# (3, 33, 33) -> (3, 32, 32)
nn.Sigmoid()
)
# Sanity check-
decoder(encoder_op).shape
# torch.Size([32, 3, 32, 32])
class AutoEncoder(nn.Module):
def __init__(self):
super().__init__()
self.encoder = encoder
self.decoder = decoder
def forward(self, x):
x = self.encoder(x)
x = self.decoder(x)
return x
# Initialize an autoencoder instance-
model = AutoEncoder()
# Move model to (GPU) device-
model.to(device)
# Specify optimizer and loss function-
optimizer = torch.optim.Adam(model.parameters(), lr = 0.001)
loss_fn = F.mse_loss
num_epochs = 15
# Python3 lists to hold training metrics-
trainining_loss = []
validation_loss = []
def compute_epoch_loss_autoencoder(model, data_loader, loss_fn, device):
model.eval()
curr_loss, num_examples = 0., 0
with torch.no_grad():
for features, _ in data_loader:
features = features.to(device)
logits = model(features)
loss = loss_fn(logits, features, reduction='sum')
num_examples += features.size(0)
curr_loss += loss
curr_loss = curr_loss / num_examples
return curr_loss
start_time = time.time()
for epoch in range(num_epochs):
running_loss = 0.0
model.train()
for batch_idx, (features, _) in enumerate(train_loader):
features = features.to(device)
# forward and back prop-
logits = model(features) # make predictions using model
loss = loss_fn(logits, features)
optimizer.zero_grad()
# Perform backprop-
loss.backward()
# Update model parameters-
optimizer.step()
# Compute model's performance-
running_loss += loss.item() * features.size(0)
# Compute loss using training dataset-
epoch_loss = running_loss / len(train_dataset)
trainining_loss.append(epoch_loss)
# Compute loss using validation dataset-
val_loss = compute_epoch_loss_autoencoder(
model, test_loader,
loss_fn, device
)
validation_loss.append(val_loss)
print(f"Epoch = {epoch + 1}: Autoencoder train loss = {epoch_loss:.4f} & val loss = {val_loss:.4f}")
end_time = time.time()
# Get some validation images-
for img, label in test_loader:
break
img.shape, label.shape
# (torch.Size([64, 3, 32, 32]), torch.Size([64]))
img.to(device)
# Pass batch size = 64 images through encoder to get latent space representations-
model.encoder(img)
This line gives me the error:
RuntimeError Traceback (most recent call
last)
in ()
----> 1 model.encoder(img)
4 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/conv.py in
_conv_forward(self, input, weight, bias)
438 _pair(0), self.dilation, self.groups)
439 return F.conv2d(input, weight, bias, self.stride,
--> 440 self.padding, self.dilation, self.groups)
441
442 def forward(self, input: Tensor) -> Tensor:
RuntimeError: Input type (torch.FloatTensor) and weight type
(torch.cuda.FloatTensor) should be the same or input should be a MKLDNN tensor and weight is a dense tensor
What's going wrong?
Thanks!
I am new to pytorch.
I am trying to use a different approach when training MNIST dataset in pytorch.
Normally, we put all shuffled numbers for each batch.
What I want to do is (assuming by batch size is 20 and there are 1000 training examples).
Make a batch look this this (I wrote only labels for simplicity)
batch_1 = [0, 0, 1, 0, ... ] only 0 and 1 for this batch
batch_2 = [6, 6, 7, 7, ... ] only 6 and 7 for this batch
batch_3 = [2, 3, 2, 2, ... ] only 2 and 3 for this batch
batch_4 = [4, 5, 5, 4, ... ] only 4 and 5 for this batch
batch_5 = [6, 6, 7, 7, ... ] only 6 and 7 for this batch
batch_6 = [8, 9, 9, 9, ... ] only 8 and 9 for this batch
batch_7 = [2, 3, 2, 2, ... ] only 2 and 3 for this batch
batch_8 = [4, 5, 5, 4, ... ] only 4 and 5 for this batch
...
batch_50 = [4, 5, 5, 4, ... ] only 4 and 5 for this batch
How can I do this in pytorch??
Thank you
So far this is my implementation for training / testing.
I will expand/manipulate this code below to develop the above.
import torch
from torch.utils.data import DataLoader, Subset
import torchvision.datasets as datasets
from torchvision import transforms
import numpy as np
mnist_trainset = datasets.MNIST(root='./data', train=True,
download=True,
transform=transforms.Compose([transforms.ToTensor()]))
mnist_testset = datasets.MNIST(root='./data',
train=False,
download=True,
transform=transforms.Compose([transforms.ToTensor()]))
class_inds = [torch.where(mnist_trainset.targets == class_idx)[0]
for class_idx in mnist_trainset.class_to_idx.values()]
train_dataloaders = [
DataLoader(dataset=Subset(mnist_trainset, inds),
batch_size=50,
shuffle=True,
drop_last=False
)
for inds in class_inds
]
test_dataloader = torch.utils.data.DataLoader(mnist_testset,
batch_size=50,
shuffle=False)
class Model(torch.nn.Module):
def __init__(self):
super(Model, self).__init__()
self.linear_1 = torch.nn.Linear(784, 256)
self.linear_2 = torch.nn.Linear(256, 10)
self.sigmoid = torch.nn.Sigmoid()
def forward(self, x):
x = x.reshape(x.size(0), -1)
x = self.linear_1(x)
x = self.sigmoid(x)
pred = self.linear_2(x)
return pred
model = Model()
optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
epochs = 20
criterion = torch.nn.CrossEntropyLoss()
test_acc = list()
for epoch in range(epochs):
print(f"epoch {epoch} started")
model.train()
itr = 0
iterators = list(map(iter, train_dataloaders))
while iterators:
itr = itr + 1
iterator = np.random.choice(iterators)
try:
image, label = next(iterator)
optimizer.zero_grad()
pred = model(image)
loss = criterion(pred, label)
loss.backward()
optimizer.step()
except StopIteration:
iterators.remove(iterator)
model.eval()
total = 0
for itr, (image, label) in enumerate(test_dataloader):
pred = model(image)
loss = criterion(pred, label)
# we now need softmax because we are testing.
pred = torch.nn.functional.softmax(pred, dim=1)
for i, p in enumerate(pred):
if label[i] == torch.max(p.data, 0)[1]:
total = total + 1
accuracy = total / len(mnist_testset)
# append accuracy here
test_acc.append(accuracy)
My network transposes an image, with size 62*71, to a vector of 124 outputs. In the test, I got the same output for each input. I checked 4000 cases.
I cannot seem to signify the problem because the learning seems to be fine, there is an improvement of the error and get a relatively low error.
Someone maybe knows what is the problem?
#load data
data_in= np.transpose(np.loadtxt("images_in_10000.csv", delimiter=',',dtype=np.float32))
data_out= np.transpose(np.loadtxt("out_to_image_10000.csv", delimiter=',',dtype=np.float32))
x_train = data_in[0:6000, :]
x_test = data_in[6000:10001,:]
y_train = data_out[0:6000, :]
y_test = data_out[6000:10001, :]
#parametersa
batch=100
epochs=7
learning_rate=0.01
n = x_test.shape[1] #4392
m = x_train.shape[0] #6000
d = y_test.shape[1] #124
l = y_test.shape[0] #4000
trainX = tf.placeholder(tf.float32, [batch, n])
trainY = tf.placeholder(tf.float32, [batch, d])
testX = tf.placeholder(tf.float32, [l, n])
testY = tf.placeholder(tf.float32, [l, d])
W_c1= tf.Variable(tf.random_normal([5, 5, 1, 32]))
W_c2= tf.Variable(tf.random_normal([5, 5, 32, 64]))
W_fc= tf.Variable(tf.random_normal([18 * 16 * 64, 128]))
W_out= tf.Variable(tf.random_normal([128, d]))
b_c1= tf.Variable(tf.random_normal([32]))
b_c2=tf.Variable(tf.random_normal([64]))
b_fc=tf.Variable(tf.random_normal([128]))
b_out=tf.Variable(tf.random_normal([d]))
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def maxpool2d(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
def convolutional_neural_network(x):
x = tf.reshape(x, shape=[-1,61,72, 1])
conv1 = tf.nn.relu(conv2d(x, W_c1) + b_c1)
conv1 = maxpool2d(conv1)
conv2 = tf.nn.relu(conv2d(conv1, W_c2) + b_c2)
conv2 = maxpool2d(conv2)
fc = tf.reshape(conv2, [-1, 18 * 16 * 64])
fc = tf.nn.relu(tf.matmul(fc, W_fc) + b_fc)
output = tf.matmul(fc, W_out) + b_out
return output
prediction = convolutional_neural_network(trainX)
cost =tf.reduce_mean(tf.pow(prediction-trainY,2))
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
prediction_t = convolutional_neural_network(testX)
losstest = tf.reduce_mean(tf.pow(prediction_t - testY, 2))
k=0
a = np.linspace(0, m - batch, m / batch, dtype=np.int32)
costshow = [0] * (len(a) * epochs)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(epochs):
epoch_loss = 0
for i in (np.linspace(0,m - batch, m / batch, dtype=np.int32)):
x = x_train[i:i + batch, :]
y = y_train[i:i + batch, :]
sess.run(optimizer, feed_dict={trainX: x, trainY: y})
cost_val = sess.run(cost, feed_dict={trainX: x, trainY: y})
costshow[k]=cost_val
print("Epoch=", '%04d' % (epoch + 1), "loss=", " {:.9f}".format(cost_val))
k = k + 1
print("finsh train-small ")
result = sess.run(prediction_t, feed_dict={testX: x_test})
test_loss = sess.run(losstest, feed_dict={testX: np.asarray(x_test), testY: np.asarray(y_test)})
print("Testing loss=", test_loss)
The metric behind a picture is clearly defined. The values of an image often ranges from 0-1 or 0-255. For CNN's you should normalize your input values (0-1).
Thus you have to be careful with your weight initialization. For example, if your have a bias of 0.6 and a value of 0.6, you get a 1.2 as image value and your plotting program thinks you are in the 0-255 range and everything is black.
So try to use the glorot-initializer for the weights and zero-initializer for the bias initializer:
Weights:
tf.get_variable("weight", shape=[5, 5, 1, 32], initializer=tf.glorot_uniform_initializer())
Bias:
tf.get_variable("bias", shape=[32], initializer=tf.zeros_initializer())
Furthermore, tf.Variabel is deprecated. It is better to use tf.get_variable.
My network starts to learn and looks okay on the first batch and then suddenly stops with TypeError on the second batch only! Why was it okay on the first batch then? Or why did it break after the first? Stupefying error... Here are the details:
I have built a CNN that is trying to predict 124 features for each image. The images are of size 61 x 72 pixels and the output vector of numbers are of size 124 x 1.
The images are floating point matrices with numbers between 1 and -1.
The information I'm trying to predict is in a CSV file, with each line describing an image. When I load the data for the training process I process each line and reshape them, also get the pictures the network is learning.
When I run my program, I get the following error on the second batch, however:
"TypeError: Fetch argument 2.7674865e+09 has invalid type , must be a string or Tensor. (Can not convert a float32 into a Tensor or Operation.)"
Can you please help pinpoint what the problem is? Here's my code:
import tensorflow as tf
import numpy as np
data_in=np.loadtxt(open("images.csv"), delimiter=',',dtype=np.float32);
data_out=np.loadtxt(open("outputmix-124.csv"),
delimiter=',',dtype=np.float32);
x_train = data_in[0:6000, :]
x_test = data_in[6000:10000,:]
y_train = data_out[0:6000, :]
y_test = data_out[6000:10000, :]
batch=600
epochs=10
n = x_test.shape[1] #4392
m = x_train.shape[0] #6000
d = y_test.shape[1] #124
l = y_test.shape[0] #4000
trainX= tf.placeholder(tf.float32, [batch, n], name="X")
trainY = tf.placeholder(tf.float32, [batch, d])
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def maxpool2d(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],
padding='SAME')
def convolutional_neural_network(x):
weights = {'W_c1': tf.Variable(tf.random_normal([5, 5, 1, 32])),
'W_c2': tf.Variable(tf.random_normal([5, 5, 32, 64])),
'W_fc': tf.Variable(tf.random_normal([18 * 16 * 64, 1024])),
'out': tf.Variable(tf.random_normal([1024, d]))}
biases = {'b_c1': tf.Variable(tf.random_normal([32])),
'b_c2': tf.Variable(tf.random_normal([64])),
'b_fc': tf.Variable(tf.random_normal([1024])),
'out': tf.Variable(tf.random_normal([d]))}
x = tf.reshape(x, shape=[-1,61,72, 1])
conv1 = tf.nn.relu(conv2d(x, weights['W_c1']) + biases['b_c1'])
conv1 = maxpool2d(conv1)
conv2 = tf.nn.relu(conv2d(conv1, weights['W_c2']) + biases['b_c2'])
conv2 = maxpool2d(conv2)
fc = tf.reshape(conv2, [-1, 18 * 16 * 64])
fc = tf.nn.relu(tf.matmul(fc, weights['W_fc']) + biases['b_fc'])
fc = tf.nn.dropout(fc, keep_rate)
output = tf.matmul(fc, weights['out']) + biases['out']
return output
def train_neural_network(x):
prediction = convolutional_neural_network(x)
cost =tf.reduce_mean(tf.pow(prediction-trainY,2))
optimizer = tf.train.AdamOptimizer().minimize(cost)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(epochs):
epoch_loss = 0
for i in (np.linspace(0,m - batch, m / batch, dtype=np.int32)):
x = x_train[i:i + batch, :]
y = y_train[i:i + batch, :]
sess.run(optimizer, feed_dict={trainX: x, trainY: y})
cost = sess.run(cost, feed_dict={trainX: x, trainY: y})
print("Epoch=", '%04d' % (epoch + 1), "loss=", "
{:.9f}".format(cost))
epoch_loss += cost
print('Epoch', epoch, 'completed out of', epochs, 'loss:',
epoch_loss)
train_neural_network(trainX)
This is a fairly typical mistake. The problem is with the variable cost. First you assign the loss calculation tensor to it in the second line of the function train_neural_network():
cost =tf.reduce_mean(tf.pow(prediction-trainY,2))
Then when you run the training and the cost calculation, you do this, and this is where it gets messed up:
cost = sess.run(cost, feed_dict={trainX: x, trainY: y})
because you assign the value of the loss to cost, which is now a simple floating point number, instead of a Tensor. The next time around sess.run() gets a floating point number instead of a tensor as a first argument, and the error above is printed.
Use something like cost_val for storing the value of the loss and leave cost to store the reference to the tensor. You of course need to update the lines that print the value as well, so these three lines I've changed:
cost_val = sess.run(cost, feed_dict={trainX: x, trainY: y})
print("Epoch=", '%04d' % (epoch + 1), "loss=", " {:.9f}".format(cost_val))
epoch_loss += cost_val
I'm posting the full revised version here (tested code; note I've generated test data instead of loading; this is a loadable and testable example for anyone but you need to change it back to load your actual data):
import tensorflow as tf
import numpy as np
keep_rate = 0.8
#data_in=np.loadtxt(open("images.csv"), delimiter=',',dtype=np.float32);
#data_out=np.loadtxt(open("outputmix-124.csv"),
# delimiter=',',dtype=np.float32);
data_in = np.random.normal( size = ( 10000, 4392 ) )
data_out = np.random.normal( size = ( 10000, 124 ) )
x_train = data_in[0:6000, :]
x_test = data_in[6000:10000,:]
y_train = data_out[0:6000, :]
y_test = data_out[6000:10000, :]
batch=600
epochs=10
n = x_test.shape[1] #4392
m = x_train.shape[0] #6000
d = y_test.shape[1] #124
l = y_test.shape[0] #4000
trainX = tf.placeholder(tf.float32, [batch, n], name="X")
trainY = tf.placeholder(tf.float32, [batch, d])
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def maxpool2d(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],
padding='SAME')
def convolutional_neural_network(x):
weights = {'W_c1': tf.Variable(tf.random_normal([5, 5, 1, 32])),
'W_c2': tf.Variable(tf.random_normal([5, 5, 32, 64])),
'W_fc': tf.Variable(tf.random_normal([18 * 16 * 64, 1024])),
'out': tf.Variable(tf.random_normal([1024, d]))}
biases = {'b_c1': tf.Variable(tf.random_normal([32])),
'b_c2': tf.Variable(tf.random_normal([64])),
'b_fc': tf.Variable(tf.random_normal([1024])),
'out': tf.Variable(tf.random_normal([d]))}
x = tf.reshape(x, shape=[-1,61,72, 1])
conv1 = tf.nn.relu(conv2d(x, weights['W_c1']) + biases['b_c1'])
conv1 = maxpool2d(conv1)
conv2 = tf.nn.relu(conv2d(conv1, weights['W_c2']) + biases['b_c2'])
conv2 = maxpool2d(conv2)
fc = tf.reshape(conv2, [-1, 18 * 16 * 64])
fc = tf.nn.relu(tf.matmul(fc, weights['W_fc']) + biases['b_fc'])
fc = tf.nn.dropout(fc, keep_rate)
output = tf.matmul(fc, weights['out']) + biases['out']
return output
def train_neural_network(x):
prediction = convolutional_neural_network(x)
cost =tf.reduce_mean(tf.pow(prediction-trainY,2))
optimizer = tf.train.AdamOptimizer().minimize(cost)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(epochs):
epoch_loss = 0
for i in (np.linspace(0,m - batch, m / batch, dtype=np.int32)):
x = x_train[i:i + batch, :]
y = y_train[i:i + batch, :]
sess.run(optimizer, feed_dict={trainX: x, trainY: y})
cost_val = sess.run(cost, feed_dict={trainX: x, trainY: y})
print("Epoch=", '%04d' % (epoch + 1), "loss=", " {:.9f}".format(cost_val))
epoch_loss += cost_val
print('Epoch', epoch, 'completed out of', epochs, 'loss:',
epoch_loss)
train_neural_network(trainX)
I am trying to fit an LSTM network to a sin function. Currently, as far as I understand Keras, my code does only predict the next value. According to this link: Many to one and many to many LSTM examples in Keras it is a many to one model. However, my goal is to implement a Many-to-many model. Basically, I want to be able to predict let's say 10 values, to a given time. When I am trying to use
return_sequences=True (see line model.add(..)), which is supposed to be the solution, the following error occurs:
ValueError: Error when checking target: expected lstm_8 to have 3 dimensions, but got array with shape (689, 1)
Unfortunately, I have absolutely no clue why this happens. Is there a general rule how the input shape needs to be when using return_sequences=True ? Furthermore what exactly would I need to change? Thanks for any help.
import pandas
import numpy as np
import matplotlib.pylab as plt
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
import sklearn
from keras.models import Sequential
from keras.layers import Activation, LSTM
from keras import optimizers
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot
#generate sin function with noise
x = np.arange(0, 100, 0.1)
noise = np.random.uniform(-0.1, 0.1, size=(1000,))
Y = np.sin(x) + noise
# Perform feature scaling
scaler = MinMaxScaler()
Y = scaler.fit_transform(Y.reshape(-1, 1))
# split in train and test
train_size = int(len(Y) * 0.7)
test_size = len(Y) - train_size
train, test = Y[0:train_size,:], Y[train_size:len(Y),:]
def create_dataset(dataset, look_back=1):
dataX, dataY = [], []
for i in range(len(dataset)-look_back-1):
a = dataset[i:(i+look_back), 0]
dataX.append(a)
dataY.append(dataset[i + look_back, 0])
return np.array(dataX), np.array(dataY)
# reshape into X=t and Y=t+1
look_back = 10
X_train, y_train = create_dataset(train, look_back)
X_test, y_test = create_dataset(test, look_back)
# LSTM network expects the input data in form of [samples, time steps, features]
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
np.set_printoptions(threshold=np.inf)
# compile model
model = Sequential()
model.add(LSTM(1, input_shape=(look_back, 1)))#, return_sequences=True)) <== uncomment this
model.compile(loss='mean_squared_error', optimizer='adam')
SVG(model_to_dot(model).create(prog='dot', format='svg'))
model.fit(X_train, y_train, validation_data=(X_test, y_test),
batch_size=10, epochs=10, verbose=2)
prediction = model.predict(X_test, batch_size=1, verbose=0)
prediction.reshape(-1)
#Transform back to original representation
Y = scaler.inverse_transform(Y)
prediction = scaler.inverse_transform(prediction)
plt.plot(np.arange(0,Y.shape[0]), Y)
plt.plot(np.arange(Y.shape[0] - X_test.shape[0] , Y.shape[0]), prediction, 'red')
plt.show()
error = mean_squared_error(y_test, prediction)
print(error)
The problem is not the input, but the output.
The error says: "Error when checking target", target = y_train and y_test.
Because your lstm returns a sequence (return_sequences=True) the output dimention will be: (n_batch,lookback,1).
You can verify it by using model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
lstm_1 (LSTM) (None, 10, 1) 12
=================================================================
Total params: 12
Trainable params: 12
Non-trainable params: 0
_________________________________________________________________
You will need to change your create_dataset function so each ground truth will be shaped (lookback,1).
Something you might want to do:
for each seqeuence x in the train set,its y will be the next proceedings sequence.
For example, lets say we would like to learn something easier, the seqeuence will be the previous number plus 1 --> 1,2,3,4,5,6,7,8,9,10.
For loockback=4:
X_train[0] = 1,2,3,4
y_train[0] will be: 2,3,4,5
X_train[1] = 2,3,4,5
y_train[1] will be: 3,4,5,6
and so on...
I have simulated the data as #DvirSamuel suggested, and provided code for a LSTM and FNN. Note that for the LSTM, network_lstm.add(layers.Dense(1, activation = None)) is required if return_sequences = True is included in the previous layer.
## Simulate data.
np.random.seed(20180826)
Z = np.random.randint(0, 10, size = (11000, 1))
for i in range(10):
Z = np.concatenate((Z, (Z[:, -1].reshape(Z.shape[0], 1) + 1)), axis = 1)
X = Z[:, :-1]
Y = Z[:, 1:]
print(X.shape)
print(Y.shape)
## Training and validation data.
split = 10000
X_train = X[:split, :]
X_valid = X[split:, :]
Y_train = Y[:split, :]
Y_valid = Y[split:, :]
print(X_train.shape)
print(Y_train.shape)
print(X_valid.shape)
print(Y_valid.shape)
Code for a LSTM model:
## LSTM model.
X_lstm_train = X_train.reshape(X_train.shape[0], X_train.shape[1], 1)
X_lstm_valid = X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1)
Y_lstm_train = Y_train.reshape(Y_train.shape[0], Y_train.shape[1], 1)
Y_lstm_valid = Y_valid.reshape(Y_valid.shape[0], Y_valid.shape[1], 1)
# Define model.
network_lstm = models.Sequential()
network_lstm.add(layers.LSTM(64, activation = 'relu', input_shape = (X_lstm_train.shape[1], 1),
return_sequences = True))
network_lstm.add(layers.Dense(1, activation = None))
network_lstm.summary()
# Compile model.
network_lstm.compile(optimizer = 'rmsprop', loss = 'mean_squared_error')
# Fit model.
history_lstm = network_lstm.fit(X_lstm_train, Y_lstm_train, epochs = 5, batch_size = 32, verbose = True,
validation_data = (X_lstm_valid, Y_lstm_valid))
## Extract loss over epochs and predict.
# Extract loss.
loss_lstm = history_lstm.history['loss']
val_loss_lstm = history_lstm.history['val_loss']
epochs_lstm = range(1, len(loss_lstm) + 1)
plt.plot(epochs_lstm, loss_lstm, 'black', label = 'Training Loss')
plt.plot(epochs_lstm, val_loss_lstm, 'red', label = 'Validation Loss')
plt.title('LSTM: Training and Validation Loss')
plt.legend()
plt.title('First in Sequence')
plt.scatter(Y_train[:, 0], network_lstm.predict(X_lstm_train)[:, 0], alpha = 0.1)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.show()
plt.scatter(Y_valid[:, 0], network_lstm.predict(X_lstm_valid)[:, 0], alpha = 0.1)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.show()
plt.title('Last in Sequence')
plt.scatter(Y_train[:, -1], network_lstm.predict(X_lstm_train)[:, -1], alpha = 0.1)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.show()
plt.scatter(Y_valid[:, -1], network_lstm.predict(X_lstm_valid)[:, -1], alpha = 0.1)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.show()
Code for a FNN model:
## FNN model.
# Define model.
network_fnn = models.Sequential()
network_fnn.add(layers.Dense(64, activation = 'relu', input_shape = (X_train.shape[1],)))
network_fnn.add(Dense(10, activation = None))
network_fnn.summary()
# Compile model.
network_fnn.compile(optimizer = 'rmsprop', loss = 'mean_squared_error')
# Fit model.
history_fnn = network_fnn.fit(X_train, Y_train, epochs = 5, batch_size = 32, verbose = True,
validation_data = (X_valid, Y_valid))
## Extract loss over epochs.
# Extract loss.
loss_fnn = history_fnn.history['loss']
val_loss_fnn = history_fnn.history['val_loss']
epochs_fnn = range(1, len(loss_fnn) + 1)
plt.plot(epochs_fnn, loss_fnn, 'black', label = 'Training Loss')
plt.plot(epochs_fnn, val_loss_fnn, 'red', label = 'Validation Loss')
plt.title('FNN: Training and Validation Loss')
plt.legend()
plt.title('First in Sequence')
plt.scatter(Y_train[:, 1], network_fnn.predict(X_train)[:, 1], alpha = 0.1)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.show()
plt.scatter(Y_valid[:, 1], network_fnn.predict(X_valid)[:, 1], alpha = 0.1)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.show()
plt.title('Last in Sequence')
plt.scatter(Y_train[:, -1], network_fnn.predict(X_train)[:, -1], alpha = 0.1)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.show()
plt.scatter(Y_valid[:, -1], network_fnn.predict(X_valid)[:, -1], alpha = 0.1)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.show()
Shouldn't this:
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
be like this:
X_train = np.reshape((X_train.shape[0], X_train.shape[1], 1))
X_test = np.reshape((X_test.shape[0], X_test.shape[1], 1))
Can this be your problem? (1 years after xD)