When creating a LSTM layer using the functional moded below error is thrown
ValueError: Input 0 is incompatible with layer lstm_1: expected ndim=3, found ndim=2
Code to reproduce the issue
import keras
from keras.layers import Input, LSTM, Dense
from keras.models import Model
import numpy as np
def build_lstm(lstm_input, hidden_dim=256):
out = LSTM(hidden_dim, input_shape=(1, 129))(lstm_input)
return out
def test_lstm():
lstm_input = Input(shape=(129, ), name='lstm_input')
out = build_lstm(lstm_input)
predictions = Dense(256, activation='softmax')(out)
model = Model(inputs=lstm_input, outputs=predictions)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
model.summary()
return
Error Log
(NPI_Arch_Keras) C:\workspaces\NPL\project\NPI_Arch_Keras\npi>python npi_lstm_test.py
Using TensorFlow backend.
Traceback (most recent call last):
File "npi_lstm_test.py", line 35, in <module>
test_lstm()
File "npi_lstm_test.py", line 21, in test_lstm
out = build_lstm(lstm_input)
File "npi_lstm_test.py", line 15, in build_lstm
out = LSTM(hidden_dim, input_shape=(1, 129))(lstm_input)
File "C:\MyProgramFiles\Anaconda3\envs\NPI_Arch_Keras\lib\site-packages\keras\layers\recurrent.py", line 532, in __call__
return super(RNN, self).__call__(inputs, **kwargs)
File "C:\MyProgramFiles\Anaconda3\envs\NPI_Arch_Keras\lib\site-packages\keras\engine\base_layer.py", line 414, in __call__
self.assert_input_compatibility(inputs)
File "C:\MyProgramFiles\Anaconda3\envs\NPI_Arch_Keras\lib\site-packages\keras\engine\base_layer.py", line 311, in assert_input_compatibility
str(K.ndim(x)))
ValueError: Input 0 is incompatible with layer lstm_1: expected ndim=3, found ndim=2
The below version of the code which uses the Sequential model runs successfully
import keras
from keras.models import Sequential
from keras.layers import LSTM, Dense
import numpy as np
def build_lstm(hidden_dim=256, input_shape=None):
model = Sequential()
model.add(LSTM(hidden_dim, return_sequences=True, input_shape=input_shape))
model.add(LSTM(hidden_dim))
return model
def test_lstm():
model = Sequential()
model.add(build_lstm(input_shape=(1, 129)))
model.add(Dense(256, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
model.summary()
return
Success log
(NPI_Arch_Keras) C:\workspaces\NPL\project\NPI_Arch_Keras\npi>python npi_lstm_working.py
Using TensorFlow backend.
WARNING:tensorflow:From C:\MyProgramFiles\Anaconda3\envs\NPI_Arch_Keras\lib\site-packages\tensorflow\python\framework\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequential_2 (Sequential) (None, 256) 920576
_________________________________________________________________
dense_1 (Dense) (None, 256) 65792
=================================================================
Total params: 986,368
Trainable params: 986,368
Non-trainable params: 0
_________________________________________________________________
This code should work in the functional model:
in = Input(shape=(1, 129))
lstm_1 = LSTM(256, return_sequences=True)(in)
lstm_2 = LSTM(256)(lstm_1)
out = Dense(256)(lstm_2)
model = Model(in, out)
Related
I am trying to train a 1D-CNN on tabular data to then use LRP on it (as it has been done here). I am stuck at the model.fit() part of the implementation. Up to that point everything seems to work.
Here is the error I get:
model_history = model.fit(X_train_smote, y_train_smote, batch_size=100, epochs=100, validation_split = 0.2)
Epoch 1/100
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Input In [60], in <cell line: 1>()
----> 1 model_history = model.fit(X_train_smote, y_train_smote, batch_size=100, epochs=100, validation_split = 0.2)
File D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\utils\traceback_utils.py:67, in filter_traceback.<locals>.error_handler(*args, **kwargs)
65 except Exception as e: # pylint: disable=broad-except
66 filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67 raise e.with_traceback(filtered_tb) from None
68 finally:
69 del filtered_tb
File ~\AppData\Local\Temp\__autograph_generated_filelswqetod.py:15, in outer_factory.<locals>.inner_factory.<locals>.tf__train_function(iterator)
13 try:
14 do_return = True
---> 15 retval_ = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope)
16 except:
17 do_return = False
ValueError: in user code:
File "D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\engine\training.py", line 1051, in train_function *
return step_function(self, iterator)
File "D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\engine\training.py", line 1040, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\engine\training.py", line 1030, in run_step **
outputs = model.train_step(data)
File "D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\engine\training.py", line 890, in train_step
loss = self.compute_loss(x, y, y_pred, sample_weight)
File "D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\engine\training.py", line 948, in compute_loss
return self.compiled_loss(
File "D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\engine\compile_utils.py", line 201, in __call__
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
File "D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\losses.py", line 139, in __call__
losses = call_fn(y_true, y_pred)
File "D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\losses.py", line 243, in call **
return ag_fn(y_true, y_pred, **self._fn_kwargs)
File "D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\losses.py", line 1787, in categorical_crossentropy
return backend.categorical_crossentropy(
File "D:\Programme\Anaconda\envs\LRP_innvestigate\lib\site-packages\keras\backend.py", line 5119, in categorical_crossentropy
target.shape.assert_is_compatible_with(output.shape)
ValueError: Shapes (None,) and (None, 6, 2) are incompatible
Yet, for the sake of clarity here is the rest of my current implementation (adapted from a Kaggle notebook). The model structure is based on the feedback of my previous question here on SO.
import matplotlib.pyplot as plt
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib.colors import ListedColormap
from sklearn.metrics import plot_confusion_matrix
from scipy.stats import norm, boxcox
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
from collections import Counter
from scipy import stats
import tensorflow as tf
import warnings
warnings.simplefilter(action='ignore', category=Warning)
dataset = pd.read_csv('F:/Programmieren/this_one/data/Churn_Modelling.csv')
# split into variables and target
X = dataset.iloc[:, 3:-1].values
y = dataset.iloc[:, -1].values
# here gender is encoded
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
X[:, 2] = le.fit_transform(X[:, 2])
# one hot encoding the country (as explained in Ullah et al.)
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [1])], remainder='passthrough')
X = np.array(ct.fit_transform(X))
# split the data
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size= .2)
# upsample minority class with SMOTE
import imblearn
from imblearn.over_sampling import SMOTENC
#get original class distribution
counter = Counter(y)
print(counter)
#SMOTENC is used instead of SMOTE because there are multiple categorical features in the dataset
oversample = SMOTENC(categorical_features=[0, 1, 2, 4, 9, 10])
X_train_smote, y_train_smote = oversample.fit_resample(X_train, y_train)
#get new class distribution
counter = Counter(y_train_smote)
print(counter)
# normalize values to range 0-1 as explained in Ullah et al.
from sklearn.preprocessing import MinMaxScaler
mms = MinMaxScaler()
X_train_smote = mms.fit_transform(X_train_smote)
X_test = mms.transform(X_test)
#record-wise normalization for relative value comparison as stated by one of the authors I was in contact with
from sklearn.preprocessing import normalize
X_train_smote = normalize(X_train_smote, axis=1, norm='l1')
X_test = normalize(X_test, axis=1, norm='l1')
#reshape data for CNN
sample_size = X_train_smote.shape[0] # number of samples in train set
time_steps = X_train_smote.shape[1] # number of features in train set
input_dimension = 1
train_data_reshaped = X_train_smote.reshape(sample_size,time_steps,input_dimension)
print("After reshape train data set shape:\n", train_data_reshaped.shape)
print("1 Sample shape:\n",train_data_reshaped[0].shape)
print("An example sample:\n", train_data_reshaped[0])
#reshape test data as well
test_data_reshaped = X_test.reshape(X_test.shape[0],X_test.shape[1],1)
import keras
from keras.models import Sequential
from keras.layers import Dense, Conv1D
#create model as explained in the paper
model = Sequential()
model.add(Conv1D(filters=25, kernel_size=3, activation='relu', input_shape=(12,1)))
model.add(Conv1D(50, 3))
model.add(Conv1D(100, 3))
model.add(Dense(2200, activation='relu'))
model.add(Dense(2, activation='relu'))
model.add(Dense(2, activation='softmax'))
model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])
model.summary()
#output of model.summary()
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 10, 25) 100
conv1d_1 (Conv1D) (None, 8, 50) 3800
conv1d_2 (Conv1D) (None, 6, 100) 15100
dense (Dense) (None, 6, 2200) 222200
dense_1 (Dense) (None, 6, 2) 4402
dense_2 (Dense) (None, 6, 2) 6
=================================================================
Total params: 245,608
Trainable params: 245,608
Non-trainable params: 0
_________________________________________________________________
Are there any major flaws in my current approach that I am not aware of?
You should add a Flatten() layer before the output layer as the output shape is (6,2) and not (2,).
#Add Flatten layer
model.add(Flatten())
model.add(Dense(2, activation='softmax'))
Also kindly change the loss to sparse_categorical_crossentopy because you are using integers(0,1) as the labels.
#change loss to sparse_categorical_crossentropy
model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])
Please refer to the gist for working code. Thank you!
I upgraded to Tensorflow 2.0 and there is no tf.summary.FileWriter("tf_graphs", sess.graph). I was looking through some other StackOverflow questions on this and they said to use tf.compat.v1.summary etc. Surely there must be a way to graph and visualize a tf.keras model in Tensorflow version 2. What is it? I'm looking for a tensorboard output like the one below. Thank you!
You can visualize the graph of any tf.function decorated function, but first, you have to trace its execution.
Visualizing the graph of a Keras model means to visualize it's call method.
By default, this method is not tf.function decorated and therefore you have to wrap the model call in a function correctly decorated and execute it.
import tensorflow as tf
model = tf.keras.Sequential(
[
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(32, activation="relu"),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation="softmax"),
]
)
#tf.function
def traceme(x):
return model(x)
logdir = "log"
writer = tf.summary.create_file_writer(logdir)
tf.summary.trace_on(graph=True, profiler=True)
# Forward pass
traceme(tf.zeros((1, 28, 28, 1)))
with writer.as_default():
tf.summary.trace_export(name="model_trace", step=0, profiler_outdir=logdir)
According to the docs, you can use Tensorboard to visualise graphs once your model has been trained.
First, define your model and run it. Then, open Tensorboard and switch to the Graph tab.
Minimal Compilable Example
This example is taken from the docs. First, define your model and data.
# Relevant imports.
%load_ext tensorboard
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from datetime import datetime
from packaging import version
import tensorflow as tf
from tensorflow import keras
# Define the model.
model = keras.models.Sequential([
keras.layers.Flatten(input_shape=(28, 28)),
keras.layers.Dense(32, activation='relu'),
keras.layers.Dropout(0.2),
keras.layers.Dense(10, activation='softmax')
])
model.compile(
optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
(train_images, train_labels), _ = keras.datasets.fashion_mnist.load_data()
train_images = train_images / 255.0
Next, train your model. Here, you will need to define a callback for Tensorboard to use for visualising stats and graphs.
# Define the Keras TensorBoard callback.
logdir="logs/fit/" + datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir)
# Train the model.
model.fit(
train_images,
train_labels,
batch_size=64,
epochs=5,
callbacks=[tensorboard_callback])
After training, in your notebook, run
%tensorboard --logdir logs
And switch to the Graph tab in the navbar:
You will see a graph that looks a lot like this:
Here is the solution for tf2.x with Graph visualization of subclassed model/layer
import tensorflow as tf
print("TensorFlow version:", tf.__version__)
from tensorflow.keras.layers import Dense, Flatten, Conv2D
from tensorflow.keras import Model,Input
class MyModel(Model):
def __init__(self, dim):
super(MyModel, self).__init__()
self.conv1 = Conv2D(16, 3, activation='relu')
self.conv2 = Conv2D(32, 3, activation='relu')
self.conv3 = Conv2D(8, 3, activation='relu')
self.flatten = Flatten()
self.d1 = Dense(128, activation='relu')
self.d2 = Dense(1)
def call(self, x):
x = self.conv1(x)
x = self.conv2(x)
x = self.conv3(x)
x = self.flatten(x)
x = self.d1(x)
return self.d2(x)
def build_graph(self):
x = Input(shape=(dim))
return Model(inputs=[x], outputs=self.call(x))
dim = (28, 28, 1)
# Create an instance of the model
model = MyModel((dim))
model.build((None, *dim))
model.build_graph().summary()
tf.keras.utils.plot_model(model.build_graph(), to_file="model.png",
expand_nested=True, show_shapes=True)
the output is
TensorFlow version: 2.5.0
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_1 (InputLayer) [(None, 28, 28, 1)] 0
_________________________________________________________________
conv2d (Conv2D) (None, 26, 26, 16) 160
_________________________________________________________________
conv2d_1 (Conv2D) (None, 24, 24, 32) 4640
_________________________________________________________________
conv2d_2 (Conv2D) (None, 22, 22, 8) 2312
_________________________________________________________________
flatten (Flatten) (None, 3872) 0
_________________________________________________________________
dense (Dense) (None, 128) 495744
_________________________________________________________________
dense_1 (Dense) (None, 1) 129
=================================================================
Total params: 502,985
Trainable params: 502,985
Non-trainable params: 0
Here is also a graph visualization
Here's what is working for me at the moment (TF 2.0.0), based on the tf.keras.callbacks.TensorBoard code:
# After model has been compiled
from tensorflow.python.ops import summary_ops_v2
from tensorflow.python.keras.backend import get_graph
tb_path = '/tmp/tensorboard/'
tb_writer = tf.summary.create_file_writer(tb_path)
with tb_writer.as_default():
if not model.run_eagerly:
summary_ops_v2.graph(get_graph(), step=0)
Another option is to use this website: https://lutzroeder.github.io/netron/
which generate a graph with a .h5 or .tflite file.
The github repo it's based on may be found here:
https://github.com/lutzroeder/netron
I have a list of matrices called charMatrixList, of length 40744. I convert this list to numpy array, and the shape changes to (40744,32,30). This numpy array is passed as an input to the neural network.
The errors I'm getting are related to the shape of the Conv2D layer output, when passed as an input into an LSTM layer.
from keras.models import Sequential
from keras.layers import Embedding,LSTM,Flatten,Conv2D,Reshape
import numpy as np
def phase22(charMatrixList ):
model = Sequential()
model.add(Conv2D(32, (3, 3), strides=(1,1) , padding="same", activation="relu",input_shape=(40744,32,30)))
model.add(LSTM(16, return_sequences=True))
model.add(LSTM(16, return_sequences=True))
model.add(Flatten())
model.compile('rmsprop', 'mse')
input_array = charMatrixList
model.compile('rmsprop', 'mse')
output_array = model.predict(input_array)
return output_array
p2out = phase22(charMatrixList)
I'm getting the below error :
Traceback (most recent call last):
File "<ipython-input-56-f615f91b6704>", line 1, in <module>
p2out = phase22(np.array(charMatrixList) )
File "<ipython-input-55-9a4fd292a04f>", line 4, in phase22
model.add(LSTM(16, return_sequences=True))
File "C:\Users\Kishore\Anaconda3\lib\site-packages\keras\engine\sequential.py", line 185, in add
output_tensor = layer(self.outputs[0])
File "C:\Users\Kishore\Anaconda3\lib\site-packages\keras\layers\recurrent.py", line 500, in __call__
return super(RNN, self).__call__(inputs, **kwargs)
File "C:\Users\Kishore\Anaconda3\lib\site-packages\keras\engine\base_layer.py", line 414, in __call__
self.assert_input_compatibility(inputs)
File "C:\Users\Kishore\Anaconda3\lib\site-packages\keras\engine\base_layer.py", line 311, in assert_input_compatibility
str(K.ndim(x)))
ValueError: Input 0 is incompatible with layer lstm_11: expected ndim=3, found ndim=4
Keras ignores the first dimension when defining input size because that is just the number of training examples, m. Keras is able to work with any m, so it only cares about the actual input dimensions. That is why Kears sees (40744,32,30) as 4 dimensions.
I'm confused by the dimensions of your input, is 40744 the number of training examples? If it is do input_size = (32, 30).
If your input has 3 dimensions include number of training examples in your input, ie. charMatrixList = (m, 40744,32,30)
I have a problem with renaming layers. Below is the simplest example illustrating the problem:
from keras.layers import Dense, Conv2D, Flatten, Input
from keras.models import Model
inputs = Input(shape=(784,))
x = Dense(64, activation='relu')(inputs)
x = Dense(64, activation='relu')(x)
predictions = Dense(10, activation='softmax')(x)
# This creates a model that includes
# the Input layer and three Dense layers
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model.summary())
for i, layer in enumerate(model.layers):
layer.name = 'layer' + str(i)
print(model.summary())
model.save('temp')
It fails with the message:
Traceback (most recent call last):
File "scripts/save_load.py", line 24, in <module>
model.save('temp')
File "/lib/python3.6/site-packages/keras/engine/topology.py", line 2416, in save
save_model(self, filepath, overwrite)
File "/lib/python3.6/site-packages/keras/models.py", line 101, in save_model
'config': model.get_config()
File "/lib/python3.6/site-packages/keras/engine/topology.py", line 2281, in get_config
new_node_index = node_conversion_map[node_key]
KeyError: 'layer0_ib-0'
What am I doing wrong?
I know I can pass names to the layer constructor, it seems not to fail in this case, but is there any chance to improve my solution?
I'm using KERAS (2.0.8) on python3.5 on Windows 64bit PC.
I'm trying to utilize vgg16 with my own full-connect layers.
But, my model.fit_generator throws TypeError with strange keyword
TypeError: run() got an unexpected keyword argument 'metirics'
Of course, I never use 'metirics' in my code...
Here is my code.
def go_vgg():
train_datagen = ImageDataGenerator(rescale=1.0 /255)
validation_datagen=ImageDataGenerator()
train_generator = train_datagen.flow_from_directory(train_data_dir,
target_size=(image_size,image_size),
batch_size=batch_size,
save_to_dir=check_dir,
save_format='png',
shuffle = False,
class_mode = 'binary')
validation_generator = validation_datagen.flow_from_directory(validation_data_dir,
target_size=(image_size,image_size),
batch_size=batch_size,
save_to_dir=check_dir,
save_format='png',
shuffle=False,
class_mode = 'binary')
bmodel = VGG16(include_top=False, weights='imagenet', input_tensor=None, input_shape=(image_size,image_size,3))
bmodel.summary()
smodel = Sequential()
smodel.add(Flatten(input_shape=(7,7,512)) )
smodel.summary()
smodel.add(Dense(256, activation='relu'))
smodel.add(Dropout(0.5))
smodel.add(Dense(1, activation = 'sigmoid'))
model = Model(input=bmodel.input, outputs = smodel(bmodel.output))
for layer in model.layers[:15]:
layer.trainable = False
model.compile(loss='binary_crossentropy',
optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
metirics = ['accuracy'])
history =model.fit_generator(train_generator, steps_per_epoch=2000,
epochs = np_epoch)
The summary result is as follows (I mostely omitted bmodel.summary())
____________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_1 (InputLayer) (None, 224, 224, 3) 0
.
.
.
Epoch 1/2
Traceback (most recent call last):
File "test4.py", line 94, in <module>
main()
File "test4.py", line 86, in main
go_vgg()
File "test4.py", line 80, in go_vgg
epochs = np_epoch)
File "D:\python\lib\site-packages\keras\legacy\interfaces.py", line 87, in wra
pper
return func(*args, **kwargs)
File "D:\python\lib\site-packages\keras\engine\training.py", line 2042, in fit
_generator
class_weight=class_weight)
File "D:\python\lib\site-packages\keras\engine\training.py", line 1762, in tra
in_on_batch
outputs = self.train_function(ins)
File "D:\python\lib\site-packages\keras\backend\tensorflow_backend.py", line 2
273, in __call__
**self.session_kwargs)
TypeError: run() got an unexpected keyword argument 'metirics'
I have no idea where this 'metirics' comes from.
Any help to solve this error would be highly appreciated!
Of course you are using such a keyword, right here:
model.compile(loss='binary_crossentropy',
optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
metirics = ['accuracy'])
The problem is that you misspelled metrics as metirics. Just correct it to metrics:
model.compile(loss='binary_crossentropy',
optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
metrics = ['accuracy'])