Input to reshape is a tensor with 'batch_size' values, but the requested shape requires a multiple of 'n_features' - python-3.x

I'm trying to make own attention model and I found example code in here:
https://www.kaggle.com/takuok/bidirectional-lstm-and-attention-lb-0-043
and it works just fine when I run it without modification.
But my own data contain only numeric values, I had to change example code.
so I erase embedding part in example code and plus, this is what I fixed.
xtr = np.reshape(xtr, (xtr.shape[0], 1, xtr.shape[1]))
# xtr.shape() = (n_sample_train, 1, 150), y.shape() = (n_sample_train, 6)
xte = np.reshape(xte, (xte.shape[0], 1, xte.shape[1]))
# xtr.shape() = (n_sample_test, 1, 150)
model = BidLstm(maxlen, max_features)
model.compile(loss='binary_crossentropy', optimizer='adam',
metrics=['accuracy'])
and my BidLstm func looks like,
def BidLstm(maxlen, max_features):
inp = Input(shape=(1,150))
#x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp) -> I don't need embedding since my own data is numeric.
x = Bidirectional(LSTM(300, return_sequences=True, dropout=0.25,
recurrent_dropout=0.25))(inp)
x = Attention(maxlen)(x)
x = Dense(256, activation="relu")(x)
x = Dropout(0.25)(x)
x = Dense(6, activation="sigmoid")(x)
model = Model(inputs=inp, outputs=x)
return model
and it said,
InvalidArgumentErrorTraceback (most recent call last)
<ipython-input-62-929955370368> in <module>
29
30 early = EarlyStopping(monitor="val_loss", mode="min", patience=1)
---> 31 model.fit(xtr, y, batch_size=128, epochs=15, validation_split=0.1, callbacks=[early])
32 #model.fit(xtr, y, batch_size=256, epochs=1, validation_split=0.1)
33
/usr/local/lib/python3.5/dist-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1037 initial_epoch=initial_epoch,
1038 steps_per_epoch=steps_per_epoch,
-> 1039 validation_steps=validation_steps)
1040
1041 def evaluate(self, x=None, y=None,
/usr/local/lib/python3.5/dist-packages/keras/engine/training_arrays.py in fit_loop(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
197 ins_batch[i] = ins_batch[i].toarray()
198
--> 199 outs = f(ins_batch)
200 outs = to_list(outs)
201 for l, o in zip(out_labels, outs):
/usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs)
2713 return self._legacy_call(inputs)
2714
-> 2715 return self._call(inputs)
2716 else:
2717 if py_any(is_tensor(x) for x in inputs):
/usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py in _call(self, inputs)
2673 fetched = self._callable_fn(*array_vals, run_metadata=self.run_metadata)
2674 else:
-> 2675 fetched = self._callable_fn(*array_vals)
2676 return fetched[:len(self.outputs)]
2677
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in __call__(self, *args, **kwargs)
1437 ret = tf_session.TF_SessionRunCallable(
1438 self._session._session, self._handle, args, status,
-> 1439 run_metadata_ptr)
1440 if run_metadata:
1441 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/errors_impl.py in __exit__(self, type_arg, value_arg, traceback_arg)
526 None, None,
527 compat.as_text(c_api.TF_Message(self.status.status)),
--> 528 c_api.TF_GetCode(self.status.status))
529 # Delete the underlying status object from memory otherwise it stays alive
530 # as there is a reference to status from this from the traceback due to
InvalidArgumentError: Input to reshape is a tensor with 128 values, but the requested shape requires a multiple of 150
[[{{node attention_16/Reshape_2}}]]
[[{{node loss_5/mul}}]]
I think something wrong in loss function saids in here:
Input to reshape is a tensor with 2 * "batch_size" values, but the requested shape has "batch_size"
but I don't know which part to fix it.
my keras and tensorflow versions are 2.2.4 and 1.13.0-rc0
please help. thanks.
Edit 1
I've change my batch size, like keras saids, multiple of 150(batch_size = 150). than it reports
Train on 143613 samples, validate on 15958 samples
Epoch 1/15
143400/143613 [============================>.] - ETA: 0s - loss: 0.1505 - acc: 0.9619
InvalidArgumentError: Input to reshape is a tensor with 63 values, but the requested shape requires a multiple of 150
[[{{node attention_18/Reshape_2}}]]
[[{{node metrics_6/acc/Mean_1}}]]
and details is same as before. what should I do?

Your input shape must be (150,1).
LSTM shapes are (batch, steps, features). It's pointless to use LSTMs with 1 step only. (Unless you are using custom training loops with stateful=True, which is not your case).

Related

TypeError: update() got an unexpected keyword argument 'force'

I use word2vec and biLstm to implement sentiment analysis for movie reviews. When I train my model on Jupyter notebook, I always get the TypeError: update() got an unexpected keyword argument 'force' at the last batch of the first epoch.
here is my code:
batch_size = 50
result = model.fit(
X_train,
Y_train,
validation_data=(X_test, Y_test),
batch_size=batch_size,
epochs=5
)
and the error:
Train on 1600 samples, validate on 400 samples
Epoch 1/5
1550/1600 [============================>.] - ETA: 2s - loss: 0.7257 - acc: 0.4961
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-360-8eaf5ac2ad33> in <module>()
17 validation_data=(X_test, Y_test),
18 batch_size=batch_size,
---> 19 epochs=5
20 )
~/opt/anaconda3/envs/WDPS/lib/python3.6/site-packages/keras/models.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
958 initial_epoch=initial_epoch,
959 steps_per_epoch=steps_per_epoch,
--> 960 validation_steps=validation_steps)
961
962 def evaluate(self, x, y, batch_size=32, verbose=1,
~/opt/anaconda3/envs/WDPS/lib/python3.6/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1648 initial_epoch=initial_epoch,
1649 steps_per_epoch=steps_per_epoch,
-> 1650 validation_steps=validation_steps)
1651
1652 def evaluate(self, x=None, y=None,
~/opt/anaconda3/envs/WDPS/lib/python3.6/site-packages/keras/engine/training.py in _fit_loop(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
1231 for l, o in zip(out_labels, val_outs):
1232 epoch_logs['val_' + l] = o
-> 1233 callbacks.on_epoch_end(epoch, epoch_logs)
1234 if callback_model.stop_training:
1235 break
~/opt/anaconda3/envs/WDPS/lib/python3.6/site-packages/keras/callbacks.py in on_epoch_end(self, epoch, logs)
71 logs = logs or {}
72 for callback in self.callbacks:
---> 73 callback.on_epoch_end(epoch, logs)
74
75 def on_batch_begin(self, batch, logs=None):
~/opt/anaconda3/envs/WDPS/lib/python3.6/site-packages/keras/callbacks.py in on_epoch_end(self, epoch, logs)
304 self.log_values.append((k, logs[k]))
305 if self.verbose:
--> 306 self.progbar.update(self.seen, self.log_values, force=True)
307
308
TypeError: update() got an unexpected keyword argument 'force'
At first, I have a line of code verbose = 1 under the epochs=5. There will be the same error and the arrow point to verbose = 1. Then I change it to verbose=2 or just delete verbose. But I still have the problem.
I tried to change the batch size and the amount of training set. but it still didn't work out.
It always showed at the last batch.
python version= 3.6.2
keras version = 2.1.1

Incompatible shape problem with Keras ( segmentation model) for batch_size>1

I am trying to do semantic segmentation using Unet from segmentation model for multi channel (>3) image.
The code works if the batch_size =1. But if I change the batch_size to other values (e.g. 2) then error occurs (InvalidArgumentError: Incompatible shapes):
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
<ipython-input-19-15dc3666afa8> in <module>
22 validation_steps = 1,
23 callbacks=build_callbacks(),
---> 24 verbose = 1)
25
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name +
90 '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1424 use_multiprocessing=use_multiprocessing,
1425 shuffle=shuffle,
-> 1426 initial_epoch=initial_epoch)
1427
1428 #interfaces.legacy_generator_methods_support
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
189 outs = model.train_on_batch(x, y,
190 sample_weight=sample_weight,
--> 191 class_weight=class_weight)
192
193 if not isinstance(outs, list):
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight)
1218 ins = x + y + sample_weights
1219 self._make_train_function()
-> 1220 outputs = self.train_function(ins)
1221 if len(outputs) == 1:
1222 return outputs[0]
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs)
2659 return self._legacy_call(inputs)
2660
-> 2661 return self._call(inputs)
2662 else:
2663 if py_any(is_tensor(x) for x in inputs):
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in _call(self, inputs)
2629 symbol_vals,
2630 session)
-> 2631 fetched = self._callable_fn(*array_vals)
2632 return fetched[:len(self.outputs)]
2633
~/.virtualenvs/sm/lib/python3.6/site-packages/tensorflow_core/python/client/session.py in __call__(self, *args, **kwargs)
1470 ret = tf_session.TF_SessionRunCallable(self._session._session,
1471 self._handle, args,
-> 1472 run_metadata_ptr)
1473 if run_metadata:
1474 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
InvalidArgumentError: Incompatible shapes: [2,256,256,1] vs. [2,256,256]
[[{{node loss_1/model_4_loss/mul}}]]
I tried to play around by following different posts in the forum but could not solve it. Here, is a portion of code which runs for batch_size=1.
batch_size = 1 # CHANGING ‘batch_size ‘ value other than 1 gives error
train_image_files = glob(patch_img + "/**/*.tif")
# simple_image_generator() is used to work with multi channel (>3) images (the function is
at the end)
train_image_generator = simple_image_generator(train_image_files,
batch_size=batch_size,
rotation_range=45,
horizontal_flip=True,
vertical_flip=True)
train_mask_files = glob(patch_ann + "/**/*.tif")
train_mask_generator = simple_image_generator(train_mask_files,
batch_size=batch_size)
test_image_files = glob(test_img + "/**/*.tif")
test_image_generator = simple_image_generator(test_image_files,
batch_size=batch_size,
rotation_range=45,
horizontal_flip=True,
vertical_flip=True)
test_mask_files = glob(test_ann + "/**/*.tif")
test_mask_generator = simple_image_generator(test_mask_files,
batch_size=batch_size)
train_generator = (pair for pair in zip(train_image_generator, train_mask_generator))
test_generator = (pair for pair in zip(test_image_generator, test_mask_generator))
.
.
num_channels = 8 # no. of channel
base_model = sm.Unet(backbone_name='resnet34', encoder_weights='imagenet')
inp = Input(shape=( None, None, num_channels))
layer_1 = Conv2D( 3, (1, 1))(inp) # map N channels data to 3 channels
out = base_model(layer_1)
model = Model(inp, out, name=base_model.name)
model.summary()
model.compile(
optimizer = keras.optimizers.Adam(lr=learning_rate),
loss = sm.losses.bce_jaccard_loss,
metrics = ['accuracy',sm.metrics.iou_score]
)
model_history = model.fit_generator(train_generator,
epochs = 1,
steps_per_epoch = 1,
validation_data = test_generator,
validation_steps = 1,
callbacks = build_callbacks(),
verbose = 1)
Additional Information:
I am not using the default imageGenerator provided by keras. I am using ‘simple_image_generator’ (slightly modified)
def simple_image_generator(files, batch_size=32,
rotation_range=0, horizontal_flip=False,
vertical_flip=False):
while True:
# select batch_size number of samples without replacement
batch_files = sample(files, batch_size)
# array for images
batch_X = []
# loop over images of the current batch
for idx, input_path in enumerate(batch_files):
image = np.array(imread(input_path), dtype=float)
# process image
if horizontal_flip:
# randomly flip image up/down
if choice([True, False]):
image = np.flipud(image)
if vertical_flip:
# randomly flip image left/right
if choice([True, False]):
image = np.fliplr(image)
# rotate image by random angle between
# -rotation_range <= angle < rotation_range
if rotation_range is not 0:
angle = np.random.uniform(low=-abs(rotation_range),
high=abs(rotation_range))
image = rotate(image, angle, mode='reflect',
order=1, preserve_range=True)
# put all together
batch_X += [image]
# convert lists to np.array
X = np.array(batch_X)
yield(X)
This error was solved by redefining a new image generator instead of simple_image_generator(). The simple_image_generator() worked well with the shape of the images (8 Bands) but did not cope well with the shape of the mask (1 band ).
During the execution, image_generator had 4 dimensions with [2,256,256,1] ( i.e. batch_size, (image size), bands) BUT mask_generator had 3 dimensions only vs. [2,256,256] (i.e. batch_size,(image size))
So reshaping the mask of [2,256,256] to [2,256,256, 1] solved the issue.

Tensorflow - Value Error in model.fit - How to fix

I am trying to train a Deep Neural Network using MNIST data set.
BATCH_SIZE = 100
train_data = train_data.batch(BATCH_SIZE)
validation_data = validation_data.batch(num_validation_samples)
test_data = scaled_test_data.batch(num_test_samples)
validation_inputs, validation_targets = next(iter(validation_data))
input_size = 784
output_size = 10
hidden_layer_size = 50
model = tf.keras.Sequential([
tf.keras.layers.Flatten(input_shape=(28,28,1)),
tf.keras.layers.Dense(hidden_layer_size, activation='relu'),
tf.keras.layers.Dense(hidden_layer_size, activation='relu'),
tf.keras.layers.Dense(output_size, activation='softmax')
])
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
NUM_EPOCHS = 5
model.fit(train_data, epochs=NUM_EPOCHS, validation_data=(validation_inputs,validation_targets))
The model.fit is throwing the following error
-------------------------------------------------------------------------
--
ValueError Traceback (most recent call last)
<ipython-input-58-c083185dafc6> in <module>
1 NUM_EPOCHS = 5
----> 2 model.fit(train_data, epochs=NUM_EPOCHS, validation_data=(validation_inputs,validation_targets))
~/anaconda3/envs/py3-TF2/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)
726 max_queue_size=max_queue_size,
727 workers=workers,
--> 728 use_multiprocessing=use_multiprocessing)
729
730 def evaluate(self,
~/anaconda3/envs/py3-TF2/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py in fit(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, **kwargs)
222 validation_data=validation_data,
223 validation_steps=validation_steps,
--> 224 distribution_strategy=strategy)
225
226 total_samples = _get_total_number_of_samples(training_data_adapter)
~/anaconda3/envs/py3-TF2/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py in _process_training_inputs(model, x, y, batch_size, epochs, sample_weights, class_weights, steps_per_epoch, validation_split, validation_data, validation_steps, shuffle, distribution_strategy, max_queue_size, workers, use_multiprocessing)
562 class_weights=class_weights,
563 steps=validation_steps,
--> 564 distribution_strategy=distribution_strategy)
565 elif validation_steps:
566 raise ValueError('`validation_steps` should not be specified if '
~/anaconda3/envs/py3-TF2/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py in _process_inputs(model, x, y, batch_size, epochs, sample_weights, class_weights, shuffle, steps, distribution_strategy, max_queue_size, workers, use_multiprocessing)
604 max_queue_size=max_queue_size,
605 workers=workers,
--> 606 use_multiprocessing=use_multiprocessing)
607 # As a fallback for the data type that does not work with
608 # _standardize_user_data, use the _prepare_model_with_inputs.
~/anaconda3/envs/py3-TF2/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/data_adapter.py in __init__(self, x, y, sample_weights, batch_size, epochs, steps, shuffle, **kwargs)
252 if not batch_size:
253 raise ValueError(
--> 254 "`batch_size` or `steps` is required for `Tensor` or `NumPy`"
255 " input data.")
256
ValueError: `batch_size` or `steps` is required for `Tensor` or `NumPy` input data.
The training and validation data are obtained from MNIST dataset. Some part of the data are taken as training data and some as testing data.
What am I doing wrong here?
Update
As per Dominques suggestion, I have changed model.fit to
model.fit(train_data, batch_size=128, epochs=NUM_EPOCHS, validation_data=(validation_inputs,validation_targets))
But now, I get the following error
ValueError: The `batch_size` argument must not be specified for the given input type. Received input: <BatchDataset shapes: ((None, 28, 28, 1), (None,)), types: (tf.float32, tf.int64)>, batch_size: 128
The tf doc will give you more clues why you get the error.
https://www.tensorflow.org/api_docs/python/tf/keras/Model#fit
validation_data: Data on which to evaluate the loss and any model metrics at the end of each epoch. The model will not be trained on this data. validation_data will override validation_split. validation_data could be:
• tuple (x_val, y_val) of Numpy arrays or tensors
• tuple (x_val, y_val, val_sample_weights) of Numpy arrays
• dataset
For the first two cases, batch_size must be provided. For the last case, validation_steps must be provided.
Since You already have the validation dataset batched, consider to use it directly and specify validation steps as below.
BATCH_SIZE = 100
train_data = train_data.batch(BATCH_SIZE)
validation_data = validation_data.batch(BATCH_SIZE)
...
model.fit(train_data, epochs=NUM_EPOCHS, validation_data=validation_data,validation_steps=1)
You need to specify the batch size, i.e. how many data points should be included in each iteration. If you look at the documentation you will see that there is no default value set.
https://www.tensorflow.org/api_docs/python/tf/keras/Sequential
you can set the value by adding batch_size to the fit command. Good values are normally numbers along the line of 2**n, as this allows for more efficient processing with multiple cores. For you this shouldn't make a strong difference though :)
model.fit(train_data,
batch_size=128
epochs=NUM_EPOCHS,
validation_data=(validation_inputs,validation_targets))
Why nobody mention i don't know but your problem is Y_train data. You don't supply it as an argument to your model..
model.fit(X_Train, y_train, batch_size=None, epochs=1, verbose=1, callbacks=None, validation_split=0.0, validation_data=None, shuffle=True, class_weight=None, sample_weight=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None, validation_freq=1, max_queue_size=10, workers=1, use_multiprocessing=False)
Instead of y_train you are giving :
model.fit(train_data, batch_size=128 ....
And getting an Error saying :
ValueError: `batch_size` or `steps` is required for `Tensor` or `NumPy` input data.
I hope it helps.
model.fit(train_data, epochs=NUM_EPOCHS, validation_data=(validation_inputs, validation_targets), verbose=2)
change to (by adding validation_steps=1) will do the trick
model.fit(train_data, epochs=NUM_EPOCHS, validation_data=(validation_inputs, validation_targets),validation_steps=1, verbose=2)
I changed the input_shape=(28,28,1) to input_shape=(28,28,3) and it worked for me.

InvalidArgumentError when I use model.fit()

I am trying to build a custom layer in keras/tensorflow. The purpose would be to extend it as a RNN. The problem appears during training, where, apparently, there is an incompatibility with the output shape and the training dataset.
X_train.shape
(100, 5)
X_target.shape
(100, 5)
This is the custom layer:
class MyLayer(Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(MyLayer, self).__init__(**kwargs)
def build(self, input_shape):
print(self.output_dim)
print(input_shape)
# Create a trainable weight variable for this layer.
self.kernel = self.add_weight(name='kernel',
shape=(self.output_dim, input_shape[1]),
initializer='uniform',
trainable=True)
super(MyLayer, self).build(input_shape) # Be sure to call this at the end
def call(self, x):
print('shape x: ')
print(x.shape)
print('shape kernel: ')
print(self.kernel.shape)
matrix = tf.transpose(self.kernel)
print('matrix')
print(matrix.shape)
prod = K.dot(x, matrix)
print('after product')
print(prod.shape)
return prod
def compute_output_shape(self, input_shape):
print('Compute output shape')
print(input_shape)
print(self.output_dim)
return (input_shape[0], self.output_dim)
model = Sequential()
model.add(MyLayer(5, batch_input_shape=(100, 5)))
model.compile(optimizer='adam', loss='mse')
# fit model
model.fit(X_train, X_target)
tf.keras.__version__ = '2.1.6-tf'
Since I am passing a x_training with a shape of (100,5), I would expect that, by multiplying it by a matrix of (5,5), I would obtain a matrix of (100, 5), which is the same shape as the target and thus, I would be able to train, in this case, a 5x5 weight matrix. Instead, I get this:
InvalidArgumentError Traceback (most recent call last)
<ipython-input-77-4dee23ead957> in <module>()
6 model.compile(optimizer='adam', loss='mse')
7 # fit model
----> 8 model.fit(X_train, X_target)#, epochs=300, verbose=0)
~/anaconda3/envs/ldsa/lib/python3.5/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1037 initial_epoch=initial_epoch,
1038 steps_per_epoch=steps_per_epoch,
-> 1039 validation_steps=validation_steps)
1040
1041 def evaluate(self, x=None, y=None,
~/anaconda3/envs/ldsa/lib/python3.5/site-packages/keras/engine/training_arrays.py in fit_loop(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
197 ins_batch[i] = ins_batch[i].toarray()
198
--> 199 outs = f(ins_batch)
200 outs = to_list(outs)
201 for l, o in zip(out_labels, outs):
~/anaconda3/envs/ldsa/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs)
2713 return self._legacy_call(inputs)
2714
-> 2715 return self._call(inputs)
2716 else:
2717 if py_any(is_tensor(x) for x in inputs):
~/anaconda3/envs/ldsa/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py in _call(self, inputs)
2673 fetched = self._callable_fn(*array_vals, run_metadata=self.run_metadata)
2674 else:
-> 2675 fetched = self._callable_fn(*array_vals)
2676 return fetched[:len(self.outputs)]
2677
~/anaconda3/envs/ldsa/lib/python3.5/site-packages/tensorflow/python/client/session.py in __call__(self, *args, **kwargs)
1380 ret = tf_session.TF_SessionRunCallable(
1381 self._session._session, self._handle, args, status,
-> 1382 run_metadata_ptr)
1383 if run_metadata:
1384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
~/anaconda3/envs/ldsa/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py in __exit__(self, type_arg, value_arg, traceback_arg)
517 None, None,
518 compat.as_text(c_api.TF_Message(self.status.status)),
--> 519 c_api.TF_GetCode(self.status.status))
520 # Delete the underlying status object from memory otherwise it stays alive
521 # as there is a reference to status from this from the traceback due to
InvalidArgumentError: Incompatible shapes: [100,5] vs. [32,5]
[[Node: training_12/Adam/gradients/loss_17/my_layer_19_loss/sub_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _class=["loc:#training_12/Adam/gradients/loss_17/my_layer_19_loss/sub_grad/Reshape"], _device="/job:localhost/replica:0/task:0/device:CPU:0"](training_12/Adam/gradients/loss_17/my_layer_19_loss/sub_grad/Shape, training_12/Adam/gradients/loss_17/my_layer_19_loss/sub_grad/Shape_1)]]
My surprise comes from this: Incompatible shapes: [100,5] vs. [32,5], where does this 32 come from?

Dask DataFrame for fitting Keras model

I have a Dask DataFrame that I want to use for fitting a Keras autoencoder model:
DataFrame:
import dask.dataframe as dd
input_df = dd.read_csv(file_path)
input_df.dtypes
_2 float64
_3 float64
_4 float64
_5 float64 ...
Keras model:
autoencoder = Sequential()
autoencoder.add(Dense(dense[0], input_shape=(dense[0],), activation = 'relu' ))
autoencoder.add(Dense(dense[1], activation = 'relu' ))
autoencoder.add(Dense(dense[2], activation = 'relu' ))
autoencoder.add(Dense(dense[3], activation = 'relu' ))
autoencoder.add(Dense(dense[0], activation = 'relu' ))
autoencoder.compile(loss='mse',
optimizer='adam',
metrics=['mse'])
When I pass the DataFrame for fitting:
autoencoder.fit(input_df, input_df,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_split = val_split)
I get the error:
TypeError Traceback (most recent call last)
<ipython-input-23-d0480d8a460d> in <module>()
3 epochs=epochs,
4 verbose=1,
----> 5 validation_split = val_split)
~/anaconda3/envs/py36/lib/python3.6/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
950 sample_weight=sample_weight,
951 class_weight=class_weight,
--> 952 batch_size=batch_size)
953 # Prepare validation data.
954 do_validation = False
~/anaconda3/envs/py36/lib/python3.6/site-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)
799 for (ref, sw, cw, mode) in
800 zip(y, sample_weights, class_weights,
--> 801 feed_sample_weight_modes)
802 ]
803 # Check that all arrays have the same length.
~/anaconda3/envs/py36/lib/python3.6/site-packages/keras/engine/training.py in <listcomp>(.0)
797 sample_weights = [
798 standardize_weights(ref, sw, cw, mode)
--> 799 for (ref, sw, cw, mode) in
800 zip(y, sample_weights, class_weights,
801 feed_sample_weight_modes)
~/anaconda3/envs/py36/lib/python3.6/site-packages/keras/engine/training_utils.py in standardize_weights(y, sample_weight, class_weight, sample_weight_mode)
522 else:
523 if sample_weight_mode is None:
--> 524 return np.ones((y.shape[0],), dtype=K.floatx())
525 else:
526 return np.ones((y.shape[0], y.shape[1]), dtype=K.floatx())
~/anaconda3/envs/py36/lib/python3.6/site-packages/numpy/core/numeric.py in ones(shape, dtype, order)
201
202 """
--> 203 a = empty(shape, dtype, order)
204 multiarray.copyto(a, 1, casting='unsafe')
205 return a
TypeError: 'float' object cannot be interpreted as an integer
Would appreciate some help! Thanks!

Resources