AttributeError: 'generator' object has no attribute 'ndim' - keras

I'm working on Triplets networks on Keras to find the image similarity. However, I'm getting an error when feeding triplets to the model. Request you to kindly help on this.
Basically I'm trying to feel the model with 3 inputs(anchor, positive, negative)
I'm working on Python3 and fitting the model with fit_model using Keras. This is my function to train the model:
def train(trainDB, testDB, n_iter, batch_size, evaluate_every, test_size,
loss_every):
print("Starting training process!")
print("-------------------------------------")
best = -1
t_start = time.time()
inputs=trainDB.getTripletTrainData(batch_size)
targets=np.ones([batch_size])
for i in range(0, n_iter):
loss=tripletNet.fit(inputs, targets)
#print("Loss: {0}".format(loss))
if i % evaluate_every == 0:
print("Time for {0} iterations: {1}".format(i, time.time()-t_start))
val_acc = self.test_oneshot(testDB, test_size)
if val_acc > best:
print("Current best: {0}, previous best: {1}".format(val_acc, best))
print("Saving weights to: {0} \n".format(weights_path))
self.tripletNet.save_weights(weights_path)
best=val_acc
if i % loss_every == 0:
print("iteration {}, training loss: {:.2f},".format(i,loss))
Error message:
Starting training process!
-------------------------------------
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-163-b5442e61de2d> in <module>()
----> 1 train(trainDatabase, testDatabase, n_iter, batch_size, evaluate_every, test_size, loss_every)
5 frames
<ipython-input-161-f417f0ebcfc7> in train(trainDB, testDB, n_iter, batch_size, evaluate_every, test_size, loss_every)
10
11 for i in range(0, n_iter):
---> 12 loss=tripletNet.fit(inputs, targets)
13
14 #print("Loss: {0}".format(loss))
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
950 sample_weight=sample_weight,
951 class_weight=class_weight,
--> 952 batch_size=batch_size)
953 # Prepare validation data.
954 do_validation = False
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)
749 feed_input_shapes,
750 check_batch_axis=False, # Don't enforce the batch size.
--> 751 exception_prefix='input')
752
753 if y is not None:
/usr/local/lib/python3.6/dist-packages/keras/engine/training_utils.py in standardize_input_data(data, names, shapes, check_batch_axis, exception_prefix)
90 data = data.values if data.__class__.__name__ == 'DataFrame' else data
91 data = [data]
---> 92 data = [standardize_single_array(x) for x in data]
93
94 if len(data) != len(names):
/usr/local/lib/python3.6/dist-packages/keras/engine/training_utils.py in <listcomp>(.0)
90 data = data.values if data.__class__.__name__ == 'DataFrame' else data
91 data = [data]
---> 92 data = [standardize_single_array(x) for x in data]
93
94 if len(data) != len(names):
/usr/local/lib/python3.6/dist-packages/keras/engine/training_utils.py in standardize_single_array(x)
25 'Got tensor with shape: %s' % str(shape))
26 return x
---> 27 elif x.ndim == 1:
28 x = np.expand_dims(x, 1)
29 return x
**AttributeError: 'generator' object has no attribute 'ndim'**
If i use fit_generator.. Im getting the below error..
Starting training process!
-------------------------------------
Epoch 1/1
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-194-b5442e61de2d> in <module>()
----> 1 train(trainDatabase, testDatabase, n_iter, batch_size, evaluate_every, test_size, loss_every)
3 frames
<ipython-input-193-8ad964a9916f> in train(trainDB, testDB, n_iter, batch_size, evaluate_every, test_size, loss_every)
10
11 for i in range(0, n_iter):
---> 12 loss=tripletNet.fit_generator(inputs, targets)
13
14 #print("Loss: {0}".format(loss))
/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name + '` call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1416 use_multiprocessing=use_multiprocessing,
1417 shuffle=shuffle,
-> 1418 initial_epoch=initial_epoch)
1419
1420 #interfaces.legacy_generator_methods_support
/usr/local/lib/python3.6/dist-packages/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
178 steps_done = 0
179 batch_index = 0
--> 180 while steps_done < steps_per_epoch:
181 generator_output = next(output_generator)
182
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()

Related

TypeError: update() got an unexpected keyword argument 'force'

I use word2vec and biLstm to implement sentiment analysis for movie reviews. When I train my model on Jupyter notebook, I always get the TypeError: update() got an unexpected keyword argument 'force' at the last batch of the first epoch.
here is my code:
batch_size = 50
result = model.fit(
X_train,
Y_train,
validation_data=(X_test, Y_test),
batch_size=batch_size,
epochs=5
)
and the error:
Train on 1600 samples, validate on 400 samples
Epoch 1/5
1550/1600 [============================>.] - ETA: 2s - loss: 0.7257 - acc: 0.4961
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-360-8eaf5ac2ad33> in <module>()
17 validation_data=(X_test, Y_test),
18 batch_size=batch_size,
---> 19 epochs=5
20 )
~/opt/anaconda3/envs/WDPS/lib/python3.6/site-packages/keras/models.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
958 initial_epoch=initial_epoch,
959 steps_per_epoch=steps_per_epoch,
--> 960 validation_steps=validation_steps)
961
962 def evaluate(self, x, y, batch_size=32, verbose=1,
~/opt/anaconda3/envs/WDPS/lib/python3.6/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1648 initial_epoch=initial_epoch,
1649 steps_per_epoch=steps_per_epoch,
-> 1650 validation_steps=validation_steps)
1651
1652 def evaluate(self, x=None, y=None,
~/opt/anaconda3/envs/WDPS/lib/python3.6/site-packages/keras/engine/training.py in _fit_loop(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
1231 for l, o in zip(out_labels, val_outs):
1232 epoch_logs['val_' + l] = o
-> 1233 callbacks.on_epoch_end(epoch, epoch_logs)
1234 if callback_model.stop_training:
1235 break
~/opt/anaconda3/envs/WDPS/lib/python3.6/site-packages/keras/callbacks.py in on_epoch_end(self, epoch, logs)
71 logs = logs or {}
72 for callback in self.callbacks:
---> 73 callback.on_epoch_end(epoch, logs)
74
75 def on_batch_begin(self, batch, logs=None):
~/opt/anaconda3/envs/WDPS/lib/python3.6/site-packages/keras/callbacks.py in on_epoch_end(self, epoch, logs)
304 self.log_values.append((k, logs[k]))
305 if self.verbose:
--> 306 self.progbar.update(self.seen, self.log_values, force=True)
307
308
TypeError: update() got an unexpected keyword argument 'force'
At first, I have a line of code verbose = 1 under the epochs=5. There will be the same error and the arrow point to verbose = 1. Then I change it to verbose=2 or just delete verbose. But I still have the problem.
I tried to change the batch size and the amount of training set. but it still didn't work out.
It always showed at the last batch.
python version= 3.6.2
keras version = 2.1.1

Incompatible shape problem with Keras ( segmentation model) for batch_size>1

I am trying to do semantic segmentation using Unet from segmentation model for multi channel (>3) image.
The code works if the batch_size =1. But if I change the batch_size to other values (e.g. 2) then error occurs (InvalidArgumentError: Incompatible shapes):
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
<ipython-input-19-15dc3666afa8> in <module>
22 validation_steps = 1,
23 callbacks=build_callbacks(),
---> 24 verbose = 1)
25
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name +
90 '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1424 use_multiprocessing=use_multiprocessing,
1425 shuffle=shuffle,
-> 1426 initial_epoch=initial_epoch)
1427
1428 #interfaces.legacy_generator_methods_support
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
189 outs = model.train_on_batch(x, y,
190 sample_weight=sample_weight,
--> 191 class_weight=class_weight)
192
193 if not isinstance(outs, list):
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight)
1218 ins = x + y + sample_weights
1219 self._make_train_function()
-> 1220 outputs = self.train_function(ins)
1221 if len(outputs) == 1:
1222 return outputs[0]
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs)
2659 return self._legacy_call(inputs)
2660
-> 2661 return self._call(inputs)
2662 else:
2663 if py_any(is_tensor(x) for x in inputs):
~/.virtualenvs/sm/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in _call(self, inputs)
2629 symbol_vals,
2630 session)
-> 2631 fetched = self._callable_fn(*array_vals)
2632 return fetched[:len(self.outputs)]
2633
~/.virtualenvs/sm/lib/python3.6/site-packages/tensorflow_core/python/client/session.py in __call__(self, *args, **kwargs)
1470 ret = tf_session.TF_SessionRunCallable(self._session._session,
1471 self._handle, args,
-> 1472 run_metadata_ptr)
1473 if run_metadata:
1474 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
InvalidArgumentError: Incompatible shapes: [2,256,256,1] vs. [2,256,256]
[[{{node loss_1/model_4_loss/mul}}]]
I tried to play around by following different posts in the forum but could not solve it. Here, is a portion of code which runs for batch_size=1.
batch_size = 1 # CHANGING ‘batch_size ‘ value other than 1 gives error
train_image_files = glob(patch_img + "/**/*.tif")
# simple_image_generator() is used to work with multi channel (>3) images (the function is
at the end)
train_image_generator = simple_image_generator(train_image_files,
batch_size=batch_size,
rotation_range=45,
horizontal_flip=True,
vertical_flip=True)
train_mask_files = glob(patch_ann + "/**/*.tif")
train_mask_generator = simple_image_generator(train_mask_files,
batch_size=batch_size)
test_image_files = glob(test_img + "/**/*.tif")
test_image_generator = simple_image_generator(test_image_files,
batch_size=batch_size,
rotation_range=45,
horizontal_flip=True,
vertical_flip=True)
test_mask_files = glob(test_ann + "/**/*.tif")
test_mask_generator = simple_image_generator(test_mask_files,
batch_size=batch_size)
train_generator = (pair for pair in zip(train_image_generator, train_mask_generator))
test_generator = (pair for pair in zip(test_image_generator, test_mask_generator))
.
.
num_channels = 8 # no. of channel
base_model = sm.Unet(backbone_name='resnet34', encoder_weights='imagenet')
inp = Input(shape=( None, None, num_channels))
layer_1 = Conv2D( 3, (1, 1))(inp) # map N channels data to 3 channels
out = base_model(layer_1)
model = Model(inp, out, name=base_model.name)
model.summary()
model.compile(
optimizer = keras.optimizers.Adam(lr=learning_rate),
loss = sm.losses.bce_jaccard_loss,
metrics = ['accuracy',sm.metrics.iou_score]
)
model_history = model.fit_generator(train_generator,
epochs = 1,
steps_per_epoch = 1,
validation_data = test_generator,
validation_steps = 1,
callbacks = build_callbacks(),
verbose = 1)
Additional Information:
I am not using the default imageGenerator provided by keras. I am using ‘simple_image_generator’ (slightly modified)
def simple_image_generator(files, batch_size=32,
rotation_range=0, horizontal_flip=False,
vertical_flip=False):
while True:
# select batch_size number of samples without replacement
batch_files = sample(files, batch_size)
# array for images
batch_X = []
# loop over images of the current batch
for idx, input_path in enumerate(batch_files):
image = np.array(imread(input_path), dtype=float)
# process image
if horizontal_flip:
# randomly flip image up/down
if choice([True, False]):
image = np.flipud(image)
if vertical_flip:
# randomly flip image left/right
if choice([True, False]):
image = np.fliplr(image)
# rotate image by random angle between
# -rotation_range <= angle < rotation_range
if rotation_range is not 0:
angle = np.random.uniform(low=-abs(rotation_range),
high=abs(rotation_range))
image = rotate(image, angle, mode='reflect',
order=1, preserve_range=True)
# put all together
batch_X += [image]
# convert lists to np.array
X = np.array(batch_X)
yield(X)
This error was solved by redefining a new image generator instead of simple_image_generator(). The simple_image_generator() worked well with the shape of the images (8 Bands) but did not cope well with the shape of the mask (1 band ).
During the execution, image_generator had 4 dimensions with [2,256,256,1] ( i.e. batch_size, (image size), bands) BUT mask_generator had 3 dimensions only vs. [2,256,256] (i.e. batch_size,(image size))
So reshaping the mask of [2,256,256] to [2,256,256, 1] solved the issue.

How to run Tensorflow's Keras model.fit() function on GPU in Kaggle Notebook?

I want to run my code on GPU provided by Kaggle. I am able to run my code on CPU though but unable to migrate it properly to run on Kaggle GPU I guess.
On running this
with tf.device("/device:GPU:0"):
hist = model.fit(x=X_train, y=Y_train, validation_data=(X_test, Y_test), batch_size=25, epochs=20, callbacks=callbacks_list)
and getting this error
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-28-cdb8b009cd85> in <module>
1 with tf.device("/device:GPU:0"):
----> 2 hist = model.fit(x=X_train, y=Y_train, validation_data=(X_test, Y_test), batch_size=25, epochs=20, callbacks=callbacks_list)
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)
817 self._assert_compile_was_called()
818 self._check_call_args('evaluate')
--> 819
820 func = self._select_training_loop(x)
821 return func.evaluate(
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py in fit(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)
233
234 recreate_training_iterator = (
--> 235 training_data_adapter.should_recreate_iterator(steps_per_epoch))
236 if not steps_per_epoch:
237 # TODO(b/139762795): Add step inference for when steps is None to
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py in _process_training_inputs(model, x, y, batch_size, epochs, sample_weights, class_weights, steps_per_epoch, validation_split, validation_data, validation_steps, shuffle, distribution_strategy, max_queue_size, workers, use_multiprocessing)
591 class_weights=None,
592 shuffle=False,
--> 593 steps=None,
594 distribution_strategy=None,
595 max_queue_size=10,
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py in _process_inputs(model, mode, x, y, batch_size, epochs, sample_weights, class_weights, shuffle, steps, distribution_strategy, max_queue_size, workers, use_multiprocessing)
704 """Provide a scope for running one batch."""
705 batch_logs = {'batch': step, 'size': size}
--> 706 self.callbacks._call_batch_hook(
707 mode, 'begin', step, batch_logs)
708 self.progbar.on_batch_begin(step, batch_logs)
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/data_adapter.py in __init__(self, x, y, sample_weights, sample_weight_modes, batch_size, epochs, steps, shuffle, **kwargs)
355 sample_weights = _process_numpy_inputs(sample_weights)
356
--> 357 # If sample_weights are not specified for an output use 1.0 as weights.
358 if (sample_weights is not None and
359 any([sw is None for sw in sample_weights])):
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/data_adapter.py in slice_inputs(self, indices_dataset, inputs)
381 if steps and not batch_size:
382 batch_size = int(math.ceil(num_samples/steps))
--> 383
384 if not batch_size:
385 raise ValueError(
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py in from_tensors(tensors)
564 existing iterators.
565
--> 566 Args:
567 unused_dummy: Ignored value.
568
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py in __init__(self, element)
2763 init_args: A nested structure representing the arguments to `init_func`.
2764 init_func: A TensorFlow function that will be called on `init_args` each
-> 2765 time a C++ iterator over this dataset is constructed. Returns a nested
2766 structure representing the "state" of the dataset.
2767 next_func: A TensorFlow function that will be called on the result of
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/data/util/structure.py in normalize_element(element)
111 ops.convert_to_tensor(t, name="component_%d" % i))
112 return nest.pack_sequence_as(element, normalized_components)
--> 113
114
115 def convert_legacy_structure(output_types, output_shapes, output_classes):
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py in convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, dtype_hint, ctx, accepted_result_types)
1312 return ret
1313 raise TypeError("%sCannot convert %r with type %s to Tensor: "
-> 1314 "no conversion function registered." %
1315 (_error_prefix(name), value, type(value)))
1316
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/tensor_conversion_registry.py in _default_conversion_function(***failed resolving arguments***)
50 def _default_conversion_function(value, dtype, name, as_ref):
51 del as_ref # Unused.
---> 52 return constant_op.constant(value, dtype, name=name)
53
54
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/constant_op.py in constant(value, dtype, shape, name)
256 return _eager_fill(shape.as_list(), t, ctx)
257 raise TypeError("Eager execution of tf.constant with unsupported shape "
--> 258 "(value has %d elements, shape is %s with %d elements)." %
259 (num_t, shape, shape.num_elements()))
260 g = ops.get_default_graph()
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/constant_op.py in _constant_impl(value, dtype, shape, name, verify_shape, allow_broadcast)
264 value, dtype=dtype, shape=shape, verify_shape=verify_shape,
265 allow_broadcast=allow_broadcast))
--> 266 dtype_value = attr_value_pb2.AttrValue(type=tensor_value.tensor.dtype)
267 const_tensor = g.create_op(
268 "Const", [], [dtype_value.type],
/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
94 dtype = dtypes.as_dtype(dtype).as_datatype_enum
95 ctx.ensure_initialized()
---> 96 return ops.EagerTensor(value, ctx.device_name, dtype)
97
98
RuntimeError: Can't copy Tensor with type string to device /job:localhost/replica:0/task:0/device:GPU:0.
I have also tried installing different tensorflow versions like latest tensorflow, tensorflow-gpu, tensorflow-gpu=1.12, but got no success.
Though I am able to list out CPUs and GPUs by using
from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())
Please help!
I finally got it to work. There was some unknown bug in tensorflow. It is working properly in tf-nightly build.
When I run the model like this, it's worked fine for me.
tf.debugging.set_log_device_placement(True)
try:
with tf.device('/device:XLA_GPU:0'):
X_train = tf.convert_to_tensor(x_train, dtype=tf.int32)
Y_train = tf.convert_to_tensor(y_train, dtype=tf.float32)
X_dev = tf.convert_to_tensor(x_val, dtype=tf.int32)
Y_dev = tf.convert_to_tensor(y_val, dtype=tf.float32)
_model = tf.keras.Model(review_input, preds)
opt = optimizers.Adam()
_model.compile(loss="mean_absolute_error", optimizer=opt, metrics=['acc'])
except RuntimeError as e:
print(e)
history=_model.fit(X_train, Y_train, epochs=100, batch_size=128, validation_data=(X_dev, Y_dev), verbose=1)

The truth value of an array with more than one element is ambiguous. Use a.any() or a.all(), in image classification problem

I doing image classification using predefined model vgg16, I got 89% accuracy in validation data, To increase the model accuracy, I did an image augmentation, but got some errors. please help me on how to fit for the model.
here my code.
train_datagen = ImageDataGenerator(
rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
train_datagen.fit(X_train)
I am using the input image are 64x64x3.
I am a fit model like this.
history = model.fit_generator(
train_datagen.flow(X_train,y_train),
steps_per_epoch=(X_train)/32 ,
epochs=30,
validation_data=(X_test,y_test),
validation_steps=(X_test)/32,
verbose=1)
Epoch 1/30
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-30-ff3a9aaa40da> in <module>()
5 validation_data=(X_test,y_test),
6 validation_steps=(X_test)/32,
----> 7 verbose=1)
/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name + '` call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1416 use_multiprocessing=use_multiprocessing,
1417 shuffle=shuffle,
-> 1418 initial_epoch=initial_epoch)
1419
1420 #interfaces.legacy_generator_methods_support
/usr/local/lib/python3.6/dist-packages/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
178 steps_done = 0
179 batch_index = 0
--> 180 while steps_done < steps_per_epoch:
181 generator_output = next(output_generator)
182
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
Referring to #jmetz, #suri you have the same issue with your validation_steps parameter, as you initialized it to (X_test)/32(probably not a scalar).
Check your validation_steps.shape / len(validation_steps) and your steps_per_epoch.shape / len(steps_per_epoch)(depending on the input dimensions).
They have to be scalars.
It looks like steps_per_epoch should be a scalar (single value).
You set it to (X_train)/32.

fitting a simple image generator in a keras model

I have a keras model that takes an input image and a label value.
I have a data generator that reads the image, processes it and feeds it into the net
from PIL import Image
def my_iterator():
i = 0
while True:
img_name = train_df.loc[i,'Image']
img_label = train_df.loc[i,'Id']
img = Image.open('master_train/'+str(img_name)).convert('L')
print(img.mode)
longer_side = max(img.size)
horizontal_padding = (longer_side - img.size[0]) / 2
vertical_padding = (longer_side - img.size[1]) / 2
img = img.crop((-horizontal_padding,-vertical_padding,img.size[0] + horizontal_padding,img.size[1] + vertical_padding))
img.thumbnail((128,128),Image.ANTIALIAS)
img_array = np.asarray(img,dtype='uint8')
img_array = img_array[:,:,np.newaxis]
print(img_array.ndim)
yield img_array,img_label
i = (i+1) % len(train_df)
from keras.models import Model
from keras.layers import Input,Dense
input_layer = Input(shape=(128,128,1))
x = Dense(100,activation='relu')(input_layer)
output_layer = Dense(1,activation='sigmoid')(x)
model = Model(inputs=input_layer,outputs=output_layer)
model.compile(loss='binary_crossentropy',optimizer='nadam',metrics['accuracy'])
model.summary()
training_generator = my_iterator()
model.fit(training_generator,steps_per_epoch=1)
I get the following error
AttributeError Traceback (most recent call last)
<ipython-input-189-7efa0828e76d> in <module>()
----> 1 model.fit(train_gen,steps_per_epoch=1)
~/work/venvs/keras3/lib/python3.6/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1628 sample_weight=sample_weight,
1629 class_weight=class_weight,
-> 1630 batch_size=batch_size)
1631 # Prepare validation data.
1632 do_validation = False
~/work/venvs/keras3/lib/python3.6/site-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)
1474 self._feed_input_shapes,
1475 check_batch_axis=False,
-> 1476 exception_prefix='input')
1477 y = _standardize_input_data(y, self._feed_output_names,
1478 output_shapes,
~/work/venvs/keras3/lib/python3.6/site-packages/keras/engine/training.py in _standardize_input_data(data, names, shapes, check_batch_axis, exception_prefix)
74 data = data.values if data.__class__.__name__ == 'DataFrame' else data
75 data = [data]
---> 76 data = [np.expand_dims(x, 1) if x is not None and x.ndim == 1 else x for x in data]
77
78 if len(data) != len(names):
~/work/venvs/keras3/lib/python3.6/site-packages/keras/engine/training.py in <listcomp>(.0)
74 data = data.values if data.__class__.__name__ == 'DataFrame' else data
75 data = [data]
---> 76 data = [np.expand_dims(x, 1) if x is not None and x.ndim == 1 else x for x in data]
77
78 if len(data) != len(names):
AttributeError: 'generator' object has no attribute 'ndim'
​
You should be using fit_generator to train a model using a generator, not the plain fit function.

Resources