what does set_memory_growth do in tensorflow2 - keras

Question

I'm training a cnn on ubuntu server with keras and tensorflow 2. If I run my code without the added code below for memory growth, it throws the error I've posted below. I checked my driver memory and I've posted it below. It looks like my graphics card was running out of memory. I've posted my original code, and error below. I got the added memory growth code from the SO post below. The post says it allows the gpu memory to grow, I'm wondering what that means? Also when I check after running the additional code it still shows that I have a gpu device enabled, so does that mean that my code is still being run on gpu after I run the additional code below to allow gpu memory to grow?
post referenced:
AttributeError: module 'tensorflow' has no attribute 'ConfigProto'
added code:
import tensorflow as tf
physical_devices = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(physical_devices[0], True)
code:
nvidia-smi
output:
Tue Mar 8 14:52:55 2022
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 470.103.01 Driver Version: 470.103.01 CUDA Version: 11.4 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 NVIDIA GeForce ... Off | 00000000:42:00.0 Off | N/A |
| 0% 37C P8 10W / 260W | 7970MiB / 7979MiB | 0% Default |
| | | N/A |
+-------------------------------+----------------------+----------------------+
+-----------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=============================================================================|
| 0 N/A N/A 2607 C ...a3/envs/tf-gpu/bin/python 7967MiB |
+-----------------------------------------------------------------------------+
original code:
code:
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Flatten
model = Sequential()
# CONVOLUTIONAL LAYER
model.add(Conv2D(filters=32, kernel_size=(4,4),input_shape=(28, 28, 1), activation='relu',))
# POOLING LAYER
model.add(MaxPool2D(pool_size=(2, 2)))
# FLATTEN IMAGES FROM 28 by 28 to 764 BEFORE FINAL LAYER
model.add(Flatten())
# 128 NEURONS IN DENSE HIDDEN LAYER (YOU CAN CHANGE THIS NUMBER OF NEURONS)
model.add(Dense(128, activation='relu'))
# LAST LAYER IS THE CLASSIFIER, THUS 10 POSSIBLE CLASSES
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
model.summary()
output:
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d (Conv2D) (None, 25, 25, 32) 544
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 12, 12, 32) 0
_________________________________________________________________
flatten (Flatten) (None, 4608) 0
_________________________________________________________________
dense (Dense) (None, 128) 589952
_________________________________________________________________
dense_1 (Dense) (None, 10) 1290
=================================================================
Total params: 591,786
Trainable params: 591,786
Non-trainable params: 0
code:
model.fit(x_train,y_cat_train,epochs=10)
error:
---------------------------------------------------------------------------
UnknownError Traceback (most recent call last)
<ipython-input-19-bed1e94810c6> in <module>
----> 1 model.fit(x_train,y_cat_train,epochs=10)
~/anaconda3/envs/tf-gpu/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
64 def _method_wrapper(self, *args, **kwargs):
65 if not self._in_multi_worker_mode(): # pylint: disable=protected-access
---> 66 return method(self, *args, **kwargs)
67
68 # Running inside `run_distribute_coordinator` already.
~/anaconda3/envs/tf-gpu/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
846 batch_size=batch_size):
847 callbacks.on_train_batch_begin(step)
--> 848 tmp_logs = train_function(iterator)
849 # Catch OutOfRangeError for Datasets of unknown size.
850 # This blocks until the batch has finished executing.
~/anaconda3/envs/tf-gpu/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)
578 xla_context.Exit()
579 else:
--> 580 result = self._call(*args, **kwds)
581
582 if tracing_count == self._get_tracing_count():
~/anaconda3/envs/tf-gpu/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)
642 # Lifting succeeded, so variables are initialized and we can run the
643 # stateless function.
--> 644 return self._stateless_fn(*args, **kwds)
645 else:
646 canon_args, canon_kwds = \
~/anaconda3/envs/tf-gpu/lib/python3.8/site-packages/tensorflow/python/eager/function.py in __call__(self, *args, **kwargs)
2418 with self._lock:
2419 graph_function, args, kwargs = self._maybe_define_function(args, kwargs)
-> 2420 return graph_function._filtered_call(args, kwargs) # pylint: disable=protected-access
2421
2422 #property
~/anaconda3/envs/tf-gpu/lib/python3.8/site-packages/tensorflow/python/eager/function.py in _filtered_call(self, args, kwargs)
1659 `args` and `kwargs`.
1660 """
-> 1661 return self._call_flat(
1662 (t for t in nest.flatten((args, kwargs), expand_composites=True)
1663 if isinstance(t, (ops.Tensor,
~/anaconda3/envs/tf-gpu/lib/python3.8/site-packages/tensorflow/python/eager/function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
1743 and executing_eagerly):
1744 # No tape is watching; skip to running the function.
-> 1745 return self._build_call_outputs(self._inference_function.call(
1746 ctx, args, cancellation_manager=cancellation_manager))
1747 forward_backward = self._select_forward_and_backward_functions(
~/anaconda3/envs/tf-gpu/lib/python3.8/site-packages/tensorflow/python/eager/function.py in call(self, ctx, args, cancellation_manager)
591 with _InterpolateFunctionError(self):
592 if cancellation_manager is None:
--> 593 outputs = execute.execute(
594 str(self.signature.name),
595 num_outputs=self._num_outputs,
~/anaconda3/envs/tf-gpu/lib/python3.8/site-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
57 try:
58 ctx.ensure_initialized()
---> 59 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
60 inputs, attrs, num_outputs)
61 except core._NotOkStatusException as e:
UnknownError: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
[[node sequential/conv2d/Conv2D (defined at <ipython-input-19-bed1e94810c6>:1) ]] [Op:__inference_train_function_753]
Function call stack:
train_function