Cast string to float is not supported - python-3.x

I am working on a tensorflow course and am trying to apply what I am learning to my own data. I am getting a "Cast string to float is not supported". Using the pandas dataframe INFO(), I confirm that all columns that are OBJECTS I have turned to a feature column using categorical_column_with_hash_bucket, all INT64 or FLOAT64 I have used numeric_column. Why is this error popping up?
Here is my code and error:
import tensorflow as tf
import pandas as pd
from sklearn.model_selection import train_test_split
tree_data_file = r'\\David\f\first_test_feature_cols_v2.csv'
tree_data = pd.read_csv(tree_data_file)
### Create feature columns for continuous data
b3_sum = tf.feature_column.numeric_column('b3_sum')
im3b3_s = tf.feature_column.numeric_column('im3b3_s')
imred = tf.feature_column.numeric_column('imred')
# Create feature columns for categorical data
sp1 = tf.feature_column.categorical_column_with_hash_bucket('sp1',hash_bucket_size=10)
sp2 = tf.feature_column.categorical_column_with_hash_bucket('sp2',hash_bucket_size=10)
feat_cols = [b3_sum,im3b3_s,im_red,sp1,sp2]
#TRAIN TEST SPLIT
x_data = tree_data.drop('sp_call', axis=1)
labels = tree_data['sp_call']
X_train, X_test, Y_train, Y_test = train_test_split(x_data, labels, test_size = 0.3)
input_func = tf.estimator.inputs.pandas_input_fn(x = X_train, y=Y_train, batch_size=10, num_epochs=1000, shuffle=True)
model = tf.estimator.LinearClassifier(feature_columns=feat_cols, n_classes=2)
model.train(input_fn=input_func, steps=1000)
Caused by op 'linear/head/ToFloat', defined at:
File "<string>", line 1, in <module>
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\idlelib\run.py", line 144, in main
ret = method(*args, **kwargs)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\idlelib\run.py", line 474, in runcode
exec(code, self.locals)
File "C:\Users\david\Documents\Neural_Network\test_neural_network_v1.py", line 59, in <module>
model.train(input_fn=input_func, steps=1000)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\estimator.py", line 354, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1207, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1237, in _train_model_default
features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1195, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\canned\linear.py", line 384, in _model_fn
sparse_combiner=sparse_combiner)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\canned\linear.py", line 215, in _linear_model_fn
logits=logits)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\canned\head.py", line 239, in create_estimator_spec
regularization_losses))
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\canned\head.py", line 1209, in _create_tpu_estimator_spec
features=features, mode=mode, logits=logits, labels=labels))
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\canned\head.py", line 1115, in create_loss
labels = math_ops.to_float(labels)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\math_ops.py", line 732, in to_float
return cast(x, dtypes.float32, name=name)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\math_ops.py", line 677, in cast
x = gen_math_ops.cast(x, base_type, name=name)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 1754, in cast
"Cast", x=x, DstT=DstT, Truncate=Truncate, name=name)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\util\deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 3274, in create_op
op_def=op_def)
File "C:\Users\david\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 1770, in __init__
self._traceback = tf_stack.extract_stack()
UnimplementedError (see above for traceback): Cast string to float is not supported
[[node linear/head/ToFloat (defined at C:\Users\david\Documents\Neural_Network\test_neural_network_v1.py:59) = Cast[DstT=DT_FLOAT, SrcT=DT_STRING, Truncate=false, _class=["loc:#linea...t/Switch_1"], _device="/job:localhost/replica:0/task:0/device:CPU:0"](linear/head/labels/_781)]]

Related

Evaluating pairwise distances between the output of a tf.keras.model

I am trying to create a custom loss function in tensorflow. I am using tensorflow v2.0.rc0 for running the code. Following is the code and the function min_dist_loss computes the pairwise loss between the output of the neural network. Here's the code
def min_dist_loss(_, y_pred):
distances = []
for i in range(0, 16):
for j in range(i + 1, 16):
distances.append(tf.linalg.norm(y_pred[i] - y_pred[j]))
return -tf.reduce_min(distances)
and the module is being initialized and compiled as follows
import tensorflow as tf
from tensorboard.plugins.hparams import api as hp
HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([6, 7]))
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd']))
METRIC_ACCURACY = 'accuracy'
with tf.summary.create_file_writer('logs\hparam_tuning').as_default():
hp.hparams_config(
hparams=[HP_NUM_UNITS, HP_OPTIMIZER],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')]
)
def train_test_model(logdir, hparams):
weight1 = np.random.normal(loc=0.0, scale=0.01, size=[4, hparams[HP_NUM_UNITS]])
init1 = tf.constant_initializer(weight1)
weight2 = np.random.normal(loc=0.0, scale=0.01, size=[hparams[HP_NUM_UNITS], 7])
init2 = tf.constant_initializer(weight2)
model = tf.keras.models.Sequential([
# tf.keras.layers.Flatten(),
tf.keras.layers.Dense(hparams[HP_NUM_UNITS], activation=tf.nn.sigmoid, kernel_initializer=init1),
tf.keras.layers.Dense(7, activation=tf.nn.sigmoid, kernel_initializer=init2) if hparams[HP_NUM_UNITS] == 6 else
None,
])
model.compile(
optimizer=hparams[HP_OPTIMIZER],
loss=min_dist_loss,
# metrics=['accuracy'],
)
x_train = [list(k) for k in itertools.product([0, 1], repeat=4)]
shuffle(x_train)
x_train = 2 * np.array(x_train) - 1
model.fit(
x_train, epochs=1, batch_size=16,
callbacks=[
tf.keras.callbacks.TensorBoard(logdir),
hp.KerasCallback(logdir, hparams)
],
)
Now since the tensor object y_pred in the min_dist_loss is an object of shape [?, 7], indexing with i is throwing the following error:
Traceback (most recent call last):
File "/home/pc/Documents/user/code/keras_tensorflow/src/try1.py", line 95, in <module>
run('logs\hparam_tuning' + run_name, hparams)
File "/home/pc/Documents/user/code/keras_tensorflow/src/try1.py", line 78, in run
accuracy = train_test_model(run_dir, hparams)
File "/home/pc/Documents/user/code/keras_tensorflow/src/try1.py", line 66, in train_test_model
hp.KerasCallback(logdir, hparams)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py", line 734, in fit
use_multiprocessing=use_multiprocessing)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 324, in fit
total_epochs=epochs)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 123, in run_one_epoch
batch_outs = execution_function(iterator)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 86, in execution_function
distributed_function(input_fn))
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py", line 427, in __call__
self._initialize(args, kwds, add_initializers_to=initializer_map)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py", line 370, in _initialize
*args, **kwds))
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py", line 1847, in _get_concrete_function_internal_garbage_collected
graph_function, _, _ = self._maybe_define_function(args, kwargs)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py", line 2147, in _maybe_define_function
graph_function = self._create_graph_function(args, kwargs)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py", line 2038, in _create_graph_function
capture_by_value=self._capture_by_value),
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/framework/func_graph.py", line 915, in func_graph_from_py_func
func_outputs = python_func(*func_args, **func_kwargs)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py", line 320, in wrapped_fn
return weak_wrapped_fn().__wrapped__(*args, **kwds)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 73, in distributed_function
per_replica_function, args=(model, x, y, sample_weights))
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py", line 760, in experimental_run_v2
return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py", line 1787, in call_for_each_replica
return self._call_for_each_replica(fn, args, kwargs)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py", line 2132, in _call_for_each_replica
return fn(*args, **kwargs)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/autograph/impl/api.py", line 292, in wrapper
return func(*args, **kwargs)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 264, in train_on_batch
output_loss_metrics=model._output_loss_metrics)
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_eager.py", line 311, in train_on_batch
output_loss_metrics=output_loss_metrics))
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_eager.py", line 252, in _process_single_batch
training=training))
File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_eager.py", line 166, in _model_loss
per_sample_losses = loss_fn.call(targets[i], outs[i])
IndexError: list index out of range
How do I compute the minimum distance in this setting? Any help is appreciated. Also, if there are any errors in other parts of the code, please feel free to point it out. I am new to using keras on tensorflow.
Keras is expecting you to provide the true labels as well. Since you're defining your own loss function and you're not using the true labels, you can pass some garbage labels. Eg: np.arange(16).
Change your model.fit as below and it should work
model.fit(
x_train, np.arange(x_train.shape[0]), epochs=1, batch_size=16,
callbacks=[
tf.keras.callbacks.TensorBoard(logdir),
hp.KerasCallback(logdir, hparams)
],
)

Getting Error while classifying text data using word2vec

I want to use my own word dataset for creating the embeddings. And use my own label data for training and testing my model. For that I have already created my own word embeddings using word2vec. And facing problem in training my model with label data.
I am getting error while trying to train model. My model creation code:
# create the tokenizer
tokenizer = Tokenizer()
tokenizer.fit_on_texts(X_train)
encoded_docs = tokenizer.texts_to_sequences(X_train)
max_length = max([len(s.split()) for s in X_train])
X_train = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
tokenizer = Tokenizer()
tokenizer.fit_on_texts(X_test)
encoded_docs = tokenizer.texts_to_sequences(X_test)
X_test = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
# setup the embedding layer
embeddings = Embedding(input_dim=embedding_matrix.shape[0], output_dim=embedding_matrix.shape[1],
weights=[embedding_matrix],input_length= max_length, trainable=False)
new_model = Sequential() new_model.add(embeddings)
new_model.add(Conv1D(filters=128, kernel_size=5, activation='relu'))
new_model.add(MaxPooling1D(pool_size=2)) new_model.add(Flatten())
new_model.add(Dense(1, activation='sigmoid'))
And this is how I have created embedding matrix-
embedding_matrix = np.zeros((len(model.wv.vocab), vector_dim))
for i in range(len(model.wv.vocab)):
embedding_vector = model.wv[model.wv.index2word[i]]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
By doing so I am getting the following error-
WARNING:tensorflow:From /Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py:1290: calling reduce_mean (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.
Instructions for updating:
keep_dims is deprecated, use keepdims instead
Epoch 1/10
Traceback (most recent call last):
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1322, in _do_call
return fn(*args)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1307, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1409, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[27,2] = 1049 is not in [0, 1045)
[[Node: embedding_1/GatherV2 = GatherV2[Taxis=DT_INT32, Tindices=DT_INT32, Tparams=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"](embedding_1/embeddings/read, embedding_1/Cast, embedding_1/GatherV2/axis)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/faysal/Desktop/My Computer/D/Code Workspace/Research-IoT/embedding-tut/src/main.py", line 359, in <module>
custom_keras_model(embedding_matrix, model.wv)
File "/Users/faysal/Desktop/My Computer/D/Code Workspace/Research-IoT/Collaboration/embedding-tut/src/main.py", line 295, in custom_keras_model
new_model.fit(X_train, y_train, epochs=10, verbose=2)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/keras/models.py", line 867, in fit
initial_epoch=initial_epoch)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/keras/engine/training.py", line 1598, in fit
validation_steps=validation_steps)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/keras/engine/training.py", line 1183, in _fit_loop
outs = f(ins_batch)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py", line 2273, in __call__
**self.session_kwargs)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 900, in run
run_metadata_ptr)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1316, in _do_run
run_metadata)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[27,2] = 1049 is not in [0, 1045)
[[Node: embedding_1/GatherV2 = GatherV2[Taxis=DT_INT32, Tindices=DT_INT32, Tparams=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"](embedding_1/embeddings/read, embedding_1/Cast, embedding_1/GatherV2/axis)]]
Caused by op 'embedding_1/GatherV2', defined at:
File "/Users/faysal/Desktop/My Computer/D/Code Workspace/Research-IoT/Collaboration/embedding-tut/src/main.py", line 359, in <module>
custom_keras_model(embedding_matrix, model.wv)
File "/Users/faysal/Desktop/My Computer/D/Code Workspace/Research-IoT/Collaboration/embedding-tut/src/main.py", line 278, in custom_keras_model
new_model.add(embeddings)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/keras/models.py", line 442, in add
layer(x)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/keras/engine/topology.py", line 602, in __call__
output = self.call(inputs, **kwargs)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/keras/layers/embeddings.py", line 134, in call
out = K.gather(self.embeddings, inputs)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py", line 1134, in gather
return tf.gather(reference, indices)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/ops/array_ops.py", line 2736, in gather
return gen_array_ops.gather_v2(params, indices, axis, name=name)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/ops/gen_array_ops.py", line 3065, in gather_v2
"GatherV2", params=params, indices=indices, axis=axis, name=name)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 3392, in create_op
op_def=op_def)
File "/Users/faysal/anaconda2/envs/python3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 1718, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): indices[27,2] = 1049 is not in [0, 1045)
[[Node: embedding_1/GatherV2 = GatherV2[Taxis=DT_INT32, Tindices=DT_INT32, Tparams=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"](embedding_1/embeddings/read, embedding_1/Cast, embedding_1/GatherV2/axis)]]
Process finished with exit code 1
I am getting error in fitting training data into the model. I think I have mistaken in calculting the training data shape and injecting it into the model.
You are using two different Tokenizers and you train them separately on train and test. What happens is, that your tokens do not match for training and test. Your error is caused, because a token occurs (1049) which is not is not in max_length. Even if you fix that, your model will not work, if you have two tokenizers.
What you should do it to fit your Tokenizer on all data (X_train and X_test) and use just one single Tokenizer.

Keras stateful LSTM multi-gpu error Incompatible shapes: [2540] vs. [508] batch size multiplied buy nGPU

I had the same issue but when I tried to apply the same fix I have run into another error. I am however running on 5 gpus. I have read that you need to make sure that your samples are divisible by both the batch sive and number of gpus but I have done that. I have scoured the internet for days and am unable to find anything that has been able to fix the issue I am having. I am running keras v2.0.9 and tensor flow v1.1.0
VARIABLES:
attributeTables[0] is a numpy array shape (35560, 700)
y is a numpy array shape (35560, ) I have also tried using shape (35560, 1) for y but all that happens is the "Incompatible shapes: [2540] vs. [508]" changes from that to "Incompatible shapes: [2540, 1] vs. [508, 1]"
So this says to me that the issue is only with the targetsand that the expected batch size is getting multiplied somewhere in the middle of the process only for the targets and not for attributes causing a mismatch or at least only durring validation I'm not sure.
Here is the code and error in question.
import numpy as np
from keras.models import Sequential
from keras.utils import multi_gpu_model
from keras.layers import Dense
from keras.layers import LSTM
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
def baseline_model():
# create model
print("Building Layers")
model = Sequential()
model.add(LSTM(700, batch_input_shape=(batchSize, X.shape[1], X.shape[2]), activation='tanh', return_sequences=False, stateful=True))
model.add(Dense(1))
print("Building Parallel model")
parallel_model = multi_gpu_model(model, gpus=nGPU)
# Compile model
#model.compile(loss='mean_squared_error', optimizer='adam')
print("Compiling Model")
parallel_model.compile(loss='mae', optimizer='adam', metrics=['accuracy'])
return parallel_model
def buildModel():
print("Bulding Model")
mlp = baseline_model()
print("Fitting Model")
return mlp.fit(X_train, y_train, epochs=1, batch_size=batchSize, shuffle=False, validation_data=(X_test, y_test))
print("Scaling")
scaler = StandardScaler()
X_Scaled = scaler.fit_transform(attributeTables[0])
print("Finding Batch Size")
nGPU = 5
batchSize = 500
while len(X_Scaled) % (batchSize * nGPU) != 0:
batchSize += 1
print("Filling Arrays")
X = X_Scaled.reshape((X_Scaled.shape[0], X_Scaled.shape[1], 1))
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=.8)
print("Calling buildModel()")
model = buildModel()
print("Ploting History")
plt.plot(model.history['loss'], label='train')
plt.plot(model.history['val_loss'], label='test')
plt.legend()
plt.show()
Here is my complete output.
Beginning OHLC Load
Time took : 7.571000099182129
Making gloabal copies
Time took : 0.0
Using TensorFlow backend.
Scaling
Finding Batch Size
Filling Arrays
Calling buildModel()
Bulding Model
Building Layers
C:\ProgramData\Anaconda3\lib\site-packages\sklearn\model_selection\_split.py:2010: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.
FutureWarning)
Building Parallel model
Compiling Model
Fitting Model
Train on 28448 samples, validate on 7112 samples
Epoch 1/1
Traceback (most recent call last):
File "<ipython-input-2-74c49f05bfbc>", line 1, in <module>
runfile('C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py', wdir='C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor')
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 710, in runfile
execfile(filename, namespace)
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 101, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 77, in <module>
model = buildModel()
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 57, in buildModel
return mlp.fit(X_train, y_train, epochs=1, batch_size=batchSize, shuffle=False, validation_data=(X_test, y_test))
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1631, in fit
validation_steps=validation_steps)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1213, in _fit_loop
outs = f(ins_batch)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py", line 2332, in __call__
**self.session_kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 778, in run
run_metadata_ptr)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 982, in _run
feed_dict_string, options, run_metadata)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1032, in _do_run
target_list, options, run_metadata)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1052, in _do_call
raise type(e)(node_def, op, message)
InvalidArgumentError: Incompatible shapes: [2540,1] vs. [508,1]
[[Node: training/Adam/gradients/loss/concatenate_1_loss/sub_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _class=["loc:#loss/concatenate_1_loss/sub"], _device="/job:localhost/replica:0/task:0/gpu:0"](training/Adam/gradients/loss/concatenate_1_loss/sub_grad/Shape, training/Adam/gradients/loss/concatenate_1_loss/sub_grad/Shape_1)]]
[[Node: replica_1/sequential_1/dense_1/BiasAdd/_313 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:1", send_device_incarnation=1, tensor_name="edge_1355_replica_1/sequential_1/dense_1/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
Caused by op 'training/Adam/gradients/loss/concatenate_1_loss/sub_grad/BroadcastGradientArgs', defined at:
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 245, in <module>
main()
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 241, in main
kernel.start()
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 477, in start
ioloop.IOLoop.instance().start()
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\ioloop.py", line 832, in start
self._run_callback(self._callbacks.popleft())
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\ioloop.py", line 605, in _run_callback
ret = callback()
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 265, in enter_eventloop
self.eventloop(self)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\eventloops.py", line 106, in loop_qt5
return loop_qt4(kernel)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\eventloops.py", line 99, in loop_qt4
_loop_qt(kernel.app)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\eventloops.py", line 83, in _loop_qt
app.exec_()
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\eventloops.py", line 39, in process_stream_events
kernel.do_one_iteration()
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 298, in do_one_iteration
stream.flush(zmq.POLLIN, 1)
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 352, in flush
self._handle_recv()
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 235, in dispatch_shell
handler(stream, idents, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 533, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2698, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2808, in run_ast_nodes
if self.run_code(code, result):
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2862, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-2-74c49f05bfbc>", line 1, in <module>
runfile('C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py', wdir='C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor')
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 710, in runfile
execfile(filename, namespace)
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 101, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 77, in <module>
model = buildModel()
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 57, in buildModel
return mlp.fit(X_train, y_train, epochs=1, batch_size=batchSize, shuffle=False, validation_data=(X_test, y_test))
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1608, in fit
self._make_train_function()
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 990, in _make_train_function
loss=self.total_loss)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\legacy\interfaces.py", line 87, in wrapper
return func(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\optimizers.py", line 415, in get_updates
grads = self.get_gradients(loss, params)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\optimizers.py", line 73, in get_gradients
grads = K.gradients(loss, params)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py", line 2369, in gradients
return tf.gradients(loss, variables, colocate_gradients_with_ops=True)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 560, in gradients
grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 368, in _MaybeCompile
return grad_fn() # Exit early
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 560, in <lambda>
grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\math_grad.py", line 609, in _SubGrad
rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_array_ops.py", line 411, in _broadcast_gradient_args
name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 768, in apply_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 1228, in __init__
self._traceback = _extract_stack()
...which was originally created as op 'loss/concatenate_1_loss/sub', defined at:
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 245, in <module>
main()
[elided 27 identical lines from previous traceback]
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 77, in <module>
model = buildModel()
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 55, in buildModel
mlp = baseline_model()
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 29, in baseline_model
parallel_model.compile(loss='mae', optimizer='adam', metrics=['accuracy'])
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 860, in compile
sample_weight, mask)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 460, in weighted
score_array = fn(y_true, y_pred)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\losses.py", line 13, in mean_absolute_error
return K.mean(K.abs(y_pred - y_true), axis=-1)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\math_ops.py", line 821, in binary_op_wrapper
return func(x, y, name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 2627, in _sub
result = _op_def_lib.apply_op("Sub", x=x, y=y, name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 768, in apply_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 1228, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): Incompatible shapes: [2540,1] vs. [508,1]
[[Node: training/Adam/gradients/loss/concatenate_1_loss/sub_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _class=["loc:#loss/concatenate_1_loss/sub"], _device="/job:localhost/replica:0/task:0/gpu:0"](training/Adam/gradients/loss/concatenate_1_loss/sub_grad/Shape, training/Adam/gradients/loss/concatenate_1_loss/sub_grad/Shape_1)]]
[[Node: replica_1/sequential_1/dense_1/BiasAdd/_313 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:1", send_device_incarnation=1, tensor_name="edge_1355_replica_1/sequential_1/dense_1/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
Daniel Moller's link was right when i disabled the parallel model and put it on one GPU the stateful worked no ptoblem. Currently waiting on it to train. Will post results.
I just published an experimental utility, stateful_multi_gpu, to handle stateful model training of multiple GPUs. I'm interested to know if it is of use to you.
Please also see my answer for the same question Daniel Möller referred to.

Error when using batch_input_shape for stateful LSTM in RNN

I have a dataset of shape (10000, 128) (samples= 10,000, and features=128) where the class labels are binary. I want to use RNN for model training using Keras library. I wrote the following code:
tr_C, ts_C, tr_r, ts_r = train_test_split(C, r, train_size=.8)
batch_size = 32
print('Build STATEFUL model...')
model = Sequential()
model.add(LSTM(64, (batch_size, C.shape[0], C.shape[1]), return_sequences=False, stateful=True))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print('Training...')
model.fit(tr_C, ts_r,
batch_size=batch_size, epochs=1, shuffle=False,
validation_data=(ts_C, ts_r))
But I get this error:
ValueError: Error when checking input: expected lstm_1_input to have 3 dimensions, but got array with shape (8000, 128)
I don't understand this error. How can I fix it?
Thank you
You need to do following steps:
Reshape C by:
C = C.reshape((c.shape[0], c.shape[1], 1))
Adjust LSTM layer:
model.add(LSTM(64, (batch_size, C.shape[1], C.shape[2]), return_sequences=False, stateful=True))
I had the same issue but when I tried to apply the same fix I have run into another error. I am however running on 5 gpus. I have read that you need to make sure that your samples are divisible by both the batch sive and number of gpus but I have done that. I have scoured the internet for days and am unable to find anything that has been able to fix the issue I am having. I am running keras v2.0.9 and tensor flow v1.1.0
VARIABLES:
attributeTables[0] is a numpy array shape (35560, 700)
y is a numpy array shape (35560, ) I have also tried using shape (35560, 1) for y but all that happens is the "Incompatible shapes: [2540] vs. [508]" changes from that to "Incompatible shapes: [2540, 1] vs. [508, 1]"
So this says to me that the issue is only with the targetsand that the expected batch size is getting multiplied somewhere in the middle of the process only for the targets and not for attributes causing a mismatch or at least only durring validation I'm not sure.
Here is the code and error in question.
import numpy as np
from keras.models import Sequential
from keras.utils import multi_gpu_model
from keras.layers import Dense
from keras.layers import LSTM
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
def baseline_model():
# create model
print("Building Layers")
model = Sequential()
model.add(LSTM(700, batch_input_shape=(batchSize, X.shape[1], X.shape[2]), activation='tanh', return_sequences=False, stateful=True))
model.add(Dense(1))
print("Building Parallel model")
parallel_model = multi_gpu_model(model, gpus=nGPU)
# Compile model
#model.compile(loss='mean_squared_error', optimizer='adam')
print("Compiling Model")
parallel_model.compile(loss='mae', optimizer='adam', metrics=['accuracy'])
return parallel_model
def buildModel():
print("Bulding Model")
mlp = baseline_model()
print("Fitting Model")
return mlp.fit(X_train, y_train, epochs=1, batch_size=batchSize, shuffle=False, validation_data=(X_test, y_test))
print("Scaling")
scaler = StandardScaler()
X_Scaled = scaler.fit_transform(attributeTables[0])
print("Finding Batch Size")
nGPU = 5
batchSize = 500
while len(X_Scaled) % (batchSize * nGPU) != 0:
batchSize += 1
print("Filling Arrays")
X = X_Scaled.reshape((X_Scaled.shape[0], X_Scaled.shape[1], 1))
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=.8)
print("Calling buildModel()")
model = buildModel()
print("Ploting History")
plt.plot(model.history['loss'], label='train')
plt.plot(model.history['val_loss'], label='test')
plt.legend()
plt.show()
Here is my complete output.
Beginning OHLC Load
Time took : 7.571000099182129
Making gloabal copies
Time took : 0.0
Using TensorFlow backend.
Scaling
Finding Batch Size
Filling Arrays
Calling buildModel()
Bulding Model
Building Layers
C:\ProgramData\Anaconda3\lib\site-packages\sklearn\model_selection\_split.py:2010: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.
FutureWarning)
Building Parallel model
Compiling Model
Fitting Model
Train on 28448 samples, validate on 7112 samples
Epoch 1/1
Traceback (most recent call last):
File "<ipython-input-2-74c49f05bfbc>", line 1, in <module>
runfile('C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py', wdir='C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor')
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 710, in runfile
execfile(filename, namespace)
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 101, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 77, in <module>
model = buildModel()
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 57, in buildModel
return mlp.fit(X_train, y_train, epochs=1, batch_size=batchSize, shuffle=False, validation_data=(X_test, y_test))
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1631, in fit
validation_steps=validation_steps)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1213, in _fit_loop
outs = f(ins_batch)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py", line 2332, in __call__
**self.session_kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 778, in run
run_metadata_ptr)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 982, in _run
feed_dict_string, options, run_metadata)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1032, in _do_run
target_list, options, run_metadata)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1052, in _do_call
raise type(e)(node_def, op, message)
InvalidArgumentError: Incompatible shapes: [2540,1] vs. [508,1]
[[Node: training/Adam/gradients/loss/concatenate_1_loss/sub_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _class=["loc:#loss/concatenate_1_loss/sub"], _device="/job:localhost/replica:0/task:0/gpu:0"](training/Adam/gradients/loss/concatenate_1_loss/sub_grad/Shape, training/Adam/gradients/loss/concatenate_1_loss/sub_grad/Shape_1)]]
[[Node: replica_1/sequential_1/dense_1/BiasAdd/_313 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:1", send_device_incarnation=1, tensor_name="edge_1355_replica_1/sequential_1/dense_1/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
Caused by op 'training/Adam/gradients/loss/concatenate_1_loss/sub_grad/BroadcastGradientArgs', defined at:
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 245, in <module>
main()
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 241, in main
kernel.start()
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 477, in start
ioloop.IOLoop.instance().start()
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\ioloop.py", line 832, in start
self._run_callback(self._callbacks.popleft())
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\ioloop.py", line 605, in _run_callback
ret = callback()
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 265, in enter_eventloop
self.eventloop(self)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\eventloops.py", line 106, in loop_qt5
return loop_qt4(kernel)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\eventloops.py", line 99, in loop_qt4
_loop_qt(kernel.app)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\eventloops.py", line 83, in _loop_qt
app.exec_()
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\eventloops.py", line 39, in process_stream_events
kernel.do_one_iteration()
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 298, in do_one_iteration
stream.flush(zmq.POLLIN, 1)
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 352, in flush
self._handle_recv()
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 235, in dispatch_shell
handler(stream, idents, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 533, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2698, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2808, in run_ast_nodes
if self.run_code(code, result):
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2862, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-2-74c49f05bfbc>", line 1, in <module>
runfile('C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py', wdir='C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor')
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 710, in runfile
execfile(filename, namespace)
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 101, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 77, in <module>
model = buildModel()
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 57, in buildModel
return mlp.fit(X_train, y_train, epochs=1, batch_size=batchSize, shuffle=False, validation_data=(X_test, y_test))
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1608, in fit
self._make_train_function()
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 990, in _make_train_function
loss=self.total_loss)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\legacy\interfaces.py", line 87, in wrapper
return func(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\optimizers.py", line 415, in get_updates
grads = self.get_gradients(loss, params)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\optimizers.py", line 73, in get_gradients
grads = K.gradients(loss, params)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py", line 2369, in gradients
return tf.gradients(loss, variables, colocate_gradients_with_ops=True)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 560, in gradients
grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 368, in _MaybeCompile
return grad_fn() # Exit early
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 560, in <lambda>
grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\math_grad.py", line 609, in _SubGrad
rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_array_ops.py", line 411, in _broadcast_gradient_args
name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 768, in apply_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 1228, in __init__
self._traceback = _extract_stack()
...which was originally created as op 'loss/concatenate_1_loss/sub', defined at:
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 245, in <module>
main()
[elided 27 identical lines from previous traceback]
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 77, in <module>
model = buildModel()
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 55, in buildModel
mlp = baseline_model()
File "C:/Users/BeeAndTurtle/Documents/Programming/Python/Kraken_API_Market_Prediction/predictor/test.py", line 29, in baseline_model
parallel_model.compile(loss='mae', optimizer='adam', metrics=['accuracy'])
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 860, in compile
sample_weight, mask)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 460, in weighted
score_array = fn(y_true, y_pred)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\losses.py", line 13, in mean_absolute_error
return K.mean(K.abs(y_pred - y_true), axis=-1)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\math_ops.py", line 821, in binary_op_wrapper
return func(x, y, name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 2627, in _sub
result = _op_def_lib.apply_op("Sub", x=x, y=y, name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 768, in apply_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 1228, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): Incompatible shapes: [2540,1] vs. [508,1]
[[Node: training/Adam/gradients/loss/concatenate_1_loss/sub_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _class=["loc:#loss/concatenate_1_loss/sub"], _device="/job:localhost/replica:0/task:0/gpu:0"](training/Adam/gradients/loss/concatenate_1_loss/sub_grad/Shape, training/Adam/gradients/loss/concatenate_1_loss/sub_grad/Shape_1)]]
[[Node: replica_1/sequential_1/dense_1/BiasAdd/_313 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:1", send_device_incarnation=1, tensor_name="edge_1355_replica_1/sequential_1/dense_1/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]

Logits and Labels mismatch in Tensorflow

there is a mismatch between the logits and lables in the Tensorflow after one hot encoding.
And my batch size is 256. How can I manage to get the batch size in labels Tensor too ? I guess this issue is related to the LabelEncoder and one-hot encoder. Any help is appreciable.
Please find the code below.
from sklearn import preprocessing
le = preprocessing.LabelEncoder()
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = tf.one_hot(le.fit_transform(labels), n_classes)))
optimizer = tf.train.GradientDescentOptimizer(learning_rate = learn_rate).minimize(cost)
correct_prediction = tf.equal(tf.argmax(logits,1), tf.argmax(tf.one_hot(le.fit_transform(labels), n_classes),1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
batchSize = 256
epochs = 20 # 200epoch+.5lr = 99.6
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
total_batches = batches(batchSize, train_features, train_labels)
for epoch in range(epochs):
for batch_features, batch_labels in total_batches:
train_data = {features: batch_features, labels : batch_labels, keep_prob : 0.5}
sess.run(optimizer, feed_dict = train_data)
# Print status for every 100 epochs
if epoch % 10 == 0:
valid_accuracy = sess.run(
accuracy,
feed_dict={
features: val_features,
labels: val_labels,
keep_prob : 0.5})
print('Epoch {:<3} - Validation Accuracy: {}'.format(
epoch,
valid_accuracy))
Accuracy = sess.run(accuracy, feed_dict={features : test_features, labels :test_labels, keep_prob : 1.0})
# Save the model
saver.save(sess, save_file)
print('Trained Model Saved.')
prediction=tf.argmax(logits,1)
output_array = le.inverse_transform(prediction.eval(feed_dict={features : test_features, keep_prob: 1.0}))
prediction = np.reshape(prediction, (test_features.shape[0],1))
np.savetxt("prediction.csv", prediction, delimiter=",")
And I am getting the Invalid Argument Error as indicated below.
InvalidArgumentError: logits and labels must be same size: logits_size=[256,1161] labels_size=[1,1161]
[[Node: SoftmaxCrossEntropyWithLogits = SoftmaxCrossEntropyWithLogits[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](Reshape, Reshape_1)]]
Caused by op 'SoftmaxCrossEntropyWithLogits', defined at:
File "C:\Anaconda\envs\gpu\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:\Anaconda\envs\gpu\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\__main__.py", line 3, in <module>
app.launch_new_instance()
File "C:\Anaconda\envs\gpu\lib\site-packages\traitlets\config\application.py", line 658, in launch_instance
app.start()
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\kernelapp.py", line 477, in start
ioloop.IOLoop.instance().start()
File "C:\Anaconda\envs\gpu\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "C:\Anaconda\envs\gpu\lib\site-packages\tornado\ioloop.py", line 888, in start
handler_func(fd_obj, events)
File "C:\Anaconda\envs\gpu\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\Anaconda\envs\gpu\lib\site-packages\zmq\eventloop\zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "C:\Anaconda\envs\gpu\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "C:\Anaconda\envs\gpu\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "C:\Anaconda\envs\gpu\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\kernelbase.py", line 235, in dispatch_shell
handler(stream, idents, msg)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\zmqshell.py", line 533, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\Anaconda\envs\gpu\lib\site-packages\IPython\core\interactiveshell.py", line 2698, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\Anaconda\envs\gpu\lib\site-packages\IPython\core\interactiveshell.py", line 2802, in run_ast_nodes
if self.run_code(code, result):
File "C:\Anaconda\envs\gpu\lib\site-packages\IPython\core\interactiveshell.py", line 2862, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-5-9a6fe2134e3e>", line 52, in <module>
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = tf.one_hot(le.fit_transform(labels), n_classes)))
File "C:\Anaconda\envs\gpu\lib\site-packages\tensorflow\python\ops\nn_ops.py", line 1594, in softmax_cross_entropy_with_logits
precise_logits, labels, name=name)
File "C:\Anaconda\envs\gpu\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 2380, in _softmax_cross_entropy_with_logits
features=features, labels=labels, name=name)
File "C:\Anaconda\envs\gpu\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 767, in apply_op
op_def=op_def)
File "C:\Anaconda\envs\gpu\lib\site-packages\tensorflow\python\framework\ops.py", line 2506, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\Anaconda\envs\gpu\lib\site-packages\tensorflow\python\framework\ops.py", line 1269, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): logits and labels must be same size: logits_size=[256,1161] labels_size=[1,1161]
[[Node: SoftmaxCrossEntropyWithLogits = SoftmaxCrossEntropyWithLogits[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](Reshape, Reshape_1)]]
Issue was with tf.one_hot(le.fit_transform(labels), n_classes).
This passes a tensor where the numpy array was needed. After calling eval() for this Tensor, Issue is resolved.

Resources