TSP tutorial example of google's or-tools crashes - python-3.5

The example program for TSP problems crashes on my machine (ubuntu 16.04) with the following message:
WARNING: Logging before InitGoogleLogging() is written to STDERR
F0430 14:40:29.424594 15961 search.cc:2658] Check failed: step > 0 (0 vs. 0)
*** Check failure stack trace: ***
Aborted (core dumped)
installed binary via pip3 install ortools
changed print statements for python 3
replaced pywrapcp.DefaultRoutingSearchParameters() with pywrapcp.RoutingModel.DefaultModelParameters()
The entire program, taken from https://developers.google.com/optimization/routing/tsp/tsp#program1, is below:
from ortools.constraint_solver import pywrapcp
from ortools.constraint_solver import routing_enums_pb2
# Distance callback
class CreateDistanceCallback(object):
"""Create callback to calculate distances between points."""
def __init__(self):
"""Array of distances between points."""
self.matrix = [
[ 0, 2451, 713, 1018, 1631, 1374, 2408, 213, 2571, 875, 1420, 2145, 1972], # New York
[2451, 0, 1745, 1524, 831, 1240, 959, 2596, 403, 1589, 1374, 357, 579], # Los Angeles
[ 713, 1745, 0, 355, 920, 803, 1737, 851, 1858, 262, 940, 1453, 1260], # Chicago
[1018, 1524, 355, 0, 700, 862, 1395, 1123, 1584, 466, 1056, 1280, 987], # Minneapolis
[1631, 831, 920, 700, 0, 663, 1021, 1769, 949, 796, 879, 586, 371], # Denver
[1374, 1240, 803, 862, 663, 0, 1681, 1551, 1765, 547, 225, 887, 999], # Dallas
[2408, 959, 1737, 1395, 1021, 1681, 0, 2493, 678, 1724, 1891, 1114, 701], # Seattle
[ 213, 2596, 851, 1123, 1769, 1551, 2493, 0, 2699, 1038, 1605, 2300, 2099], # Boston
[2571, 403, 1858, 1584, 949, 1765, 678, 2699, 0, 1744, 1645, 653, 600], # San Francisco
[ 875, 1589, 262, 466, 796, 547, 1724, 1038, 1744, 0, 679, 1272, 1162], # St. Louis
[1420, 1374, 940, 1056, 879, 225, 1891, 1605, 1645, 679, 0, 1017, 1200], # Houston
[2145, 357, 1453, 1280, 586, 887, 1114, 2300, 653, 1272, 1017, 0, 504], # Phoenix
[1972, 579, 1260, 987, 371, 999, 701, 2099, 600, 1162, 1200, 504, 0]] # Salt Lake City
def Distance(self, from_node, to_node):
return int(self.matrix[from_node][to_node])
def main():
# Cities
city_names = ["New York", "Los Angeles", "Chicago", "Minneapolis", "Denver", "Dallas", "Seattle",
"Boston", "San Francisco", "St. Louis", "Houston", "Phoenix", "Salt Lake City"]
tsp_size = len(city_names)
num_routes = 1 # The number of routes, which is 1 in the TSP.
# Nodes are indexed from 0 to tsp_size - 1. The depot is the starting node of the route.
depot = 0
# Create routing model
if tsp_size > 0:
routing = pywrapcp.RoutingModel(tsp_size, num_routes, depot)
search_parameters = pywrapcp.RoutingModel.DefaultModelParameters()
# Create the distance callback, which takes two arguments (the from and to node indices)
# and returns the distance between these nodes.
dist_between_nodes = CreateDistanceCallback()
dist_callback = dist_between_nodes.Distance
routing.SetArcCostEvaluatorOfAllVehicles(dist_callback)
# Solve, returns a solution if any.
assignment = routing.SolveWithParameters(search_parameters)
if assignment:
# Solution cost.
print("Total distance: {} miles".format(assignment.ObjectiveValue()))
# Inspect solution.
# Only one route here; otherwise iterate from 0 to routing.vehicles() - 1
route_number = 0
index = routing.Start(route_number) # Index of the variable for the starting node.
route = ''
while not routing.IsEnd(index):
# Convert variable indices to node indices in the displayed route.
route += str(city_names[routing.IndexToNode(index)]) + ' -> '
index = assignment.Value(routing.NextVar(index))
route += str(city_names[routing.IndexToNode(index)])
print("Route:\n\n{}".format(route))
else:
print('No solution found.')
else:
print('Specify an instance greater than 0.')
if __name__ == '__main__':
main()

Related

Error using RandomizedSearchCV on a LSTM model

I want to find the optimal number of neurons and layers for a LSTM model and I have created the following:
def build_model(n_hidden=1, n_neurons=30, learning_rate=3e-3, input_shape=[20,11]):
model = keras.models.Sequential()
options = {"input_shape": input_shape}
for layer in range(n_hidden):
model.add(keras.layers.Dense(n_neurons, activation="swish", **options))
options = {}
model.add(keras.layers.Dense(10, **options))
#optimizer = keras.optimizers.SGD(learning_rate)
model.compile(loss="mse", optimizer='adam')
return model
keras_reg = keras.wrappers.scikit_learn.KerasRegressor(build_model)
from scipy.stats import reciprocal
from sklearn.model_selection import RandomizedSearchCV
param_distribs = {
"n_hidden": [0, 1, 2, 3],
"n_neurons": np.arange(1, 100),
#"learning_rate": reciprocal(3e-4, 3e-2),
}
rnd_search_cv = RandomizedSearchCV(keras_reg, param_distribs, n_iter=10)
rnd_search_cv.fit(x_tr, y_tr , epochs=100,
callbacks=[keras.callbacks.EarlyStopping(patience=10)])
The dataset used is composed of 11 features and in this model it looks for the 20-in previous times steps and aims to predict the following 10 time steps. When I run the following code I have the following error:
> /home/use/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_validation.py:372: FitFailedWarning:
50 fits failed out of a total of 50.
The score on these train-test partitions for these parameters will be set to nan.
If these failures are not expected, you can try to debug them by setting error_score='raise'.
Below are more details about the failures:
> --------------------------------------------------------------------------------
1 fits failed with the following error:
Traceback (most recent call last):
File "/home/use/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_validation.py", line 680, in _fit_and_score
estimator.fit(X_train, y_train, **fit_params)
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/wrappers/scikit_learn.py", line 175, in fit
history = self.model.fit(x, y, **fit_args)
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 70, in error_handler
raise e.with_traceback(filtered_tb) from None
File "/home/use/anaconda3/lib/python3.9/site-packages/tensorflow/python/eager/execute.py", line 52, in quick_execute
tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
tensorflow.python.framework.errors_impl.InvalidArgumentError: Graph execution error:
Detected at node 'gradient_tape/mean_squared_error/BroadcastGradientArgs' defined at (most recent call last):
File "/home/use/anaconda3/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/use/anaconda3/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/use/anaconda3/lib/python3.9/site-packages/ipykernel_launcher.py", line 17, in <module>
app.launch_new_instance()
File "/home/use/anaconda3/lib/python3.9/site-packages/traitlets/config/application.py", line 846, in launch_instance
app.start()
File "/home/use/anaconda3/lib/python3.9/site-packages/ipykernel/kernelapp.py", line 712, in start
self.io_loop.start()
File "/home/use/anaconda3/lib/python3.9/site-packages/tornado/platform/asyncio.py", line 199, in start
self.asyncio_loop.run_forever()
File "/home/use/anaconda3/lib/python3.9/asyncio/base_events.py", line 601, in run_forever
self._run_once()
File "/home/use/anaconda3/lib/python3.9/asyncio/base_events.py", line 1905, in _run_once
handle._run()
File "/home/use/anaconda3/lib/python3.9/asyncio/events.py", line 80, in _run
self._context.run(self._callback, *self._args)
File "/home/use/anaconda3/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 510, in dispatch_queue
await self.process_one()
File "/home/use/anaconda3/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 499, in process_one
await dispatch(*args)
File "/home/use/anaconda3/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 406, in dispatch_shell
await result
File "/home/use/anaconda3/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 730, in execute_request
reply_content = await reply_content
File "/home/use/anaconda3/lib/python3.9/site-packages/ipykernel/ipkernel.py", line 390, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/use/anaconda3/lib/python3.9/site-packages/ipykernel/zmqshell.py", line 528, in run_cell
return super().run_cell(*args, **kwargs)
File "/home/use/anaconda3/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 2914, in run_cell
result = self._run_cell(
File "/home/use/anaconda3/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 2960, in _run_cell
return runner(coro)
File "/home/use/anaconda3/lib/python3.9/site-packages/IPython/core/async_helpers.py", line 78, in _pseudo_sync_runner
coro.send(None)
File "/home/use/anaconda3/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 3185, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "/home/use/anaconda3/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 3377, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "/home/use/anaconda3/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 3457, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "/tmp/ipykernel_10055/1251208469.py", line 9, in <module>
rnd_search_cv.fit(x_tr, y_tr , epochs=100,
File "/home/use/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_search.py", line 891, in fit
self._run_search(evaluate_candidates)
File "/home/use/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_search.py", line 1766, in _run_search
evaluate_candidates(
File "/home/use/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_search.py", line 838, in evaluate_candidates
out = parallel(
File "/home/use/anaconda3/lib/python3.9/site-packages/joblib/parallel.py", line 1043, in __call__
if self.dispatch_one_batch(iterator):
File "/home/use/anaconda3/lib/python3.9/site-packages/joblib/parallel.py", line 861, in dispatch_one_batch
self._dispatch(tasks)
File "/home/use/anaconda3/lib/python3.9/site-packages/joblib/parallel.py", line 779, in _dispatch
job = self._backend.apply_async(batch, callback=cb)
File "/home/use/anaconda3/lib/python3.9/site-packages/joblib/_parallel_backends.py", line 208, in apply_async
result = ImmediateResult(func)
File "/home/use/anaconda3/lib/python3.9/site-packages/joblib/_parallel_backends.py", line 572, in __init__
self.results = batch()
File "/home/use/anaconda3/lib/python3.9/site-packages/joblib/parallel.py", line 262, in __call__
return [func(*args, **kwargs)
File "/home/use/anaconda3/lib/python3.9/site-packages/joblib/parallel.py", line 262, in <listcomp>
return [func(*args, **kwargs)
File "/home/use/anaconda3/lib/python3.9/site-packages/sklearn/utils/fixes.py", line 216, in __call__
return self.function(*args, **kwargs)
File "/home/use/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_validation.py", line 680, in _fit_and_score
estimator.fit(X_train, y_train, **fit_params)
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/wrappers/scikit_learn.py", line 175, in fit
history = self.model.fit(x, y, **fit_args)
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 65, in error_handler
return fn(*args, **kwargs)
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/engine/training.py", line 1650, in fit
tmp_logs = self.train_function(iterator)
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/engine/training.py", line 1249, in train_function
return step_function(self, iterator)
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/engine/training.py", line 1233, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/engine/training.py", line 1222, in run_step
outputs = model.train_step(data)
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/engine/training.py", line 1027, in train_step
self.optimizer.minimize(loss, self.trainable_variables, tape=tape)
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/optimizers/optimizer_experimental/optimizer.py", line 526, in minimize
grads_and_vars = self.compute_gradients(loss, var_list, tape)
File "/home/use/anaconda3/lib/python3.9/site-packages/keras/optimizers/optimizer_experimental/optimizer.py", line 259, in compute_gradients
grads = tape.gradient(loss, var_list)
Node: 'gradient_tape/mean_squared_error/BroadcastGradientArgs'
Incompatible shapes: [32,20,10] vs. [32,10]
[[{{node gradient_tape/mean_squared_error/BroadcastGradientArgs}}]] [Op:__inference_train_function_1033752]

A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable

I am using the following code for machine learning purposes (I am also quite new to python and pytorch). Basically, I think the problem is that multitasking is not happening for some reason.
I am using code from here: https://raw.githubusercontent.com/harryhan618/LaneNet/master/demo_test.py
The purpose of the code is draw lane markings on an image.
import cv2
import torch
import os
os.environ['KMP_DUPLICATE_LIB_OK']='True'
from lane_files.model import LaneNet
from lane_files.utils.transforms import *
from lane_files.utils.postprocess import embedding_post_process
if __name__=='__main__':
net = LaneNet(pretrained=False, embed_dim=7, delta_v=.5, delta_d=3.)
transform = Compose(Resize((800, 288)), ToTensor(),
Normalize(mean=(0.3598, 0.3653, 0.3662), std=(0.2573, 0.2663, 0.2756)))
img = cv2.imread('data/train_images/frame0.png')
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # RGB for net model input
x = transform(img)[0]
x.unsqueeze_(0)
save_dict = torch.load('lane_files/experiments/exp0/exp0_best.pth', map_location='cpu')
net.load_state_dict(save_dict['net'])
net.eval()
output = net(x)
embedding = output['embedding']
embedding = embedding.detach().cpu().numpy()
embedding = np.transpose(embedding[0], (1, 2, 0))
binary_seg = output['binary_seg']
bin_seg_prob = binary_seg.detach().cpu().numpy()
bin_seg_pred = np.argmax(bin_seg_prob, axis=1)[0]
seg_img = np.zeros_like(img)
lane_seg_img = embedding_post_process(embedding, bin_seg_pred, 0.5)
color = np.array([[255, 125, 0], [0, 255, 0], [0, 0, 255], [0, 255, 255]], dtype='uint8')
for i, lane_idx in enumerate(np.unique(lane_seg_img)):
seg_img[lane_seg_img == lane_idx] = color[i]
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
img = cv2.resize(img, (800, 288))
img = cv2.addWeighted(src1=seg_img, alpha=0.8, src2=img, beta=1., gamma=0.)
cv2.imshow("", img)
cv2.waitKey(5000)
cv2.destroyAllWindows()
Expected result: An image displayed with lane markings on it
Actual result:
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:/Users/sarim/PycharmProjects/thesis/pytorch_learning.py", line 36, in <module>
lane_seg_img = embedding_post_process(embedding, bin_seg_pred, 0.5)
File "C:\Users\sarim\PycharmProjects\thesis\lane_files\utils\postprocess.py", line 29, in embedding_post_process
mean_shift.fit(embedding_reshaped)
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\sklearn\cluster\mean_shift_.py", line 424, in fit
cluster_all=self.cluster_all, n_jobs=self.n_jobs)
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\sklearn\cluster\mean_shift_.py", line 204, in mean_shift
(seed, X, nbrs, max_iter) for seed in seeds)
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\parallel.py", line 934, in __call__
self.retrieve()
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\parallel.py", line 833, in retrieve
self._output.extend(job.get(timeout=self.timeout))
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\_parallel_backends.py", line 521, in wrap_future_result
return future.result(timeout=timeout)
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\concurrent\futures\_base.py", line 435, in result
return self.__get_result()
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\concurrent\futures\_base.py", line 384, in __get_result
raise self._exception
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\externals\loky\_base.py", line 625, in _invoke_callbacks
callback(self)
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\parallel.py", line 309, in __call__
self.parallel.dispatch_next()
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\parallel.py", line 731, in dispatch_next
if not self.dispatch_one_batch(self._original_iterator):
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\parallel.py", line 759, in dispatch_one_batch
self._dispatch(tasks)
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\parallel.py", line 716, in _dispatch
job = self._backend.apply_async(batch, callback=cb)
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\_parallel_backends.py", line 510, in apply_async
future = self._workers.submit(SafeFunction(func))
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\externals\loky\reusable_executor.py", line 151, in submit
fn, *args, **kwargs)
File "C:\Users\sarim\AppData\Local\Programs\Python\Python37\lib\site-packages\joblib\externals\loky\process_executor.py", line 1022, in submit
raise self._flags.broken
joblib.externals.loky.process_executor.BrokenProcessPool: A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable.

InvalidArgumentError (see above for traceback): Input to reshape is a tensor with 35000 values, but the requested shape requires a multiple of 7500

I'm using this code for a convolutional neural network.
learning_rate = 0.001
X = tf.placeholder(tf.float32, [None, 50*50])
X_img = tf.reshape(X, [-1, 50, 50, 3])
Y = tf.placeholder(tf.float32, [None, 26])
W1 = tf.Variable(tf.random_normal([5, 5, 3, 32]))
L1 = tf.nn.conv2d(X_img, W1, strides=[1, 1, 1, 1], padding='SAME')
L1 = tf.nn.relu(L1)
L1 = tf.nn.max_pool(L1, ksize=[1, 5, 5, 1], strides=[1, 5, 5, 1], padding='SAME')
W2 = tf.Variable(tf.random_normal([5, 5, 32, 64]))
L2 = tf.nn.conv2d(L1, W2, strides=[1, 1, 1, 1], padding='SAME')
L2 = tf.nn.relu(L2)
L2 = tf.nn.max_pool(L2, ksize=[1, 5, 5, 1], strides=[1, 5, 5, 1], padding='SAME')
L2_flat = tf.reshape(L2, [-1, 5*5*64])
W3 = tf.get_variable("W3", shape=[5*5*64, 26], initializer=tf.contrib.layers.xavier_initializer())
print(W3)
b = tf.Variable(tf.random_normal([26]))
logits = tf.matmul(L2_flat, W3) + b
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=Y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
training_epochs = 15
batch_size = 14
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print('Learning started. It takes sometime.')
for epoch in range(training_epochs):
avg_cost = 0
total_batch = int(len(train_input) / batch_size)
for i in range(total_batch):
start = ((i+1) * batch_size) - batch_size
end = ((i+1) * batch_size)
batch_xs = train_input[start:end]
batch_ys = train_label[start:end]
feed_dict = {X: batch_xs, Y: batch_ys}
c, _ = sess.run([cost, optimizer], feed_dict=feed_dict)
avg_cost += c / total_batch
print('Epoch:', '%04d' % (epoch +1), 'cost = ', '{:.9f}'.format(avg_cost))
print('Learning Finished')
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print('Accuracy:', sess.run(accuracy, feed_dict={
X: test_input, Y: test_label}))
I'm reading 50x50 RGB images. I have 406 pictures and 26 labels.
I'm using 5x5 filter and my train inputs shape appear
print(train_input.shape) (1218, 2500)
I don't understand why '1218' appear and what this number mean.
And also don't understand the following errors.
InvalidArgumentError Traceback (most recent call last)
~\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1321 try:
-> 1322 return fn(*args)
1323 except errors.OpError as e:
~\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1306 return self._call_tf_sessionrun(
-> 1307 options, feed_dict, fetch_list, target_list, run_metadata)
1308
~\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1408 self._session, options, feed_dict, fetch_list, target_list,
-> 1409 run_metadata)
1410 else:
InvalidArgumentError: Input to reshape is a tensor with 35000 values, but the requested shape requires a multiple of 7500
[[Node: Reshape = Reshape[T=DT_FLOAT, Tshape=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](_arg_Placeholder_0_0, Reshape/shape)]]
During handling of the above exception, another exception occurred:
InvalidArgumentError Traceback (most recent call last)
<ipython-input-12-e97c33c78b58> in <module>()
16 batch_ys = train_label[start:end]
17 feed_dict = {X: batch_xs, Y: batch_ys}
---> 18 c, _ = sess.run([cost, optimizer], feed_dict=feed_dict)
19 avg_cost += c / total_batch
20 print('Epoch:', '%04d' % (epoch +1), 'cost = ', '{:.9f}'.format(avg_cost))
~\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in run(self, fetches, feed_dict, options, run_metadata)
898 try:
899 result = self._run(None, fetches, feed_dict, options_ptr,
--> 900 run_metadata_ptr)
901 if run_metadata:
902 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
~\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1133 if final_fetches or final_targets or (handle and feed_dict_tensor):
1134 results = self._do_run(handle, final_targets, final_fetches,
-> 1135 feed_dict_tensor, options, run_metadata)
1136 else:
1137 results = []
~\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1314 if handle is None:
1315 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1316 run_metadata)
1317 else:
1318 return self._do_call(_prun_fn, handle, feeds, fetches)
~\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1333 except KeyError:
1334 pass
-> 1335 raise type(e)(node_def, op, message)
1336
1337 def _extend_graph(self):
InvalidArgumentError: Input to reshape is a tensor with 35000 values, but the requested shape requires a multiple of 7500
[[Node: Reshape = Reshape[T=DT_FLOAT, Tshape=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](_arg_Placeholder_0_0, Reshape/shape)]]
Caused by op 'Reshape', defined at:
File "C:\Users\sunghee hong\Anaconda3\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:\Users\sunghee hong\Anaconda3\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\traitlets\config\application.py", line 658, in launch_instance
app.start()
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 486, in start
self.io_loop.start()
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\tornado\platform\asyncio.py", line 127, in start
self.asyncio_loop.run_forever()
File "C:\Users\sunghee hong\Anaconda3\lib\asyncio\base_events.py", line 422, in run_forever
self._run_once()
File "C:\Users\sunghee hong\Anaconda3\lib\asyncio\base_events.py", line 1432, in _run_once
handle._run()
File "C:\Users\sunghee hong\Anaconda3\lib\asyncio\events.py", line 145, in _run
self._callback(*self._args)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\tornado\platform\asyncio.py", line 117, in _handle_events
handler_func(fileobj, events)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\tornado\stack_context.py", line 276, in null_wrapper
return fn(*args, **kwargs)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 450, in _handle_events
self._handle_recv()
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 480, in _handle_recv
self._run_callback(callback, msg)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 432, in _run_callback
callback(*args, **kwargs)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\tornado\stack_context.py", line 276, in null_wrapper
return fn(*args, **kwargs)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2903, in run_ast_nodes
if self.run_code(code, result):
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-11-02a422681da1>", line 4, in <module>
X_img = tf.reshape(X, [-1, 50, 50, 3]) #[batch, width, height, image channel(RGB:3, GRAY:1)], batch size는 가변할 수 있어서 대부분 -1
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_array_ops.py", line 6112, in reshape
"Reshape", tensor=tensor, shape=shape, name=name)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\Users\sunghee hong\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 1718, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): Input to reshape is a tensor with 35000 values, but the requested shape requires a multiple of 7500
[[Node: Reshape = Reshape[T=DT_FLOAT, Tshape=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](_arg_Placeholder_0_0, Reshape/shape)]]
​
How can I solve this problem?
Thank you.
Your question 1.
print(train_input.shape) (1218, 2500)
The 1218 value comes from X = tf.placeholder(tf.float32, [None, 50*50]) and you are feeding in 406 50x50 RGB images(3 channels).
Or in other words, by this line of code, you are requesting input tensor of size [406, 3, 50, 50] -> [?, 2500]. So you get 1218 = 406*3.
Your question 2.
It's kind of weird for your input tensor to have 35000 values, please have another check around the line X_img = tf.reshape(X, [-1, 50, 50, 3]) for the size of X

Could not set Native Handle Error Tensorflow

I'm in the middle of training my net on tensorflow and I get this error at some part of the training process, though I can not tell exactly what is causing it because it will always happen at a seemingly random training sample at a more or less random epoch of training. I'm at my wits end with this and would like it if someone could shed light on it below.
Traceback (most recent call last):
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1361, in _do_call
return fn(*args)
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1340, in _run_fn
target_list, status, run_metadata)
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py", line 516, in __exit__
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.AbortedError: Operation received an exception:Status: 3, message: could not set native handle, in file tensorflow/core/kernels/mkl_conv_ops.cc:652
[[Node: SRGAN_g/Conv4_4/Conv2D = _MklConv2D[T=DT_FLOAT, _kernel="MklOp", data_format="NHWC", padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](SRGAN_g/pixelshufflerx2/1/4/Relu, SRGAN_g/Conv4/W_conv2d/read, SRGAN_g/pixelshufflerx2/1/4/Relu:1, DMT/_132)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 289, in <module>
train()
File "main.py", line 149, in train
errM, _ = sess.run([mse_loss, g_optim_init], {t_image: b_imgs_96, t_target_image: b_imgs_384})
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 905, in run
run_metadata_ptr)
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1137, in _run
feed_dict_tensor, options, run_metadata)
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1355, in _do_run
options, run_metadata)
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1374, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.AbortedError: Operation received an exception:Status: 3, message: could not set native handle, in file tensorflow/core/kernels/mkl_conv_ops.cc:652
[[Node: SRGAN_g/Conv4_4/Conv2D = _MklConv2D[T=DT_FLOAT, _kernel="MklOp", data_format="NHWC", padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](SRGAN_g/pixelshufflerx2/1/4/Relu, SRGAN_g/Conv4/W_conv2d/read, SRGAN_g/pixelshufflerx2/1/4/Relu:1, DMT/_132)]]
Caused by op 'SRGAN_g/Conv4_4/Conv2D', defined at:
File "main.py", line 289, in <module>
train()
File "main.py", line 53, in train
net_g = SRGAN_g(t_image, is_train=True, reuse=False)
File "/global/home/hpc4142/srgan-master_TF/model.py", line 51, in SRGAN_g
recursive_layers[i] = Conv2dReuse(recursive_layers[i], 256, (3, 3), (1, 1), act=None, padding='SAME', W_init=w_init, name='n256s1/2/%s' %i, group = "Conv4")
File "/global/home/hpc4142/srgan-master_TF/nu_layers.py", line 461, in Conv2dReuse
group = group)
File "/global/home/hpc4142/srgan-master_TF/nu_layers.py", line 270, in __init__
self.outputs = act( tf.nn.conv2d(self.inputs, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) + b )
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 631, in conv2d
data_format=data_format, dilations=dilations, name=name)
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 3271, in create_op
op_def=op_def)
File "/global/home/hpc4142/.local/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 1650, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
AbortedError (see above for traceback): Operation received an exception:Status: 3, message: could not set native handle, in file tensorflow/core/kernels/mkl_conv_ops.cc:652
[[Node: SRGAN_g/Conv4_4/Conv2D = _MklConv2D[T=DT_FLOAT, _kernel="MklOp", data_format="NHWC", padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](SRGAN_g/pixelshufflerx2/1/4/Relu, SRGAN_g/Conv4/W_conv2d/read, SRGAN_g/pixelshufflerx2/1/4/Relu:1, DMT/_132)]]

Logits and Labels mismatch in Tensorflow

there is a mismatch between the logits and lables in the Tensorflow after one hot encoding.
And my batch size is 256. How can I manage to get the batch size in labels Tensor too ? I guess this issue is related to the LabelEncoder and one-hot encoder. Any help is appreciable.
Please find the code below.
from sklearn import preprocessing
le = preprocessing.LabelEncoder()
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = tf.one_hot(le.fit_transform(labels), n_classes)))
optimizer = tf.train.GradientDescentOptimizer(learning_rate = learn_rate).minimize(cost)
correct_prediction = tf.equal(tf.argmax(logits,1), tf.argmax(tf.one_hot(le.fit_transform(labels), n_classes),1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
batchSize = 256
epochs = 20 # 200epoch+.5lr = 99.6
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
total_batches = batches(batchSize, train_features, train_labels)
for epoch in range(epochs):
for batch_features, batch_labels in total_batches:
train_data = {features: batch_features, labels : batch_labels, keep_prob : 0.5}
sess.run(optimizer, feed_dict = train_data)
# Print status for every 100 epochs
if epoch % 10 == 0:
valid_accuracy = sess.run(
accuracy,
feed_dict={
features: val_features,
labels: val_labels,
keep_prob : 0.5})
print('Epoch {:<3} - Validation Accuracy: {}'.format(
epoch,
valid_accuracy))
Accuracy = sess.run(accuracy, feed_dict={features : test_features, labels :test_labels, keep_prob : 1.0})
# Save the model
saver.save(sess, save_file)
print('Trained Model Saved.')
prediction=tf.argmax(logits,1)
output_array = le.inverse_transform(prediction.eval(feed_dict={features : test_features, keep_prob: 1.0}))
prediction = np.reshape(prediction, (test_features.shape[0],1))
np.savetxt("prediction.csv", prediction, delimiter=",")
And I am getting the Invalid Argument Error as indicated below.
InvalidArgumentError: logits and labels must be same size: logits_size=[256,1161] labels_size=[1,1161]
[[Node: SoftmaxCrossEntropyWithLogits = SoftmaxCrossEntropyWithLogits[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](Reshape, Reshape_1)]]
Caused by op 'SoftmaxCrossEntropyWithLogits', defined at:
File "C:\Anaconda\envs\gpu\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:\Anaconda\envs\gpu\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\__main__.py", line 3, in <module>
app.launch_new_instance()
File "C:\Anaconda\envs\gpu\lib\site-packages\traitlets\config\application.py", line 658, in launch_instance
app.start()
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\kernelapp.py", line 477, in start
ioloop.IOLoop.instance().start()
File "C:\Anaconda\envs\gpu\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "C:\Anaconda\envs\gpu\lib\site-packages\tornado\ioloop.py", line 888, in start
handler_func(fd_obj, events)
File "C:\Anaconda\envs\gpu\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\Anaconda\envs\gpu\lib\site-packages\zmq\eventloop\zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "C:\Anaconda\envs\gpu\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "C:\Anaconda\envs\gpu\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "C:\Anaconda\envs\gpu\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\kernelbase.py", line 235, in dispatch_shell
handler(stream, idents, msg)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\Anaconda\envs\gpu\lib\site-packages\ipykernel\zmqshell.py", line 533, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\Anaconda\envs\gpu\lib\site-packages\IPython\core\interactiveshell.py", line 2698, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\Anaconda\envs\gpu\lib\site-packages\IPython\core\interactiveshell.py", line 2802, in run_ast_nodes
if self.run_code(code, result):
File "C:\Anaconda\envs\gpu\lib\site-packages\IPython\core\interactiveshell.py", line 2862, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-5-9a6fe2134e3e>", line 52, in <module>
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = tf.one_hot(le.fit_transform(labels), n_classes)))
File "C:\Anaconda\envs\gpu\lib\site-packages\tensorflow\python\ops\nn_ops.py", line 1594, in softmax_cross_entropy_with_logits
precise_logits, labels, name=name)
File "C:\Anaconda\envs\gpu\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 2380, in _softmax_cross_entropy_with_logits
features=features, labels=labels, name=name)
File "C:\Anaconda\envs\gpu\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 767, in apply_op
op_def=op_def)
File "C:\Anaconda\envs\gpu\lib\site-packages\tensorflow\python\framework\ops.py", line 2506, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\Anaconda\envs\gpu\lib\site-packages\tensorflow\python\framework\ops.py", line 1269, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): logits and labels must be same size: logits_size=[256,1161] labels_size=[1,1161]
[[Node: SoftmaxCrossEntropyWithLogits = SoftmaxCrossEntropyWithLogits[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](Reshape, Reshape_1)]]
Issue was with tf.one_hot(le.fit_transform(labels), n_classes).
This passes a tensor where the numpy array was needed. After calling eval() for this Tensor, Issue is resolved.

Resources