Related
I am trying to run GradientExplainer method of SHAP on a trained Inception V3 model. I could do it without a problem on a VGG16 and MobileNet. The difference between Inception and those two models is the input dimension shape being 299 instead of 224.
But when I try SHAP on Inception I receive a long, confusing error that I cannot figure out and wasn't able to find an answer for here or elsewhere. Here is the error. Has someone here been able to visualize Inception V3 results using SHAP's GradientExplainer?
'''
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1376 try:
-> 1377 return fn(*args)
1378 except errors.OpError as e:
9 frames
InvalidArgumentError: 2 root error(s) found.
(0) INVALID_ARGUMENT: You must feed a value for placeholder tensor 'input_1' with dtype float and shape [?,299,299,3]
[[{{node input_1}}]]
[[dense_2/Softmax/_34069]]
(1) INVALID_ARGUMENT: You must feed a value for placeholder tensor 'input_1' with dtype float and shape [?,299,299,3]
[[{{node input_1}}]]
0 successful operations.
0 derived errors ignored.
During handling of the above exception, another exception occurred:
InvalidArgumentError Traceback (most recent call last)
/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1394 '\nsession_config.graph_options.rewrite_options.'
1395 'disable_meta_optimizer = True')
-> 1396 raise type(e)(node_def, op, message) # pylint: disable=no-value-for-parameter
1397
1398 def _extend_graph(self):
InvalidArgumentError: Graph execution error:
Detected at node 'input_1' defined at (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.8/dist-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/usr/local/lib/python3.8/dist-packages/traitlets/config/application.py", line 992, in launch_instance
app.start()
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelapp.py", line 612, in start
self.io_loop.start()
File "/usr/local/lib/python3.8/dist-packages/tornado/platform/asyncio.py", line 149, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.8/asyncio/base_events.py", line 570, in run_forever
self._run_once()
File "/usr/lib/python3.8/asyncio/base_events.py", line 1859, in _run_once
handle._run()
File "/usr/lib/python3.8/asyncio/events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.8/dist-packages/tornado/ioloop.py", line 690, in <lambda>
lambda f: self._run_callback(functools.partial(callback, future))
File "/usr/local/lib/python3.8/dist-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 787, in inner
self.run()
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 748, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 365, in process_one
yield gen.maybe_future(dispatch(*args))
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 268, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 543, in execute_request
self.do_execute(
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/ipkernel.py", line 306, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/zmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 2854, in run_cell
result = self._run_cell(
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 2881, in _run_cell
return runner(coro)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/async_helpers.py", line 68, in _pseudo_sync_runner
coro.send(None)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3057, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3249, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3326, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-28-2966d0b0f166>", line 2, in <module>
model = tf.keras.models.load_model(f'/content/drive/MyDrive/Colab Notebooks/models/{model_name}')
File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/save.py", line 209, in load_model
return saved_model_load.load(filepath_str, compile, options)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 135, in load
keras_loader.load_layers(compile=compile)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 399, in load_layers
self.loaded_nodes[node_metadata.node_id] = self._load_layer(
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 441, in _load_layer
obj, setter = self._revive_from_config(identifier, metadata, node_id)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 459, in _revive_from_config
self._revive_layer_or_model_from_config(metadata, node_id))
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 523, in _revive_layer_or_model_from_config
obj = layers_module.deserialize(
File "/usr/local/lib/python3.8/dist-packages/keras/layers/serialization.py", line 205, in deserialize
return generic_utils.deserialize_keras_object(
File "/usr/local/lib/python3.8/dist-packages/keras/utils/generic_utils.py", line 686, in deserialize_keras_object
deserialized_obj = cls.from_config(cls_config)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer_v1.py", line 515, in from_config
return cls(**config)
File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/input_layer.py", line 200, in __init__
input_tensor = backend.placeholder(
File "/usr/local/lib/python3.8/dist-packages/keras/backend.py", line 1386, in placeholder
x = tf.compat.v1.placeholder(dtype, shape=shape, name=name)
Node: 'input_1'
Detected at node 'input_1' defined at (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.8/dist-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/usr/local/lib/python3.8/dist-packages/traitlets/config/application.py", line 992, in launch_instance
app.start()
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelapp.py", line 612, in start
self.io_loop.start()
File "/usr/local/lib/python3.8/dist-packages/tornado/platform/asyncio.py", line 149, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.8/asyncio/base_events.py", line 570, in run_forever
self._run_once()
File "/usr/lib/python3.8/asyncio/base_events.py", line 1859, in _run_once
handle._run()
File "/usr/lib/python3.8/asyncio/events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.8/dist-packages/tornado/ioloop.py", line 690, in <lambda>
lambda f: self._run_callback(functools.partial(callback, future))
File "/usr/local/lib/python3.8/dist-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 787, in inner
self.run()
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 748, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 365, in process_one
yield gen.maybe_future(dispatch(*args))
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 268, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 543, in execute_request
self.do_execute(
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/ipkernel.py", line 306, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/zmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 2854, in run_cell
result = self._run_cell(
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 2881, in _run_cell
return runner(coro)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/async_helpers.py", line 68, in _pseudo_sync_runner
coro.send(None)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3057, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3249, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3326, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-28-2966d0b0f166>", line 2, in <module>
model = tf.keras.models.load_model(f'/content/drive/MyDrive/Colab Notebooks/models/{model_name}')
File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/save.py", line 209, in load_model
return saved_model_load.load(filepath_str, compile, options)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 135, in load
keras_loader.load_layers(compile=compile)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 399, in load_layers
self.loaded_nodes[node_metadata.node_id] = self._load_layer(
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 441, in _load_layer
obj, setter = self._revive_from_config(identifier, metadata, node_id)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 459, in _revive_from_config
self._revive_layer_or_model_from_config(metadata, node_id))
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 523, in _revive_layer_or_model_from_config
obj = layers_module.deserialize(
File "/usr/local/lib/python3.8/dist-packages/keras/layers/serialization.py", line 205, in deserialize
return generic_utils.deserialize_keras_object(
File "/usr/local/lib/python3.8/dist-packages/keras/utils/generic_utils.py", line 686, in deserialize_keras_object
deserialized_obj = cls.from_config(cls_config)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer_v1.py", line 515, in from_config
return cls(**config)
File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/input_layer.py", line 200, in __init__
input_tensor = backend.placeholder(
File "/usr/local/lib/python3.8/dist-packages/keras/backend.py", line 1386, in placeholder
x = tf.compat.v1.placeholder(dtype, shape=shape, name=name)
Node: 'input_1'
2 root error(s) found.
(0) INVALID_ARGUMENT: You must feed a value for placeholder tensor 'input_1' with dtype float and shape [?,299,299,3]
[[{{node input_1}}]]
[[dense_2/Softmax/_34069]]
(1) INVALID_ARGUMENT: You must feed a value for placeholder tensor 'input_1' with dtype float and shape [?,299,299,3]
[[{{node input_1}}]]
0 successful operations.
0 derived errors ignored.
Original stack trace for 'input_1':
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.8/dist-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/usr/local/lib/python3.8/dist-packages/traitlets/config/application.py", line 992, in launch_instance
app.start()
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelapp.py", line 612, in start
self.io_loop.start()
File "/usr/local/lib/python3.8/dist-packages/tornado/platform/asyncio.py", line 149, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.8/asyncio/base_events.py", line 570, in run_forever
self._run_once()
File "/usr/lib/python3.8/asyncio/base_events.py", line 1859, in _run_once
handle._run()
File "/usr/lib/python3.8/asyncio/events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.8/dist-packages/tornado/ioloop.py", line 690, in <lambda>
lambda f: self._run_callback(functools.partial(callback, future))
File "/usr/local/lib/python3.8/dist-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 787, in inner
self.run()
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 748, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 365, in process_one
yield gen.maybe_future(dispatch(*args))
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 268, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 543, in execute_request
self.do_execute(
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/ipkernel.py", line 306, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/zmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 2854, in run_cell
result = self._run_cell(
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 2881, in _run_cell
return runner(coro)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/async_helpers.py", line 68, in _pseudo_sync_runner
coro.send(None)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3057, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3249, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3326, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-28-2966d0b0f166>", line 2, in <module>
model = tf.keras.models.load_model(f'/content/drive/MyDrive/Colab Notebooks/models/{model_name}')
File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/save.py", line 209, in load_model
return saved_model_load.load(filepath_str, compile, options)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 135, in load
keras_loader.load_layers(compile=compile)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 399, in load_layers
self.loaded_nodes[node_metadata.node_id] = self._load_layer(
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 441, in _load_layer
obj, setter = self._revive_from_config(identifier, metadata, node_id)
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 459, in _revive_from_config
self._revive_layer_or_model_from_config(metadata, node_id))
File "/usr/local/lib/python3.8/dist-packages/keras/saving/saved_model/load.py", line 523, in _revive_layer_or_model_from_config
obj = layers_module.deserialize(
File "/usr/local/lib/python3.8/dist-packages/keras/layers/serialization.py", line 205, in deserialize
return generic_utils.deserialize_keras_object(
File "/usr/local/lib/python3.8/dist-packages/keras/utils/generic_utils.py", line 686, in deserialize_keras_object
deserialized_obj = cls.from_config(cls_config)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer_v1.py", line 515, in from_config
return cls(**config)
File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/input_layer.py", line 200, in __init__
input_tensor = backend.placeholder(
File "/usr/local/lib/python3.8/dist-packages/keras/backend.py", line 1386, in placeholder
x = tf.compat.v1.placeholder(dtype, shape=shape, name=name)
File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/array_ops.py", line 3346, in placeholder
return gen_array_ops.placeholder(dtype=dtype, shape=shape, name=name)
File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_array_ops.py", line 6893, in placeholder
_, _, _op, _outputs = _op_def_library._apply_op_helper(
File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 797, in _apply_op_helper
op = g._create_op_internal(op_type_name, inputs, dtypes=None,
File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3754, in _create_op_internal
ret = Operation(
File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 2133, in __init__
self._traceback = tf_stack.extract_stack_for_node(self._c_op)
'''
Here is the error log that I get when I click on Kernel Error:
Traceback (most recent call last):
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/tornado/web.py", line 1699, in
_execute result = await result
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/tornado/gen.py", line 742, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-
packages/notebook/services/sessions/handlers.py", line 72, in post type=mtype))
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/tornado/gen.py", line 735, in run
value = future.result()
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/tornado/gen.py", line 742, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site
packages/notebook/services/sessions/sessionmanager.py", line 88, in create_session
kernel_id = yield self.start_kernel_for_session(session_id, path, name, type, kernel_name)
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/tornado/gen.py", line 735, in run
value = future.result()
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/tornado/gen.py", line 742, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-
packages/notebook/services/sessions/sessionmanager.py", line 101, in start_kernel_for_session
self.kernel_manager.start_kernel(path=kernel_path, kernel_name=kernel_name)
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/tornado/gen.py", line 735, in run
value = future.result()
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-
packages/notebook/services/kernels/kernelmanager.py", line 168, in start_kernel
super(MappingKernelManager, self).start_kernel(**kwargs)
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-
packages/jupyter_client/multikernelmanager.py", line 110, in start_kernel
km.start_kernel(**kwargs)
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/jupyter_client/manager.py", line
259, in start_kernel**kw)
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/jupyter_client/manager.py", line 204,
in _launch_kernel return launch_kernel(kernel_cmd, **kw)
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/site-packages/jupyter_client/launcher.py", line
138, in launch_kernel proc = Popen(cmd, **kwargs)
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/subprocess.py", line 800, in __init__
restore_signals, start_new_session)
File "/Users/Brahmbha/opt/anaconda3/lib/python3.7/subprocess.py", line 1551, in _execute_child
raise child_exception_type(errno_num, err_msg, err_filename)
FileNotFoundError: [Errno 2] No such file or directory:
'/Users/Brahmbha/anaconda2/envs/ipykernel_py3/bin/python':
'/Users/Brahmbha/anaconda2/envs/ipykernel_py3/bin/python'
It looks like your jupyter notebook could be trying to use a python enviroment that has been moved or deleted.
Try this in cmd or anaconda:
python -m ipykernel install --user
See this thread for more:
https://github.com/jupyter/notebook/issues/4079
I have set up Airflow on an AWS EC2 server with Ubuntu 18.04, Python 3.6.9. The DB backend is a 5.7.26 MySQL, I am using a LocalExecutor. The setup is nothing more than:
apt-get install python3 python3-pip python3-venv libmysqlclient-dev
I install Airflow with pip3 install apache-airflow[mysql]==1.10.9, init DB connection, start the webserver and it works both normally and with -D. The scheduler, however, works only when run in the foreground. Trying to run it as a daemon fails with the following trace:
Traceback (most recent call last):
File "/usr/local/bin/airflow", line 37, in <module> args.func(args)
File "/usr/local/lib/python3.6/dist-packages/airflow/utils/cli.py", line 75, in wrapper return f(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/bin/cli.py", line 1032, in schedulerjob.run()
File "/usr/local/lib/python3.6/dist-packages/airflow/jobs/base_job.py", line 215, in run session.commit()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 1036, in commit self.transaction.commit()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 503, in commitself._prepare_impl()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 482, in _prepare_impl self.session.flush()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 2496, in flushself._flush(objects)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 2637, in _flush transaction.rollback(_capture_exception=True)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/util/langhelpers.py", line 69, in __exit__exc_value, with_traceback=exc_tb,
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/util/compat.py", line 178, in raise_raise exception
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 2597, in _flush flush_context.execute()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute rec.execute(self)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute uow,
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/persistence.py", line 213, in save_obj) in _organize_states_for_save(base_mapper, states, uowtransaction):
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/persistence.py", line 374, in _organize_states_for_save base_mapper, uowtransaction, states
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/persistence.py", line 1602, in _connections_for_states connection = uowtransaction.transaction.connection(base_mapper)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 313, in connection return self._connection_for_bind(bind, execution_options)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 420, in _connection_for_bind conn = self._parent._connection_for_bind(bind, execution_options)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 432, in _connection_for_bind conn = bind._contextual_connect()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 2251, in _contextual_connect self._wrap_pool_connect(self.pool.connect, None),
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 2285, in _wrap_pool_connect return fn()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 363, in connect return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 804, in _checkout result = pool._dialect.do_ping(fairy.connection)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 138, in do_pingdbapi_connection.ping(False)
File "/usr/local/lib/python3.6/dist-packages/pymysql/connections.py", line 546, in ping self._execute_command(COMMAND.COM_PING, "")
File "/usr/local/lib/python3.6/dist-packages/pymysql/connections.py", line 771, in _execute_command self._write_bytes(packet)
File "/usr/local/lib/python3.6/dist-packages/pymysql/connections.py", line 711, in _write_bytesself._sock.settimeout(self._write_timeout)
OSError: [Errno 9] Bad file descriptor
Exception ignored in: <function _ConnectionRecord.checkout.<locals>.<lambda> at 0x7f2eb025e268>
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 503, in <lambda>
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 702, in _finalize_fairy
File "/usr/lib/python3.6/logging/__init__.py", line 1337, in error
File "/usr/lib/python3.6/logging/__init__.py", line 1444, in _log
File "/usr/lib/python3.6/logging/__init__.py", line 1454, in handle
File "/usr/lib/python3.6/logging/__init__.py", line 1516, in callHandlers
File "/usr/lib/python3.6/logging/__init__.py", line 865, in handle
File "/usr/lib/python3.6/logging/__init__.py", line 1071, in emit
File "/usr/lib/python3.6/logging/__init__.py", line 1061, in _open
NameError: name 'open' is not defined
I am trying to set up a server for semi-production purposes, so this is actually a blocker for me. I would be grateful for any advice.
EDIT
I tried using mysql-connector-python==8.0.18 and the scheduler did run as a daemon, but an attempt to open a dag failed with the following trace:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python3.6/dist-packages/flask/_compat.py", line 39, in reraise
raise value
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python3.6/dist-packages/flask_admin/base.py", line 69, in inner
return self._run_view(f, *args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/flask_admin/base.py", line 368, in _run_view
return fn(self, *args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/flask_login/utils.py", line 258, in decorated_view
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/www/utils.py", line 384, in view_func
return f(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/www/utils.py", line 290, in wrapper
return f(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/utils/db.py", line 74, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/www/views.py", line 1559, in tree
start_date=min_date, end_date=base_date, session=session)
File "/usr/local/lib/python3.6/dist-packages/airflow/utils/db.py", line 70, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/models/dag.py", line 837, in get_task_instances
tis = tis.order_by(TaskInstance.execution_date).all()
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3233, in all
return list(self)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3389, in __iter__
return self._execute_and_instances(context)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3414, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 982, in execute
return meth(self, multiparams, params)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 293, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1101, in _execute_clauseelement
distilled_params,
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
e, statement, parameters, cursor, context
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1478, in _handle_dbapi_exception
util.reraise(*exc_info)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 588, in do_execute
cursor.execute(statement, parameters)
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor_cext.py", line 272, in execute
self._handle_result(result)
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor_cext.py", line 163, in _handle_result
self._handle_resultset()
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor_cext.py", line 651, in _handle_resultset
self._rows = self._cnx.get_rows()[0]
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/connection_cext.py", line 301, in get_rows
else self._cmysql.fetch_row()
SystemError: <built-in method fetch_row of _mysql_connector.MySQL object at 0x3cdaa10> returned a result with an error set
EDIT 2
What finally did work was using mysqlclient by specifying the SQLAlchemy connection string to begin with mysql+mysqldb. I am not giving this as an answer, because the initial problem persists.
I'm on Manjaro Linux KDE and every time I try to install a module with pip it gives me this error:
Defaulting to user installation because normal site-packages is not writeable
ERROR:
Exception:
Traceback (most recent call last):
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/base_command.py", line 186, in _main
status = self.run(options, args)
File "/usr/lib/python3.8/site-packages/pip/_internal/commands/install.py", line 279, in run
session = self.get_default_session(options)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 74, in get_default_session
self._session = self.enter_context(self._build_session(options))
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 84, in _build_session
session = PipSession(
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 241, in __init__
self.headers["User-Agent"] = user_agent()
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 132, in user_agent
zip(["name", "version", "id"], distro.linux_distribution()),
File "/usr/lib/python3.8/site-packages/distro.py", line 122, in linux_distribution
return _distro.linux_distribution(full_distribution_name)
File "/usr/lib/python3.8/site-packages/distro.py", line 677, in linux_distribution
self.version(),
File "/usr/lib/python3.8/site-packages/distro.py", line 737, in version
self.lsb_release_attr('release'),
File "/usr/lib/python3.8/site-packages/distro.py", line 899, in lsb_release_attr
return self._lsb_release_info.get(attribute, '')
File "/usr/lib/python3.8/site-packages/distro.py", line 552, in __get__
ret = obj.__dict__[self._fname] = self._f(obj)
File "/usr/lib/python3.8/site-packages/distro.py", line 1012, in _lsb_release_info
stdout = subprocess.check_output(cmd, stderr=devnull)
File "/usr/lib/python3.8/subprocess.py", line 411, in check_output
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 145, in __new__
process = cls.create_process(command, stdin, cwd=cwd, env=env, shell=shell)
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 121, in create_process
shlex.split(command),
File "/usr/lib/python3.8/shlex.py", line 311, in split
return list(lex)
File "/usr/lib/python3.8/shlex.py", line 300, in __next__
token = self.get_token()
File "/usr/lib/python3.8/shlex.py", line 109, in get_token
raw = self.read_token()
File "/usr/lib/python3.8/shlex.py", line 140, in read_token
nextchar = self.instream.read(1)
AttributeError: 'tuple' object has no attribute 'read'
Traceback (most recent call last):
File "/usr/bin/pip3", line 11, in <module>
load_entry_point('pip==20.0.2', 'console_scripts', 'pip3')()
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/main.py", line 75, in main
return command.main(cmd_args)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/base_command.py", line 105, in main
return self._main(args)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/base_command.py", line 224, in _main
self.handle_pip_version_check(options)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 143, in handle_pip_version_check
session = self._build_session(
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 84, in _build_session
session = PipSession(
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 241, in __init__
self.headers["User-Agent"] = user_agent()
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 132, in user_agent
zip(["name", "version", "id"], distro.linux_distribution()),
File "/usr/lib/python3.8/site-packages/distro.py", line 122, in linux_distribution
return _distro.linux_distribution(full_distribution_name)
File "/usr/lib/python3.8/site-packages/distro.py", line 677, in linux_distribution
self.version(),
File "/usr/lib/python3.8/site-packages/distro.py", line 737, in version
self.lsb_release_attr('release'),
File "/usr/lib/python3.8/site-packages/distro.py", line 899, in lsb_release_attr
return self._lsb_release_info.get(attribute, '')
File "/usr/lib/python3.8/site-packages/distro.py", line 552, in __get__
ret = obj.__dict__[self._fname] = self._f(obj)
File "/usr/lib/python3.8/site-packages/distro.py", line 1012, in _lsb_release_info
stdout = subprocess.check_output(cmd, stderr=devnull)
File "/usr/lib/python3.8/subprocess.py", line 411, in check_output
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 145, in __new__
process = cls.create_process(command, stdin, cwd=cwd, env=env, shell=shell)
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 121, in create_process
shlex.split(command),
File "/usr/lib/python3.8/shlex.py", line 311, in split
return list(lex)
File "/usr/lib/python3.8/shlex.py", line 300, in __next__
token = self.get_token()
File "/usr/lib/python3.8/shlex.py", line 109, in get_token
raw = self.read_token()
File "/usr/lib/python3.8/shlex.py", line 140, in read_token
nextchar = self.instream.read(1)
AttributeError: 'tuple' object has no attribute 'read'
Also, when I try to run a program where I imported selenium or any other installed module from pip it gives me the same error.
This is my pip version: pip 20.0.2 from /usr/lib/python3.8/site-packages/pip (python 3.8)
Thank you all in advance!
I have installed Python 3.6 with Anaconda 3.
The installation was successful however, when I launch a notebook, there is
kernel error immediatly with the following error message :
Traceback (most recent call last):
File "C:\Users\Lionel\Anaconda3\lib\site-
packages\notebook\base\handlers.py", line 516, in wrapper
result = yield gen.maybe_future(method(self, *args, **kwargs))
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\gen.py", line
1055, in run
value = future.result()
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\concurrent.py",
line 238, in result
raise_exc_info(self._exc_info)
File "<string>", line 4, in raise_exc_info
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\gen.py", line
1063, in run
yielded = self.gen.throw(*exc_info)
File "C:\Users\Lionel\Anaconda3\lib\site-
packages\notebook\services\sessions\handlers.py", line 75, in post
type=mtype))
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\gen.py", line
1055, in run
value = future.result()
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\concurrent.py",
line 238, in result
raise_exc_info(self._exc_info)
File "<string>", line 4, in raise_exc_info
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\gen.py", line
1063, in run
yielded = self.gen.throw(*exc_info)
File "C:\Users\Lionel\Anaconda3\lib\site-
packages\notebook\services\sessions\sessionmanager.py", line 79, in
create_session
kernel_id = yield self.start_kernel_for_session(session_id, path, name,
type, kernel_name)
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\gen.py", line
1055,
in run
value = future.result()
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\concurrent.py",
line 238, in result
raise_exc_info(self._exc_info)
File "<string>", line 4, in raise_exc_info
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\gen.py", line
1063, in run
yielded = self.gen.throw(*exc_info)
File "C:\Users\Lionel\Anaconda3\lib\site-
packages\notebook\services\sessions\sessionmanager.py", line 92, in
start_kernel_for_session
self.kernel_manager.start_kernel(path=kernel_path, kernel_name=kernel_name)
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\gen.py", line
1055, in run
value = future.result()
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\concurrent.py",
line 238, in result
raise_exc_info(self._exc_info)
File "<string>", line 4, in raise_exc_info
File "C:\Users\Lionel\Anaconda3\lib\site-packages\tornado\gen.py", line 307,
in wrapper
yielded = next(result)
File "C:\Users\Lionel\Anaconda3\lib\site-
packages\notebook\services\kernels\kernelmanager.py", line 94, in
start_kernel
super(MappingKernelManager, self).start_kernel(**kwargs)
File "C:\Users\Lionel\Anaconda3\lib\site-
packages\jupyter_client\multikernelmanager.py", line 110, in start_kernel
km.start_kernel(**kwargs)
File "C:\Users\Lionel\Anaconda3\lib\site-
packages\jupyter_client\manager.py", line 257, in start_kernel
**kw)
File "C:\Users\Lionel\Anaconda3\lib\site-
packages\jupyter_client\manager.py", line 203, in _launch_kernel
return launch_kernel(kernel_cmd, **kw)
File "C:\Users\Lionel\Anaconda3\lib\site-
packages\jupyter_client\launcher.py", line 128, in launch_kernel
proc = Popen(cmd, **kwargs)
File "C:\Users\Lionel\Anaconda3\lib\subprocess.py", line 709, in __init__
restore_signals, start_new_session)
File "C:\Users\Lionel\Anaconda3\lib\subprocess.py", line 997, in
_execute_child
startupinfo)
FileNotFoundError: [WinError 2] Le fichier spécifié est introuvable
Note that before installing Anaconda 3, I uninstalled Anaconda 2 (with Python 2.7).
How can I fix this?
Finally I found a solution in the web :
I had to execute the following command :
python -m ipykernel install --user
Regards,
Lionel