Convert NER dataset to spaCy formt for Arabic text - nlp

I tried to convert a NER dataset written in English to spaCy format, everything worked fine but when I tried to do the same thing with a dataset written in Arabic:
!python -m spacy convert -c iob -s -n 10 -b nlp /gdrive/MyDrive/LearningSpacy/SimpleNER-Dataset.txt .
I got this error:
Traceback (most recent call last):
File "/usr/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.7/dist-packages/spacy/__main__.py", line 4, in <module>
setup_cli()
File "/usr/local/lib/python3.7/dist-packages/spacy/cli/_util.py", line 71, in setup_cli
command(prog_name=COMMAND)
File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/typer/main.py", line 532, in wrapper
return callback(**use_params) # type: ignore
File "/usr/local/lib/python3.7/dist-packages/spacy/cli/convert.py", line 74, in convert_cli
converter = _get_converter(msg, converter, input_path)
File "/usr/local/lib/python3.7/dist-packages/spacy/cli/convert.py", line 257, in _get_converter
input_data = file_.read()
File "/usr/lib/python3.7/codecs.py", line 322, in decode
(result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byt
This is how the SimpleNER-Dataset.txt looks like:

Related

Airflow scheduler works normally, fails with -D

I have set up Airflow on an AWS EC2 server with Ubuntu 18.04, Python 3.6.9. The DB backend is a 5.7.26 MySQL, I am using a LocalExecutor. The setup is nothing more than:
apt-get install python3 python3-pip python3-venv libmysqlclient-dev
I install Airflow with pip3 install apache-airflow[mysql]==1.10.9, init DB connection, start the webserver and it works both normally and with -D. The scheduler, however, works only when run in the foreground. Trying to run it as a daemon fails with the following trace:
Traceback (most recent call last):
File "/usr/local/bin/airflow", line 37, in <module> args.func(args)
File "/usr/local/lib/python3.6/dist-packages/airflow/utils/cli.py", line 75, in wrapper return f(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/bin/cli.py", line 1032, in schedulerjob.run()
File "/usr/local/lib/python3.6/dist-packages/airflow/jobs/base_job.py", line 215, in run session.commit()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 1036, in commit self.transaction.commit()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 503, in commitself._prepare_impl()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 482, in _prepare_impl self.session.flush()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 2496, in flushself._flush(objects)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 2637, in _flush transaction.rollback(_capture_exception=True)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/util/langhelpers.py", line 69, in __exit__exc_value, with_traceback=exc_tb,
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/util/compat.py", line 178, in raise_raise exception
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 2597, in _flush flush_context.execute()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute rec.execute(self)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute uow,
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/persistence.py", line 213, in save_obj) in _organize_states_for_save(base_mapper, states, uowtransaction):
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/persistence.py", line 374, in _organize_states_for_save base_mapper, uowtransaction, states
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/persistence.py", line 1602, in _connections_for_states connection = uowtransaction.transaction.connection(base_mapper)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 313, in connection return self._connection_for_bind(bind, execution_options)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 420, in _connection_for_bind conn = self._parent._connection_for_bind(bind, execution_options)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 432, in _connection_for_bind conn = bind._contextual_connect()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 2251, in _contextual_connect self._wrap_pool_connect(self.pool.connect, None),
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 2285, in _wrap_pool_connect return fn()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 363, in connect return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 804, in _checkout result = pool._dialect.do_ping(fairy.connection)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 138, in do_pingdbapi_connection.ping(False)
File "/usr/local/lib/python3.6/dist-packages/pymysql/connections.py", line 546, in ping self._execute_command(COMMAND.COM_PING, "")
File "/usr/local/lib/python3.6/dist-packages/pymysql/connections.py", line 771, in _execute_command self._write_bytes(packet)
File "/usr/local/lib/python3.6/dist-packages/pymysql/connections.py", line 711, in _write_bytesself._sock.settimeout(self._write_timeout)
OSError: [Errno 9] Bad file descriptor
Exception ignored in: <function _ConnectionRecord.checkout.<locals>.<lambda> at 0x7f2eb025e268>
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 503, in <lambda>
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 702, in _finalize_fairy
File "/usr/lib/python3.6/logging/__init__.py", line 1337, in error
File "/usr/lib/python3.6/logging/__init__.py", line 1444, in _log
File "/usr/lib/python3.6/logging/__init__.py", line 1454, in handle
File "/usr/lib/python3.6/logging/__init__.py", line 1516, in callHandlers
File "/usr/lib/python3.6/logging/__init__.py", line 865, in handle
File "/usr/lib/python3.6/logging/__init__.py", line 1071, in emit
File "/usr/lib/python3.6/logging/__init__.py", line 1061, in _open
NameError: name 'open' is not defined
I am trying to set up a server for semi-production purposes, so this is actually a blocker for me. I would be grateful for any advice.
EDIT
I tried using mysql-connector-python==8.0.18 and the scheduler did run as a daemon, but an attempt to open a dag failed with the following trace:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python3.6/dist-packages/flask/_compat.py", line 39, in reraise
raise value
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python3.6/dist-packages/flask_admin/base.py", line 69, in inner
return self._run_view(f, *args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/flask_admin/base.py", line 368, in _run_view
return fn(self, *args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/flask_login/utils.py", line 258, in decorated_view
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/www/utils.py", line 384, in view_func
return f(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/www/utils.py", line 290, in wrapper
return f(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/utils/db.py", line 74, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/www/views.py", line 1559, in tree
start_date=min_date, end_date=base_date, session=session)
File "/usr/local/lib/python3.6/dist-packages/airflow/utils/db.py", line 70, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/models/dag.py", line 837, in get_task_instances
tis = tis.order_by(TaskInstance.execution_date).all()
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3233, in all
return list(self)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3389, in __iter__
return self._execute_and_instances(context)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3414, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 982, in execute
return meth(self, multiparams, params)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 293, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1101, in _execute_clauseelement
distilled_params,
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
e, statement, parameters, cursor, context
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1478, in _handle_dbapi_exception
util.reraise(*exc_info)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 588, in do_execute
cursor.execute(statement, parameters)
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor_cext.py", line 272, in execute
self._handle_result(result)
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor_cext.py", line 163, in _handle_result
self._handle_resultset()
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor_cext.py", line 651, in _handle_resultset
self._rows = self._cnx.get_rows()[0]
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/connection_cext.py", line 301, in get_rows
else self._cmysql.fetch_row()
SystemError: <built-in method fetch_row of _mysql_connector.MySQL object at 0x3cdaa10> returned a result with an error set
EDIT 2
What finally did work was using mysqlclient by specifying the SQLAlchemy connection string to begin with mysql+mysqldb. I am not giving this as an answer, because the initial problem persists.

VSCode pytest test discovery failed on Mac

VSCode Version: 1.44.0
pytest verison: 4.4.1
Python extension: Python 2020.3.71659
VSCode popup: Test discovery error, please check the configuration settings for the tests.
OUTPUT log as below:
python /Users/eason/.vscode/extensions/ms-python.python-2020.3.71659/pythonFiles/testing_tools/run_adapter.py discover pytest -- --rootdir /Users/eason/android_guess_auto -s --cache-clear ./TestCases
Test Discovery failed:
Error: Traceback (most recent call last):
File "/Users/eason/.vscode/extensions/ms-python.python-2020.3.71659/pythonFiles/testing_tools/run_adapter.py", line 22, in <module>
main(tool, cmd, subargs, toolargs)
File "/Users/eason/.vscode/extensions/ms-python.python-2020.3.71659/pythonFiles/testing_tools/adapter/__main__.py", line 100, in main
parents, result = run(toolargs, **subargs)
File "/Users/eason/.vscode/extensions/ms-python.python-2020.3.71659/pythonFiles/testing_tools/adapter/pytest/_discovery.py", line 30, in discover
ec = _pytest_main(pytestargs, [_plugin])
File "/Users/eason/test_3env/lib/python3.6/site-packages/_pytest/config/__init__.py", line 60, in main
config = _prepareconfig(args, plugins)
File "/Users/eason/test_3env/lib/python3.6/site-packages/_pytest/config/__init__.py", line 201, in _prepareconfig
pluginmanager=pluginmanager, args=args
File "/Users/eason/test_3env/lib/python3.6/site-packages/pluggy/hooks.py", line 286, in __call__
return self._hookexec(self, self.get_hookimpls(), kwargs)
File "/Users/eason/test_3env/lib/python3.6/site-packages/pluggy/manager.py", line 93, in _hookexec
return self._inner_hookexec(hook, methods, kwargs)
File "/Users/eason/test_3env/lib/python3.6/site-packages/pluggy/manager.py", line 87, in <lambda>
firstresult=hook.spec.opts.get("firstresult") if hook.spec else False,
File "/Users/eason/test_3env/lib/python3.6/site-packages/pluggy/callers.py", line 203, in _multicall
gen.send(outcome)
File "/Users/eason/test_3env/lib/python3.6/site-packages/_pytest/helpconfig.py", line 93, in pytest_cmdline_parse
config = outcome.get_result()
File "/Users/eason/test_3env/lib/python3.6/site-packages/pluggy/callers.py", line 80, in get_result
raise ex[1].with_traceback(ex[2])
File "/Users/eason/test_3env/lib/python3.6/site-packages/pluggy/callers.py", line 187, in _multicall
res = hook_impl.function(*args)
File "/Users/eason/test_3env/lib/python3.6/site-packages/_pytest/config/__init__.py", line 679, in pytest_cmdline_parse
self.parse(args)
File "/Users/eason/test_3env/lib/python3.6/site-packages/_pytest/config/__init__.py", line 896, in parse
self._preparse(args, addopts=addopts)
File "/Users/eason/test_3env/lib/python3.6/site-packages/_pytest/config/__init__.py", line 829, in _preparse
self._initini(args)
File "/Users/eason/test_3env/lib/python3.6/site-packages/_pytest/config/__init__.py", line 752, in _initini
config=self,
File "/Users/eason/test_3env/lib/python3.6/site-packages/_pytest/config/findpaths.py", line 122, in determine_setup
rootdir, inifile, inicfg = getcfg([ancestor], config=config)
File "/Users/eason/test_3env/lib/python3.6/site-packages/_pytest/config/findpaths.py", line 35, in getcfg
iniconfig = py.iniconfig.IniConfig(p)
File "/Users/eason/test_3env/lib/python3.6/site-packages/py/_vendored_packages/iniconfig.py", line 54, in __init__
tokens = self._parse(iter(f))
File "/Users/eason/test_3env/lib/python3.6/site-packages/py/_vendored_packages/iniconfig.py", line 82, in _parse
for lineno, line in enumerate(line_iter):
File "/Users/eason/test_3env/bin/../lib/python3.6/encodings/ascii.py", line 26, in decode
return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xe6 in position 80: ordinal not in range(128)

Can't install python modules with pip on Manjaro Linux

I'm on Manjaro Linux KDE and every time I try to install a module with pip it gives me this error:
Defaulting to user installation because normal site-packages is not writeable
ERROR:
Exception:
Traceback (most recent call last):
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/base_command.py", line 186, in _main
status = self.run(options, args)
File "/usr/lib/python3.8/site-packages/pip/_internal/commands/install.py", line 279, in run
session = self.get_default_session(options)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 74, in get_default_session
self._session = self.enter_context(self._build_session(options))
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 84, in _build_session
session = PipSession(
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 241, in __init__
self.headers["User-Agent"] = user_agent()
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 132, in user_agent
zip(["name", "version", "id"], distro.linux_distribution()),
File "/usr/lib/python3.8/site-packages/distro.py", line 122, in linux_distribution
return _distro.linux_distribution(full_distribution_name)
File "/usr/lib/python3.8/site-packages/distro.py", line 677, in linux_distribution
self.version(),
File "/usr/lib/python3.8/site-packages/distro.py", line 737, in version
self.lsb_release_attr('release'),
File "/usr/lib/python3.8/site-packages/distro.py", line 899, in lsb_release_attr
return self._lsb_release_info.get(attribute, '')
File "/usr/lib/python3.8/site-packages/distro.py", line 552, in __get__
ret = obj.__dict__[self._fname] = self._f(obj)
File "/usr/lib/python3.8/site-packages/distro.py", line 1012, in _lsb_release_info
stdout = subprocess.check_output(cmd, stderr=devnull)
File "/usr/lib/python3.8/subprocess.py", line 411, in check_output
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 145, in __new__
process = cls.create_process(command, stdin, cwd=cwd, env=env, shell=shell)
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 121, in create_process
shlex.split(command),
File "/usr/lib/python3.8/shlex.py", line 311, in split
return list(lex)
File "/usr/lib/python3.8/shlex.py", line 300, in __next__
token = self.get_token()
File "/usr/lib/python3.8/shlex.py", line 109, in get_token
raw = self.read_token()
File "/usr/lib/python3.8/shlex.py", line 140, in read_token
nextchar = self.instream.read(1)
AttributeError: 'tuple' object has no attribute 'read'
Traceback (most recent call last):
File "/usr/bin/pip3", line 11, in <module>
load_entry_point('pip==20.0.2', 'console_scripts', 'pip3')()
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/main.py", line 75, in main
return command.main(cmd_args)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/base_command.py", line 105, in main
return self._main(args)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/base_command.py", line 224, in _main
self.handle_pip_version_check(options)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 143, in handle_pip_version_check
session = self._build_session(
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 84, in _build_session
session = PipSession(
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 241, in __init__
self.headers["User-Agent"] = user_agent()
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 132, in user_agent
zip(["name", "version", "id"], distro.linux_distribution()),
File "/usr/lib/python3.8/site-packages/distro.py", line 122, in linux_distribution
return _distro.linux_distribution(full_distribution_name)
File "/usr/lib/python3.8/site-packages/distro.py", line 677, in linux_distribution
self.version(),
File "/usr/lib/python3.8/site-packages/distro.py", line 737, in version
self.lsb_release_attr('release'),
File "/usr/lib/python3.8/site-packages/distro.py", line 899, in lsb_release_attr
return self._lsb_release_info.get(attribute, '')
File "/usr/lib/python3.8/site-packages/distro.py", line 552, in __get__
ret = obj.__dict__[self._fname] = self._f(obj)
File "/usr/lib/python3.8/site-packages/distro.py", line 1012, in _lsb_release_info
stdout = subprocess.check_output(cmd, stderr=devnull)
File "/usr/lib/python3.8/subprocess.py", line 411, in check_output
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 145, in __new__
process = cls.create_process(command, stdin, cwd=cwd, env=env, shell=shell)
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 121, in create_process
shlex.split(command),
File "/usr/lib/python3.8/shlex.py", line 311, in split
return list(lex)
File "/usr/lib/python3.8/shlex.py", line 300, in __next__
token = self.get_token()
File "/usr/lib/python3.8/shlex.py", line 109, in get_token
raw = self.read_token()
File "/usr/lib/python3.8/shlex.py", line 140, in read_token
nextchar = self.instream.read(1)
AttributeError: 'tuple' object has no attribute 'read'
Also, when I try to run a program where I imported selenium or any other installed module from pip it gives me the same error.
This is my pip version: pip 20.0.2 from /usr/lib/python3.8/site-packages/pip (python 3.8)
Thank you all in advance!

Problem with running flask on Python locally

from flask import Flask
app = Flask(__name__)
#app.route("/")
def hello():
return "Hello World!"
when trying to run Flask in any way I have found on the internet I still get an Error
File "c:\python36\lib\socket.py", line 673, in getfqdn
hostname, aliases, ipaddrs = gethostbyaddr(name)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8a in position 5: invalid
start byte
I guess it's because there is non ascii character in my name. I tried on my other account with only ascii characters and the problem still occurs. Result is the same with with DEBUG mode. Can anyone give me some tips on this, please? The code is th official basic example from Flask's website.
D:\kiwiAdmission\flask>flask run
* Serving Flask app "kiwi_flask.py"
* Environment: production
WARNING: Do not use the development server in a production environment.
Use a production WSGI server instead.
* Debug mode: off
Traceback (most recent call last):
File "c:\python36\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "c:\python36\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Python36\Scripts\flask.exe\__main__.py", line 9, in <module>
File "c:\python36\lib\site-packages\flask\cli.py", line 894, in main
cli.main(args=args, prog_name=name)
File "c:\python36\lib\site-packages\flask\cli.py", line 557, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "c:\python36\lib\site-packages\click\core.py", line 717, in main
rv = self.invoke(ctx)
File "c:\python36\lib\site-packages\click\core.py", line 1137, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "c:\python36\lib\site-packages\click\core.py", line 956, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "c:\python36\lib\site-packages\click\core.py", line 555, in invoke
return callback(*args, **kwargs)
File "c:\python36\lib\site-packages\click\decorators.py", line 64, in new_func
return ctx.invoke(f, obj, *args, **kwargs)
File "c:\python36\lib\site-packages\click\core.py", line 555, in invoke
return callback(*args, **kwargs)
File "c:\python36\lib\site-packages\flask\cli.py", line 771, in run_command
threaded=with_threads, ssl_context=cert)
File "c:\python36\lib\site-packages\werkzeug\serving.py", line 814, in run_sim
ple
inner()
File "c:\python36\lib\site-packages\werkzeug\serving.py", line 774, in inner
fd=fd)
File "c:\python36\lib\site-packages\werkzeug\serving.py", line 660, in make_se
rver
passthrough_errors, ssl_context, fd=fd)
File "c:\python36\lib\site-packages\werkzeug\serving.py", line 577, in __init_
_
self.address_family), handler)
File "c:\python36\lib\socketserver.py", line 453, in __init__
self.server_bind()
File "c:\python36\lib\http\server.py", line 138, in server_bind
self.server_name = socket.getfqdn(host)
File "c:\python36\lib\socket.py", line 673, in getfqdn
hostname, aliases, ipaddrs = gethostbyaddr(name)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8a in position 5: invalid
start byte
Solved, just renamed my host name! to only ascii name

Using filepaths as global variables in Python

I have a file global_vars.py that contains file paths saved as variables:
from pandas import Timestamp
final_vol_path = 'datasets/final_vols.csv'
final_price_path = 'datasets/final_prices.csv'
final_start_date = Timestamp('2017-01-01')
with other variables written in a similar fashion. However, the functions that I'm using to read in the data throw a FileNotFoundError when attempting to do the following in file1.py:
import scripts.global_vars as gv
read_data(gv.final_vol_path, gv.final_price_path) # throws FileNotFoundError
read_data('datasets/final_vols.csv', 'datasets/final_prices.csv') # this passes
Additionally, I've checked the file paths, and have gotten the following:
gv.final_vol_path == 'datasets/final_vols.csv' # returns True
gv.final_price_path == 'datasets/final_prices.csv' # returns True
Moreover, the pandas Timestamp object is processed without any problems.
Is there any explanation for why the FileNotFoundError is being thrown when attempting to access the file path as a variable from global_vars.py, but is not thrown when the actual string is passed in?
EDIT: The overall directory structure is as follows:
working_dir
L file1.py
L scripts
L global_vars.py
L datasets
L final_vols.csv
L final_prices.csv
EDIT 2: I added in a try-catch block to ensure the rest of the function doesn't break, not sure if that has affected the traceback, but here's what I get:
Traceback (most recent call last):
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\runpy.py", line
184, in _run_module_as_main
"__main__", mod_spec)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\Ananth\Anaconda3\envs\analytics-cpu\Scripts\nose2.exe\__main__.py", line 9, in <module>
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\main.py", line 306, in discover
return main(*args, **kwargs)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\main.py", line 100, in __init__
super(PluggableTestProgram, self).__init__(**kw)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\unittest\main.py", line 93, in __init__
self.parseArgs(argv)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\main.py", line 133, in parseArgs
self.createTests()
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\main.py", line 258, in createTests
self.testNames, self.module)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\loader.py", line 69, in loadTestsFromNames
for name in event.names]
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\loader.py", line 69, in <listcomp>
for name in event.names]
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\loader.py", line 84, in loadTestsFromName
result = self.session.hooks.loadTestsFromName(event)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\events.py", line 224, in __call__
result = getattr(plugin, self.method)(event)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\plugins\loader\testcases.py", line 56, in loadTestsFromName
result = util.test_from_name(name, module)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\util.py", line 106, in test_from_name
parent, obj = object_from_name(name, module)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\nose2\util.py", line 117, in object_from_name
module = __import__('.'.join(parts_copy))
File "C:\Users\Ananth\Desktop\Modules\PortfolioVARModule\tests\test_simulation.py", line 24, in <module>
gv.test_start_date)
File "C:\Users\Ananth\Desktop\Modules\PortfolioVARModule\scripts\prep_data.py", line 119, in read_data
priceDF = pd.read_csv(pricepath).dropna()
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\pandas\io\parsers.py", line 646, in parser_f
return _read(filepath_or_buffer, kwds)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\pandas\io\parsers.py", line 389, in _read
parser = TextFileReader(filepath_or_buffer, **kwds)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\pandas\io\parsers.py", line 730, in __init__
self._make_engine(self.engine)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\pandas\io\parsers.py", line 923, in _make_engine
self._engine = CParserWrapper(self.f, **self.options)
File "c:\users\ananth\anaconda3\envs\analytics-cpu\lib\site-packages\pandas\io\parsers.py", line 1390, in __init__
self._reader = _parser.TextReader(src, **kwds)
File "pandas\parser.pyx", line 373, in pandas.parser.TextReader.__cinit__ (pandas\parser.c:4184)
File "pandas\parser.pyx", line 667, in pandas.parser.TextReader._setup_parser_source (pandas\parser.c:8449)
FileNotFoundError: File b'datasets/corn_price.csv' does not exist
Problem is the addition of the letter b in front of your file's path.
You get the b because you encoded to utf-8.
Try:
read_data(str(gv.final_vol_path,'utf-8'), str(gv.final_price_path, 'utf-8'))

Resources