Airflow in docker keep restarting - python-3.x

Im trying to create a local Airflow env with Docker + ubuntu in a Windows OS.
I used the following wizard:
https://github.com/aws/aws-mwaa-local-runner
This wizard creates 2 containers (1 for DB and 1 for the Airflow)
Now I'm stuck with the following problem:
My Airflow container keeps restarting after troughing 2 exceptions:
"ERROR: You need to initialize the database. Please run airflow db init. Make sure the command is run using Airflow version 2.3.2."
Traceback (most recent call last):
File "/usr/local/bin/airflow", line 8, in
sys.exit(main())
File "/usr/local/airflow/.local/lib/python3.7/site-packages/airflow/main.py", line 38, in main
args.func(args)
File "/usr/local/airflow/.local/lib/python3.7/site-packages/airflow/cli/cli_parser.py", line 51, in command
return func(*args, **kwargs)
File "/usr/local/airflow/.local/lib/python3.7/site-packages/airflow/cli/commands/db_command.py", line 35, in initdb
db.initdb()
File "/usr/local/airflow/.local/lib/python3.7/site-packages/airflow/utils/session.py", line 71, in wrapper
return func(*args, session=session, **kwargs)
File "/usr/local/airflow/.local/lib/python3.7/site-packages/airflow/utils/db.py", line 648, in initdb
upgradedb(session=session)
File "/usr/local/airflow/.local/lib/python3.7/site-packages/airflow/utils/session.py", line 68, in wrapper
return func(*args, **kwargs)
File "/usr/local/airflow/.local/lib/python3.7/site-packages/airflow/utils/db.py", line 1449, in upgradedb
command.upgrade(config, revision=to_revision or 'heads')
File "/usr/local/lib/python3.7/site-packages/alembic/command.py", line 294, in upgrade
script.run_env()
File "/usr/local/lib/python3.7/site-packages/alembic/script/base.py", line 490, in run_env
util.load_python_file(self.dir, "env.py")
File "/usr/local/lib/python3.7/site-packages/alembic/util/pyfiles.py", line 97, in load_python_file
module = load_module_py(module_id, path)
File "/usr/local/lib/python3.7/site-packages/alembic/util/compat.py", line 182, in load_module_py
spec.loader.exec_module(module)
File "", line 728, in exec_module
File "", line 219, in _call_with_frames_removed
File "/usr/local/airflow/.local/lib/python3.7/site-packages/airflow/migrations/env.py", line 107, in
run_migrations_online()
File "/usr/local/airflow/.local/lib/python3.7/site-packages/airflow/migrations/env.py", line 101, in run_migrations_online
context.run_migrations()
File "", line 8, in run_migrations
File "/usr/local/lib/python3.7/site-packages/alembic/runtime/environment.py", line 813, in run_migrations
self.get_context().run_migrations(**kw)
File "/usr/local/lib/python3.7/site-packages/alembic/runtime/migration.py", line 548, in run_migrations
for step in self._migrations_fn(heads, self):
File "/usr/local/lib/python3.7/site-packages/alembic/command.py", line 283, in upgrade
return script._upgrade_revs(revision, rev)
File "/usr/local/lib/python3.7/site-packages/alembic/script/base.py", line 365, in _upgrade_revs
revs = list(revs)
File "/usr/local/lib/python3.7/site-packages/alembic/script/revision.py", line 1040, in _iterate_revisions
total_space.remove(rev.revision)
KeyError: '75d5ed6c2b43'"
I tried to run the following command in ubuntu for updating the DB:
"docker exec -it aws-mwaa-local-runner-202_local-runner_1 /entrypoint.sh airflow db upgrade"
but getting same error:
"..... KeyError: '75d5ed6c2b43'"
I also tried to reset and init the Airflow DB with-
docker exec -it aws-mwaa-local-runner-202_local-runner_1 /entrypoint.sh airflow initdb
docker exec -it aws-mwaa-local-runner-202_local-runner_1 /entrypoint.sh airflow reset
and still getting the "..... KeyError: '75d5ed6c2b43'" error.
waiting for response,
Thanks

Related

django 3.2- error creating superuser in sqllite3

PS C:\Users\Abdulla Zia\PycharmProjects\EnhanceImage> python manage.py createsuperuser
Username (leave blank to use 'abdullazia'): admin
Email address:
Password:
Password (again):
Traceback (most recent call last):
File "manage.py", line 22, in <module>
main()
File "manage.py", line 18, in main
execute_from_command_line(sys.argv)
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\Lib\site-packages\django\core\management\__init__.py", line 419, in execute_from_command_line
utility.execute()
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\Lib\site-packages\django\core\management\__init__.py", line 413, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\Lib\site-packages\django\core\management\base.py", line 354, in run_from_argv
self.execute(*args, **cmd_options)
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\Lib\site-packages\django\contrib\auth\management\commands\createsuperuser.py", line 79, in execute
return super().execute(*args, **options)
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\Lib\site-packages\django\core\management\base.py", line 398, in execute
output = self.handle(*args, **options)
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\Lib\site-packages\django\contrib\auth\management\commands\createsuperuser.py", line 157, in handle
validate_password(password2, self.UserModel(**fake_user_data))
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\Lib\site-packages\django\contrib\auth\password_validation.py", line 44, in validate_password
password_validators = get_default_password_validators()
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\Lib\site-packages\django\contrib\auth\password_validation.py", line 19, in get_default_password_validators
return get_password_validators(settings.AUTH_PASSWORD_VALIDATORS)
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\Lib\site-packages\django\contrib\auth\password_validation.py", line 30, in get_password_validators
validators.append(klass(**validator.get('OPTIONS', {})))
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\Lib\site-packages\django\contrib\auth\password_validation.py", line 174, in __init__
with gzip.open(password_list_path, 'rt', encoding='utf-8') as f:
File "C:\Users\Abdulla Zia\AppData\Local\Programs\Python\Python37\lib\gzip.py", line 57, in open
raise TypeError("filename must be a str or bytes object, or a file")
TypeError: filename must be a str or bytes object, or a file
performed all the migrations still getting this error. I've deleted database file and created new one still getting the same error. server is up and in running without any errors
please make migration and try to create the superuser
try this
python manage.py makemigrations
python manage.py migrate
python manage.py createsuperuser
or try this when its your initial migration
python manage.py migrate
python manage.py createsuperuser

poetry install fails with [CalledProcessError] while installing github submodule

I'm new to python and github. when I'm trying to install few submodules getting below error.
I'm using git bash on windows
pip is at latest version 20.2.3.
able to install other submodules using poetry successfully
$ poetry install -vvv
Using virtualenv: E:\Dev\venv\venv_test
Installing dependencies from lock file
[CalledProcessError]
Command '['C:\\Users\\88810\\AppData\\Local\\Temp\\tziwqwewd56\\Scripts\\python.exe', '-Im', 'ensurepip', '--upgrade', '--default-pip']' returned non-zero exit status 3221226505.
Traceback (most recent call last):
File "C:\Users\88810\.poetry\lib\poetry\_vendor\py3.6\clikit\console_application.py", line 131, in run
status_code = command.handle(parsed_args, io)
File "C:\Users\88810\.poetry\lib\poetry\_vendor\py3.6\clikit\api\command\command.py", line 120, in handle
status_code = self._do_handle(args, io)
File "C:\Users\88810\.poetry\lib\poetry\_vendor\py3.6\clikit\api\command\command.py", line 171, in _do_handle
return getattr(handler, handler_method)(args, io, self)
File "C:\Users\88810\.poetry\lib\poetry\_vendor\py3.6\cleo\commands\command.py", line 92, in wrap_handle
return self.handle()
File "C:\Users\88810\.poetry\lib\poetry\console\commands\install.py", line 69, in handle
return_code = installer.run()
File "C:\Users\88810\.poetry\lib\poetry\installation\installer.py", line 74, in run
self._do_install(local_repo)
File "C:\Users\88810\.poetry\lib\poetry\installation\installer.py", line 225, in _do_install
ops = solver.solve(use_latest=whitelist)
File "C:\Users\88810\.poetry\lib\poetry\puzzle\solver.py", line 36, in solve
packages, depths = self._solve(use_latest=use_latest)
File "C:\Users\88810\.poetry\lib\poetry\puzzle\solver.py", line 181, in _solve
self._package, self._provider, locked=locked, use_latest=use_latest
File "C:\Users\88810\.poetry\lib\poetry\mixology\__init__.py", line 7, in resolve_version
return solver.solve()
File "C:\Users\88810\.poetry\lib\poetry\mixology\version_solver.py", line 80, in solve
next = self._choose_package_version()
File "C:\Users\88810\.poetry\lib\poetry\mixology\version_solver.py", line 388, in _choose_package_version
version = self._provider.complete_package(version)
File "C:\Users\88810\.poetry\lib\poetry\puzzle\provider.py", line 601, in complete_package
self.search_for_vcs(r)
File "C:\Users\88810\.poetry\lib\poetry\puzzle\provider.py", line 175, in search_for_vcs
name=dependency.name,
File "C:\Users\88810\.poetry\lib\poetry\puzzle\provider.py", line 213, in get_package_from_vcs
package = cls.get_package_from_directory(tmp_dir, name=name)
File "C:\Users\88810\.poetry\lib\poetry\puzzle\provider.py", line 361, in get_package_from_directory
cls._execute_setup()
File "C:\Users\88810\.poetry\lib\poetry\puzzle\provider.py", line 872, in _execute_setup
EnvManager.build_venv(tmp_dir)
File "C:\Users\88810\.poetry\lib\poetry\utils\env.py", line 688, in build_venv
builder.create(str(path))
File "E:\python36\lib\venv\__init__.py", line 67, in create
self._setup_pip(context)
File "E:\python36\lib\venv\__init__.py", line 245, in _setup_pip
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
File "E:\python36\lib\subprocess.py", line 336, in check_output
**kwargs).stdout
File "C:\Users\88810\.poetry\lib\poetry\utils\_compat.py", line 206, in run
retcode, process.args, output=stdout, stderr=stderr
(venv_test)
I was able to resolve the issue after following these steps:
Upgraded python 3.6 to 3.7.4.
I had earlier installed python on E drive, installed it on C drive.

MLFlow Projects throw JSONDecode error when run

I'm trying to get MLFlow Projects to run using the MLFlow CLI and its following the tutorial leads to an error. For any project I try to run from the CLI, I get the following error
Traceback (most recent call last):
File "/home/rbc/.local/bin/mlflow", line 11, in <module>
sys.exit(cli())
File "/home/rbc/.local/lib/python3.6/site-packages/click/core.py", line 764, in __call__
return self.main(*args, **kwargs)
File "/home/rbc/.local/lib/python3.6/site-packages/click/core.py", line 717, in main
rv = self.invoke(ctx)
File "/home/rbc/.local/lib/python3.6/site-packages/click/core.py", line 1137, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/rbc/.local/lib/python3.6/site-packages/click/core.py", line 956, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/rbc/.local/lib/python3.6/site-packages/click/core.py", line 555, in invoke
return callback(*args, **kwargs)
File "/home/rbc/.local/lib/python3.6/site-packages/mlflow/cli.py", line 139, in run
run_id=run_id,
File "/home/rbc/.local/lib/python3.6/site-packages/mlflow/projects/__init__.py", line 230, in run
storage_dir=storage_dir, block=block, run_id=run_id)
File "/home/rbc/.local/lib/python3.6/site-packages/mlflow/projects/__init__.py", line 88, in _run
active_run = _create_run(uri, experiment_id, work_dir, entry_point)
File "/home/rbc/.local/lib/python3.6/site-packages/mlflow/projects/__init__.py", line 579, in _create_run
active_run = tracking.MlflowClient().create_run(experiment_id=experiment_id, tags=tags)
File "/home/rbc/.local/lib/python3.6/site-packages/mlflow/tracking/client.py", line 101, in create_run
source_version=source_version
File "/home/rbc/.local/lib/python3.6/site-packages/mlflow/store/rest_store.py", line 156, in create_run
response_proto = self._call_endpoint(CreateRun, req_body)
File "/home/rbc/.local/lib/python3.6/site-packages/mlflow/store/rest_store.py", line 66, in _call_endpoint
js_dict = json.loads(response.text)
File "/usr/lib/python3.6/json/__init__.py", line 354, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.6/json/decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.6/json/decoder.py", line 357, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Here's an example of the type of command I'm using to start the run, which comes directly from the tutorial
mlflow run https://github.com/mlflow/mlflow#examples/sklearn_elasticnet_wine -m databricks -c cluster-spec.json --experiment-id 72647065958042 -P alpha=2.0 -P l1_ratio=0.5
I've traced the error to something involving MLFLow returning empty when it tries to start a run but I can successfully run MLFlow experiments using the Databricks environment I'm connecting to so I'm not sure where the problem is, I'm running MLFlow 0.9.1 on Ubuntu 18.04
not sure if you have solved your issue, but here is how I fixed it:
the databricks-cli work with the following config without problem:
host = https://xxx.databricks.net/?o=<org_id>
token=dapixxx
but mlflow not quit happy about that, change it to:
host = https://xxx.databricks.net
username = token
password = dapixxx

Migration error at startup for Jupyter Notebook

Installed Anaconda3 (Anaconda Navigator 1.8.7) for Mac OS X (10.13.5) as part of a JuliaPro (0.6.3) install. Every time I try to launch Jupyter Notebook (Terminal or Navigator GUI), I get this:
Prestbury:~ me$ jupyter notebook Traceback (most recent call last):
File "/anaconda3/bin/jupyter-notebook", line 11, in
sys.exit(main()) File "/anaconda3/lib/python3.6/site-packages/jupyter_core/application.py",
line 266, in launch_instance
return super(JupyterApp, cls).launch_instance(argv=argv, **kwargs) File
"/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py",
line 657, in launch_instance
app.initialize(argv) File "", line 2, in initialize File
"/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py",
line 87, in catch_config_error
return method(app, *args, **kwargs) File "/anaconda3/lib/python3.6/site-packages/notebook/notebookapp.py", line
1501, in initialize
super(NotebookApp, self).initialize(argv) File "", line 2, in initialize File
"/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py",
line 87, in catch_config_error
return method(app, *args, **kwargs) File "/anaconda3/lib/python3.6/site-packages/jupyter_core/application.py",
line 242, in initialize
self.migrate_config() File "/anaconda3/lib/python3.6/site-packages/jupyter_core/application.py",
line 168, in migrate_config
migrate() File "/anaconda3/lib/python3.6/site-packages/jupyter_core/migrate.py", line
247, in migrate
with open(os.path.join(env['jupyter_config'], 'migrated'), 'w') as f: PermissionError: [Errno 13] Permission denied:
'/Users/me/.jupyter/migrated'
I have completely removed Anaconda3 and performed a fresh install after also installing nodejs. The problem persists. Any recommendation please?

canĀ“t start ocsmanager in openchange

I compiled and installed openchange. Started samba and then:
make ocsmanager-install
make pyopenchange
make pyopenchange-install
All ok without errors, but when I try to run
paster serve /etc/ocsmanager/ocsmanager.ini --pid-file /var/run/ocsmanager.pid --log-file /var/log/ocsmanager.log
It aborts and logs this (in log file):
Traceback (most recent call last):
File "/usr/bin/paster", line 9, in <module>
load_entry_point('PasteScript==1.7.5', 'console_scripts', 'paster')()
File "/usr/lib/python2.7/site-packages/paste/script/command.py", line 104, in run
invoke(command, command_name, options, args[1:])
File "/usr/lib/python2.7/site-packages/paste/script/command.py", line 143, in invoke
exit_code = runner.run(args)
File "/usr/lib/python2.7/site-packages/paste/script/command.py", line 238, in run
result = self.command()
File "/usr/lib/python2.7/site-packages/paste/script/serve.py", line 284, in command
relative_to=base, global_conf=vars)
File "/usr/lib/python2.7/site-packages/paste/script/serve.py", line 321, in loadapp
**kw)
File "/usr/lib/python2.7/site-packages/paste/deploy/loadwsgi.py", line 247, in loadapp
return loadobj(APP, uri, name=name, **kw)
File "/usr/lib/python2.7/site-packages/paste/deploy/loadwsgi.py", line 272, in loadobj
return context.create()
File "/usr/lib/python2.7/site-packages/paste/deploy/loadwsgi.py", line 710, in create
return self.object_type.invoke(self)
File "/usr/lib/python2.7/site-packages/paste/deploy/loadwsgi.py", line 146, in invoke
return fix_call(context.object, context.global_conf, **context.local_conf)
File "/usr/lib/python2.7/site-packages/paste/deploy/util.py", line 56, in fix_call
val = callable(*args, **kw)
File "/usr/local/samba/lib/python2.7/site-packages/ocsmanager/config/middleware.py", line 43, in make_app
config = load_environment(global_conf, app_conf)
File "/usr/local/samba/lib/python2.7/site-packages/ocsmanager/config/environment.py", line 157, in load_environment
mstore = mapistore.MAPIStore(config['ocsmanager']['main']['mapistore_root'])
SystemError: error in mapistore_init
Removing PID file /var/run/ocsmanager.pid
My /etc/ocsmanager/ocsmanager.conf is the installed by make scripts with the only change of the location of mapistore:
mapistore_root = /usr/local/samba/private
mapistore_data = /usr/local/samba/private/mapistore

Resources