ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the NAME value - python-3.x

I have a Django cookiecutter template. After performing all the required actions given on cookiecutter docs when I run python manage.py migrate I get this error
Traceback (most recent call last):
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\task_manager_app\manage.py", line 31, in <module>
execute_from_command_line(sys.argv)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\core\management\__init__.py", line 419, in execute_from_command_line
utility.execute()
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\core\management\__init__.py", line 413, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\core\management\base.py", line 354, in run_from_argv
self.execute(*args, **cmd_options)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\core\management\base.py", line 398, in execute
output = self.handle(*args, **options)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\core\management\base.py", line 89, in wrapped
res = handle_func(*args, **kwargs)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\core\management\commands\migrate.py", line 75, in handle
self.check(databases=[database])
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\core\management\base.py", line 419, in check
all_issues = checks.run_checks(
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\core\checks\registry.py", line 76, in run_checks
new_errors = check(app_configs=app_configs, databases=databases)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\core\checks\model_checks.py", line 34, in check_all_models
errors.extend(model.check(**kwargs))
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\db\models\base.py", line 1303, in check
*cls._check_indexes(databases),
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\db\models\base.py", line 1695, in _check_indexes
connection.features.supports_covering_indexes or
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\utils\functional.py", line 48, in __get__
res = instance.__dict__[self.name] = self.func(instance)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\db\backends\postgresql\features.py", line 93, in is_postgresql_11
return self.connection.pg_version >= 110000
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\utils\functional.py", line 48, in __get__
res = instance.__dict__[self.name] = self.func(instance)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\db\backends\postgresql\base.py", line 329, in pg_version
with self.temporary_connection():
File "C:\Python310\lib\contextlib.py", line 135, in __enter__
return next(self.gen)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\db\backends\base\base.py", line 603, in temporary_connection
with self.cursor() as cursor:
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\utils\asyncio.py", line 33, in inner
return func(*args, **kwargs)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\db\backends\base\base.py", line 259, in cursor
return self._cursor()
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\db\backends\base\base.py", line 235, in _cursor
self.ensure_connection()
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\utils\asyncio.py", line 33, in inner
return func(*args, **kwargs)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\db\backends\base\base.py", line 219, in ensure_connection
self.connect()
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\utils\asyncio.py", line 33, in inner
return func(*args, **kwargs)
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\db\backends\base\base.py", line 199, in connect
conn_params = self.get_connection_params()
File "C:\Users\Rahul Tiwari\Desktop\My_Projects\GDC\GDC-Level-10-Milestone-master\env\lib\site-packages\django\db\backends\postgresql\base.py", line 157, in get_connection_params
raise ImproperlyConfigured(
django.core.exceptions.ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the NAME value.
This is the settings base.py file (database setting part) which come by default with template
DATABASES = {
"default": env.db(
"DATABASE_URL",
default="postgres://localhost/task_manager_app",
),
}
DATABASES["default"]["ATOMIC_REQUESTS"] = True
I've set the database URL using command
export DATABASE_URL=postgres://postgres:mypass#127.0.0.1:5432/task_manager_app
Anyone help me out to get rid of this error.

First of all you have to do it as follows:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql', #Make sure you have psycopg2 installed. It is postgresql connector.
'NAME': ‘DB NAME’,
'USER': 'DB USERNAME',
'PASSWORD': 'USER PASSWORD',
'HOST': 'HOST IP',
'PORT': 'PORT',#Generally 5432
}
}

After puzzling over it a lot, I got the solution. The problem was having the character "#" in my database password.

Related

Kernel issues on jupyter

Error Starting Kernel
Traceback (most recent call last):
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tornado/web.py", line 1713, in _execute
result = await result
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tornado/gen.py", line 776, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/notebook/services/sessions/handlers.py", line 72, in post
type=mtype))
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tornado/gen.py", line 769, in run
value = future.result()
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tornado/gen.py", line 776, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/notebook/services/sessions/sessionmanager.py", line 88, in create_session
kernel_id = yield self.start_kernel_for_session(session_id, path, name, type, kernel_name)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tornado/gen.py", line 769, in run
value = future.result()
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tornado/gen.py", line 776, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/notebook/services/sessions/sessionmanager.py", line 101, in start_kernel_for_session
self.kernel_manager.start_kernel(path=kernel_path, kernel_name=kernel_name)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tornado/gen.py", line 769, in run
value = future.result()
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tornado/gen.py", line 234, in wrapper
yielded = ctx_run(next, result)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/notebook/services/kernels/kernelmanager.py", line 168, in start_kernel
super(MappingKernelManager, self).start_kernel(**kwargs)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/utils.py", line 30, in wrapped
raise e
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/utils.py", line 27, in wrapped
return loop.run_until_complete(future)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/nest_asyncio.py", line 90, in run_until_complete
return f.result()
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/asyncio/futures.py", line 178, in result
raise self._exception
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/asyncio/tasks.py", line 223, in __step
result = coro.send(None)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/multikernelmanager.py", line 214, in _async_start_kernel
starter = ensure_async(km.start_kernel(**kwargs))
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/utils.py", line 30, in wrapped
raise e
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/utils.py", line 27, in wrapped
return loop.run_until_complete(future)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/nest_asyncio.py", line 90, in run_until_complete
return f.result()
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/asyncio/futures.py", line 178, in result
raise self._exception
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/asyncio/tasks.py", line 223, in __step
result = coro.send(None)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/manager.py", line 90, in wrapper
raise e
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/manager.py", line 82, in wrapper
out = await method(self, *args, **kwargs)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/manager.py", line 394, in _async_start_kernel
await ensure_async(self._launch_kernel(kernel_cmd, **kw))
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/utils.py", line 30, in wrapped
raise e
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/utils.py", line 27, in wrapped
return loop.run_until_complete(future)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/nest_asyncio.py", line 90, in run_until_complete
return f.result()
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/asyncio/futures.py", line 178, in result
raise self._exception
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/asyncio/tasks.py", line 223, in __step
result = coro.send(None)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/manager.py", line 311, in _async_launch_kernel
connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/provisioning/local_provisioner.py", line 204, in launch_kernel
self.process = launch_kernel(cmd, **scrubbed_kwargs)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/launcher.py", line 170, in launch_kernel
raise ex
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/jupyter_client/launcher.py", line 158, in launch_kernel
proc = Popen(cmd, **kwargs)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/subprocess.py", line 775, in init
restore_signals, start_new_session)
File "/root/anaconda3/envs/TensorPy36/lib/python3.7/subprocess.py", line 1453, in _execute_child
restore_signals, start_new_session, preexec_fn)
BlockingIOError: [Errno 11] Resource temporarily unavailable
Permanent solution for this error

Django serializers.save() does not use postgres

I have the following function that runs automatically with cronjob, it makes predictions based on an anomaly detection model and it saves the results to a postgres database:
def prediction_job():
"""
This job will run once every batch_minutes minutes to predict if the IPs appear in these batch_minutes minutes are normal or malicious
and save the results to postgres
"""
ip_estimator = IpStatePrediction()
predictions = ip_estimator.predict_ip_state()
predictions_dict_final = convert_dict_for_serializer(predictions)
serializer_result = PredictionsSerializer(data=predictions_dict_final)
if serializer_result.is_valid():
serializer_result.save()
else:
logger.debug("Error with data {}.".format(serializer_result.errors))
When I run it locally after running the commands python3 manage.py runserver and python3 migrate --database postgres, it saves the results to postgres.
However on production I am getting the following error, when the function is executed:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 413, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: unrecognized token: ":"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 108, in debug_sql
yield
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 98, in execute
return super().execute(sql, params)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 66, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python3.8/site-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 413, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: unrecognized token: ":"
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47, in inner
response = get_response(request)
File "/usr/local/lib/python3.8/site-packages/django/core/handlers/base.py", line 181, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/usr/local/lib/python3.8/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/views/generic/base.py", line 70, in view
return self.dispatch(request, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/rest_framework/views.py", line 509, in dispatch
response = self.handle_exception(exc)
File "/usr/local/lib/python3.8/site-packages/rest_framework/views.py", line 469, in handle_exception
self.raise_uncaught_exception(exc)
File "/usr/local/lib/python3.8/site-packages/rest_framework/views.py", line 480, in raise_uncaught_exception
raise exc
File "/usr/local/lib/python3.8/site-packages/rest_framework/views.py", line 506, in dispatch
response = handler(request, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/rest_framework/decorators.py", line 50, in handler
return func(*args, **kwargs)
File "/anomaly_detection/views.py", line 23, in trigger_prediction_cronjob
prediction_job()
File "/anomaly_detection/access_logs_modeling.py", line 85, in prediction_job
serializer_result.save()
File "/usr/local/lib/python3.8/site-packages/rest_framework/serializers.py", line 205, in save
self.instance = self.create(validated_data)
File "/usr/local/lib/python3.8/site-packages/rest_framework/serializers.py", line 939, in create
instance = ModelClass._default_manager.create(**validated_data)
File "/usr/local/lib/python3.8/site-packages/django/db/models/manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/models/query.py", line 447, in create
obj.save(force_insert=True, using=self.db)
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 753, in save
self.save_base(using=using, force_insert=force_insert,
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 790, in save_base
updated = self._save_table(
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 895, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 933, in _do_insert
return manager._insert(
File "/usr/local/lib/python3.8/site-packages/django/db/models/manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/models/query.py", line 1254, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1397, in execute_sql
cursor.execute(sql, params)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 98, in execute
return super().execute(sql, params)
File "/usr/local/lib/python3.8/contextlib.py", line 131, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 113, in debug_sql
sql = self.db.ops.last_executed_query(self.cursor, sql, params)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/sqlite3/operations.py", line 155, in last_executed_query
params = self._quote_params_for_last_executed_query(params)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/sqlite3/operations.py", line 144, in _quote_params_for_last_executed_query
return cursor.execute(sql, params).fetchone()
sqlite3.InterfaceError: Error binding parameter 2 - probably unsupported type.

I have deleted a table from models but django still looking for it

I have deleted a model name controlapproval from my database by commenting the model and
Running python manage.py makemigrations
but when i try to migrate by using python manage.py migrate
It throws an error as below
Traceback (most recent call last):
File "manage.py", line 23, in <module>
main()
File "manage.py", line 19, in main
execute_from_command_line(sys.argv)
File "E:\FBR\FBRENV\lib\site-packages\django\core\management\__init__.py", line 381, in execute_from_command_line
utility.execute()
File "E:\FBR\FBRENV\lib\site-packages\django\core\management\__init__.py", line 375, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "E:\FBR\FBRENV\lib\site-packages\django\core\management\base.py", line 323, in run_from_argv
self.execute(*args, **cmd_options)
File "E:\FBR\FBRENV\lib\site-packages\django\core\management\base.py", line 364, in execute
output = self.handle(*args, **options)
File "E:\FBR\FBRENV\lib\site-packages\django\core\management\base.py", line 83, in wrapped
res = handle_func(*args, **kwargs)
File "E:\FBR\FBRENV\lib\site-packages\django\core\management\commands\migrate.py", line 234, in handle
fake_initial=fake_initial,
File "E:\FBR\FBRENV\lib\site-packages\django\db\migrations\executor.py", line 117, in migrate
state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)
File "E:\FBR\FBRENV\lib\site-packages\django\db\migrations\executor.py", line 147, in _migrate_all_forwards
state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)
File "E:\FBR\FBRENV\lib\site-packages\django\db\migrations\executor.py", line 245, in apply_migration
state = migration.apply(state, schema_editor)
File "E:\FBR\FBRENV\lib\site-packages\django\db\migrations\migration.py", line 124, in apply
operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
File "E:\FBR\FBRENV\lib\site-packages\django\db\migrations\operations\models.py", line 261, in database_forwards
schema_editor.delete_model(model)
File "E:\FBR\FBRENV\lib\site-packages\django\db\backends\sqlite3\schema.py", line 308, in delete_model
super().delete_model(model)
File "E:\FBR\FBRENV\lib\site-packages\django\db\backends\base\schema.py", line 326, in delete_model
"table": self.quote_name(model._meta.db_table),
File "E:\FBR\FBRENV\lib\site-packages\django\db\backends\base\schema.py", line 137, in execute
cursor.execute(sql, params)
File "E:\FBR\FBRENV\lib\site-packages\django\db\backends\utils.py", line 99, in execute
return super().execute(sql, params)
File "E:\FBR\FBRENV\lib\site-packages\django\db\backends\utils.py", line 67, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "E:\FBR\FBRENV\lib\site-packages\django\db\backends\utils.py", line 76, in _execute_with_wrappers
return executor(sql, params, many, context)
File "E:\FBR\FBRENV\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "E:\FBR\FBRENV\lib\site-packages\django\db\utils.py", line 89, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "E:\FBR\FBRENV\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "E:\FBR\FBRENV\lib\site-packages\django\db\backends\sqlite3\base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: no such table: assessment_controlapproval
Any help or guidance will be a great help
Delete your migrations folder from your project root.
Create a new migrations folder.
Add a __init__.py empty file in that folder.
Run python manage.py makemigrations and python manage.py migrate.
This will just create a fresh migrations for your project.

Airflow scheduler works normally, fails with -D

I have set up Airflow on an AWS EC2 server with Ubuntu 18.04, Python 3.6.9. The DB backend is a 5.7.26 MySQL, I am using a LocalExecutor. The setup is nothing more than:
apt-get install python3 python3-pip python3-venv libmysqlclient-dev
I install Airflow with pip3 install apache-airflow[mysql]==1.10.9, init DB connection, start the webserver and it works both normally and with -D. The scheduler, however, works only when run in the foreground. Trying to run it as a daemon fails with the following trace:
Traceback (most recent call last):
File "/usr/local/bin/airflow", line 37, in <module> args.func(args)
File "/usr/local/lib/python3.6/dist-packages/airflow/utils/cli.py", line 75, in wrapper return f(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/bin/cli.py", line 1032, in schedulerjob.run()
File "/usr/local/lib/python3.6/dist-packages/airflow/jobs/base_job.py", line 215, in run session.commit()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 1036, in commit self.transaction.commit()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 503, in commitself._prepare_impl()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 482, in _prepare_impl self.session.flush()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 2496, in flushself._flush(objects)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 2637, in _flush transaction.rollback(_capture_exception=True)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/util/langhelpers.py", line 69, in __exit__exc_value, with_traceback=exc_tb,
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/util/compat.py", line 178, in raise_raise exception
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 2597, in _flush flush_context.execute()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute rec.execute(self)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute uow,
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/persistence.py", line 213, in save_obj) in _organize_states_for_save(base_mapper, states, uowtransaction):
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/persistence.py", line 374, in _organize_states_for_save base_mapper, uowtransaction, states
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/persistence.py", line 1602, in _connections_for_states connection = uowtransaction.transaction.connection(base_mapper)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 313, in connection return self._connection_for_bind(bind, execution_options)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 420, in _connection_for_bind conn = self._parent._connection_for_bind(bind, execution_options)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/session.py", line 432, in _connection_for_bind conn = bind._contextual_connect()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 2251, in _contextual_connect self._wrap_pool_connect(self.pool.connect, None),
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 2285, in _wrap_pool_connect return fn()
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 363, in connect return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 804, in _checkout result = pool._dialect.do_ping(fairy.connection)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 138, in do_pingdbapi_connection.ping(False)
File "/usr/local/lib/python3.6/dist-packages/pymysql/connections.py", line 546, in ping self._execute_command(COMMAND.COM_PING, "")
File "/usr/local/lib/python3.6/dist-packages/pymysql/connections.py", line 771, in _execute_command self._write_bytes(packet)
File "/usr/local/lib/python3.6/dist-packages/pymysql/connections.py", line 711, in _write_bytesself._sock.settimeout(self._write_timeout)
OSError: [Errno 9] Bad file descriptor
Exception ignored in: <function _ConnectionRecord.checkout.<locals>.<lambda> at 0x7f2eb025e268>
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 503, in <lambda>
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool/base.py", line 702, in _finalize_fairy
File "/usr/lib/python3.6/logging/__init__.py", line 1337, in error
File "/usr/lib/python3.6/logging/__init__.py", line 1444, in _log
File "/usr/lib/python3.6/logging/__init__.py", line 1454, in handle
File "/usr/lib/python3.6/logging/__init__.py", line 1516, in callHandlers
File "/usr/lib/python3.6/logging/__init__.py", line 865, in handle
File "/usr/lib/python3.6/logging/__init__.py", line 1071, in emit
File "/usr/lib/python3.6/logging/__init__.py", line 1061, in _open
NameError: name 'open' is not defined
I am trying to set up a server for semi-production purposes, so this is actually a blocker for me. I would be grateful for any advice.
EDIT
I tried using mysql-connector-python==8.0.18 and the scheduler did run as a daemon, but an attempt to open a dag failed with the following trace:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python3.6/dist-packages/flask/_compat.py", line 39, in reraise
raise value
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python3.6/dist-packages/flask_admin/base.py", line 69, in inner
return self._run_view(f, *args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/flask_admin/base.py", line 368, in _run_view
return fn(self, *args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/flask_login/utils.py", line 258, in decorated_view
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/www/utils.py", line 384, in view_func
return f(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/www/utils.py", line 290, in wrapper
return f(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/utils/db.py", line 74, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/www/views.py", line 1559, in tree
start_date=min_date, end_date=base_date, session=session)
File "/usr/local/lib/python3.6/dist-packages/airflow/utils/db.py", line 70, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/airflow/models/dag.py", line 837, in get_task_instances
tis = tis.order_by(TaskInstance.execution_date).all()
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3233, in all
return list(self)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3389, in __iter__
return self._execute_and_instances(context)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3414, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 982, in execute
return meth(self, multiparams, params)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 293, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1101, in _execute_clauseelement
distilled_params,
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
e, statement, parameters, cursor, context
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1478, in _handle_dbapi_exception
util.reraise(*exc_info)
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/home/ubuntu/.local/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 588, in do_execute
cursor.execute(statement, parameters)
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor_cext.py", line 272, in execute
self._handle_result(result)
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor_cext.py", line 163, in _handle_result
self._handle_resultset()
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor_cext.py", line 651, in _handle_resultset
self._rows = self._cnx.get_rows()[0]
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/connection_cext.py", line 301, in get_rows
else self._cmysql.fetch_row()
SystemError: <built-in method fetch_row of _mysql_connector.MySQL object at 0x3cdaa10> returned a result with an error set
EDIT 2
What finally did work was using mysqlclient by specifying the SQLAlchemy connection string to begin with mysql+mysqldb. I am not giving this as an answer, because the initial problem persists.

Can't install python modules with pip on Manjaro Linux

I'm on Manjaro Linux KDE and every time I try to install a module with pip it gives me this error:
Defaulting to user installation because normal site-packages is not writeable
ERROR:
Exception:
Traceback (most recent call last):
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/base_command.py", line 186, in _main
status = self.run(options, args)
File "/usr/lib/python3.8/site-packages/pip/_internal/commands/install.py", line 279, in run
session = self.get_default_session(options)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 74, in get_default_session
self._session = self.enter_context(self._build_session(options))
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 84, in _build_session
session = PipSession(
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 241, in __init__
self.headers["User-Agent"] = user_agent()
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 132, in user_agent
zip(["name", "version", "id"], distro.linux_distribution()),
File "/usr/lib/python3.8/site-packages/distro.py", line 122, in linux_distribution
return _distro.linux_distribution(full_distribution_name)
File "/usr/lib/python3.8/site-packages/distro.py", line 677, in linux_distribution
self.version(),
File "/usr/lib/python3.8/site-packages/distro.py", line 737, in version
self.lsb_release_attr('release'),
File "/usr/lib/python3.8/site-packages/distro.py", line 899, in lsb_release_attr
return self._lsb_release_info.get(attribute, '')
File "/usr/lib/python3.8/site-packages/distro.py", line 552, in __get__
ret = obj.__dict__[self._fname] = self._f(obj)
File "/usr/lib/python3.8/site-packages/distro.py", line 1012, in _lsb_release_info
stdout = subprocess.check_output(cmd, stderr=devnull)
File "/usr/lib/python3.8/subprocess.py", line 411, in check_output
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 145, in __new__
process = cls.create_process(command, stdin, cwd=cwd, env=env, shell=shell)
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 121, in create_process
shlex.split(command),
File "/usr/lib/python3.8/shlex.py", line 311, in split
return list(lex)
File "/usr/lib/python3.8/shlex.py", line 300, in __next__
token = self.get_token()
File "/usr/lib/python3.8/shlex.py", line 109, in get_token
raw = self.read_token()
File "/usr/lib/python3.8/shlex.py", line 140, in read_token
nextchar = self.instream.read(1)
AttributeError: 'tuple' object has no attribute 'read'
Traceback (most recent call last):
File "/usr/bin/pip3", line 11, in <module>
load_entry_point('pip==20.0.2', 'console_scripts', 'pip3')()
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/main.py", line 75, in main
return command.main(cmd_args)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/base_command.py", line 105, in main
return self._main(args)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/base_command.py", line 224, in _main
self.handle_pip_version_check(options)
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 143, in handle_pip_version_check
session = self._build_session(
File "/usr/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 84, in _build_session
session = PipSession(
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 241, in __init__
self.headers["User-Agent"] = user_agent()
File "/usr/lib/python3.8/site-packages/pip/_internal/network/session.py", line 132, in user_agent
zip(["name", "version", "id"], distro.linux_distribution()),
File "/usr/lib/python3.8/site-packages/distro.py", line 122, in linux_distribution
return _distro.linux_distribution(full_distribution_name)
File "/usr/lib/python3.8/site-packages/distro.py", line 677, in linux_distribution
self.version(),
File "/usr/lib/python3.8/site-packages/distro.py", line 737, in version
self.lsb_release_attr('release'),
File "/usr/lib/python3.8/site-packages/distro.py", line 899, in lsb_release_attr
return self._lsb_release_info.get(attribute, '')
File "/usr/lib/python3.8/site-packages/distro.py", line 552, in __get__
ret = obj.__dict__[self._fname] = self._f(obj)
File "/usr/lib/python3.8/site-packages/distro.py", line 1012, in _lsb_release_info
stdout = subprocess.check_output(cmd, stderr=devnull)
File "/usr/lib/python3.8/subprocess.py", line 411, in check_output
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 145, in __new__
process = cls.create_process(command, stdin, cwd=cwd, env=env, shell=shell)
File "/usr/lib/python3.8/site-packages/run/__init__.py", line 121, in create_process
shlex.split(command),
File "/usr/lib/python3.8/shlex.py", line 311, in split
return list(lex)
File "/usr/lib/python3.8/shlex.py", line 300, in __next__
token = self.get_token()
File "/usr/lib/python3.8/shlex.py", line 109, in get_token
raw = self.read_token()
File "/usr/lib/python3.8/shlex.py", line 140, in read_token
nextchar = self.instream.read(1)
AttributeError: 'tuple' object has no attribute 'read'
Also, when I try to run a program where I imported selenium or any other installed module from pip it gives me the same error.
This is my pip version: pip 20.0.2 from /usr/lib/python3.8/site-packages/pip (python 3.8)
Thank you all in advance!

Resources