Unable to Call Dask Cluster from Airflow - python-3.x

Question

We're trying to execute a Dask DAG as part of an airflow scheduled job (Airflow only really being used here as the scheduler). I'm using the example from Dasks own docs as it gives us the same error.
Pickling has been disabled since that seemed an obvious solution, but it has no effect.
We're also using a DaskExecutor with airflow which could be part of the issue here.
So, at this point we're at a bit of a loss on how to get this working since docs and prior work are light on this.
def fib(n):
if n < 2:
return n
client = get_client()
a_future = client.submit(fib, n - 1)
b_future = client.submit(fib, n - 2)
secede()
a, b = client.gather([a_future, b_future])
rejoin()
return a + b
#dag(default_args=default_args,
schedule_interval="0 21 * * *",
tags=['test'])
def dask_future():
#task
def go_to_the_future():
print("Doing fibonacci calculation")
# these features require the dask.distributed scheduler
client = Client(DASK_CLUSTER_IP)
future = client.submit(fib, 10)
result = future.result()
print(result) # prints "55"
go_to_the_future()
dask_future = dask_future()
However we get this error:
[2022-04-13, 15:26:00 CEST] {taskinstance.py:1270} INFO - Marking task as UP_FOR_RETRY. dag_id=dask_future, task_id=go_to_the_future, execution_date=20220413T132555, start_date=20220413T132559, end_date=20220413T132600
[2022-04-13, 15:26:00 CEST] {standard_task_runner.py:88} ERROR - Failed to execute job 368 for task go_to_the_future
Traceback (most recent call last):
File "/data/venv/lib64/python3.9/site-packages/airflow/task/task_runner/standard_task_runner.py", line 85, in _start_by_fork
args.func(args, dag=self.dag)
File "/data/venv/lib64/python3.9/site-packages/airflow/cli/cli_parser.py", line 48, in command
return func(*args, **kwargs)
File "/data/venv/lib64/python3.9/site-packages/airflow/utils/cli.py", line 92, in wrapper
return f(*args, **kwargs)
File "/data/venv/lib64/python3.9/site-packages/airflow/cli/commands/task_command.py", line 292, in task_run
_run_task_by_selected_method(args, dag, ti)
File "/data/venv/lib64/python3.9/site-packages/airflow/cli/commands/task_command.py", line 107, in _run_task_by_selected_method
_run_raw_task(args, ti)
File "/data/venv/lib64/python3.9/site-packages/airflow/cli/commands/task_command.py", line 180, in _run_raw_task
ti._run_raw_task(
File "/data/venv/lib64/python3.9/site-packages/airflow/utils/session.py", line 70, in wrapper
return func(*args, session=session, **kwargs)
File "/data/venv/lib64/python3.9/site-packages/airflow/models/taskinstance.py", line 1332, in _run_raw_task
self._execute_task_with_callbacks(context)
File "/data/venv/lib64/python3.9/site-packages/airflow/models/taskinstance.py", line 1458, in _execute_task_with_callbacks
result = self._execute_task(context, self.task)
File "/data/venv/lib64/python3.9/site-packages/airflow/models/taskinstance.py", line 1514, in _execute_task
result = execute_callable(context=context)
File "/data/venv/lib64/python3.9/site-packages/airflow/decorators/base.py", line 134, in execute
return_value = super().execute(context)
File "/data/venv/lib64/python3.9/site-packages/airflow/operators/python.py", line 151, in execute
return_value = self.execute_callable()
File "/data/venv/lib64/python3.9/site-packages/airflow/operators/python.py", line 162, in execute_callable
return self.python_callable(*self.op_args, **self.op_kwargs)
File "/data/airflow/dags/dask_future.py", line 77, in go_to_the_future
result = future.result()
File "/data/venv/lib64/python3.9/site-packages/distributed/client.py", line 279, in result
raise exc.with_traceback(tb)
File "/data/venv/lib64/python3.9/site-packages/distributed/protocol/pickle.py", line 66, in loads
return pickle.loads(x)
ModuleNotFoundError: No module named 'unusual_prefix_876615abe2868d0d67afd03ccd8a249a86eb44dc_dask_future'
[2022-04-13, 15:26:00 CEST] {local_task_job.py:154} INFO - Task exited with return code 1