Method writeStream does not exist - apache-spark

Question

I'm trying to run pyspark readStream/writeStream using python multiprocessing as a databricks job. Not directly from the notebook but saving the script in .py file inside dbfs:. I can run the readStream/writeStream from notebook but when I try to do the same from the file it throws Method writeStream([]) does not exist
Code
import multiprocessing
import time
from pyspark.sql import SparkSession
def _get_c3_value(df, epochId, v):
print('inside get_c3')
A = df.collect()[0][0]
v.value = A
print(A)
def _readStream_c3(v):
spark = SparkSession.builder.getOrCreate()
df = (spark.readStream.format('delta').option('readChangeFeed','true').option('startingVersion','latest').option('_change_type', 'insert').load("dbfs:/mnt/path/to/file").writeStream.foreachBatch(lambda df, epochId: _get_c3_value(df, epochId,v)).outputMode('append').start())
df.awaitTermination()
if __name__ == '__main__':
spark = SparkSession.builder.appName('multi-processing'
).master('spark://1.2.3.4:8077').getOrCreate()
for n in spark.sparkContext.getConf().getAll():
print (n)
v = multiprocessing.Value('d', 0.0)
p1 = multiprocessing.Process(target=_readStream_c3, args=(v, ))
p1.start()
p1.join()
StdOut
Process Process-1:
Traceback (most recent call last):
File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/tmp/tmp3syog799.py", line 22, in _readStream_c3
df = (spark.readStream.format('delta').option('readChangeFeed','true').option('startingVersion','latest').option('_change_type', 'insert').load("dbfs:/mnt/path/to/file").writeStream.foreachBatch(lambda df, epochId: _get_c3_value(df, epochId)).outputMode('append').start())
File "/databricks/spark/python/pyspark/sql/dataframe.py", line 272, in writeStream
return DataStreamWriter(self)
File "/databricks/spark/python/pyspark/sql/streaming.py", line 755, in __init__
self._jwrite = df._jdf.writeStream()
File "/databricks/spark/python/lib/py4j-0.10.9.1-src.zip/py4j/java_gateway.py", line 1304, in __call__
return_value = get_return_value(
File "/databricks/spark/python/pyspark/sql/utils.py", line 117, in deco
return f(*a, **kw)
File "/databricks/spark/python/lib/py4j-0.10.9.1-src.zip/py4j/protocol.py", line 330, in
get_return_value
raise Py4JError(
py4j.protocol.Py4JError: An error occurred while calling o1311.writeStream. Trace:
py4j.Py4JException: Method writeStream([]) does not exist
at py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:341)
at py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:349)
at py4j.Gateway.invoke(Gateway.java:286)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:251)
at java.lang.Thread.run(Thread.java:748)
StdErr
ERROR: Query termination received for [id=0bf6163e-797c-4918-a057-8a90fa1f9d2d, runId=679fb0b5-ec0c-456e-939b-c0ba4c34325f], with exception: py4j.Py4JException: An exception was raised by the Python Proxy. Return Message: Object ID unknown
at py4j.Protocol.getReturnValue(Protocol.java:476)
at py4j.reflection.PythonProxyHandler.invoke(PythonProxyHandler.java:108)
at com.sun.proxy.$Proxy102.call(Unknown Source)
at org.apache.spark.sql.execution.streaming.sources.PythonForeachBatchHelper$.$anonfun$callForeachBatch$1(ForeachBatchSink.scala:208)
at org.apache.spark.sql.execution.streaming.sources.PythonForeachBatchHelper$.$anonfun$callForeachBatch$1$adapted(ForeachBatchSink.scala:208)
at org.apache.spark.sql.execution.streaming.sources.ForeachBatchSink.addBatchLegacy(ForeachBatchSink.scala:89)
at org.apache.spark.sql.execution.streaming.sources.ForeachBatchSink.$anonfun$addBatch$1(ForeachBatchSink.scala:68)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.apache.spark.util.Utils$.timeTakenMs(Utils.scala:680)
at org.apache.spark.sql.execution.streaming.sources.ForeachBatchSink.addBatch(ForeachBatchSink.scala:58)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runBatch$17(MicroBatchExecution.scala:719)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withCustomExecutionEnv$8(SQLExecution.scala:239)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:386)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withCustomExecutionEnv$1(SQLExecution.scala:186)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:968)
at org.apache.spark.sql.execution.SQLExecution$.withCustomExecutionEnv(SQLExecution.scala:141)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:336)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runBatch$16(MicroBatchExecution.scala:717)
at org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:301)
at org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:299)
at org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:73)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.runBatch(MicroBatchExecution.scala:717)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runActivatedStream$5(MicroBatchExecution.scala:283)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.withSchemaEvolution(MicroBatchExecution.scala:816)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runActivatedStream$2(MicroBatchExecution.scala:280)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:301)
at org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:299)
at org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:73)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runActivatedStream$1(MicroBatchExecution.scala:239)
at org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.execute(TriggerExecutor.scala:67)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.runActivatedStream(MicroBatchExecution.scala:233)
at org.apache.spark.sql.execution.streaming.StreamExecution.$anonfun$runStream$1(StreamExecution.scala:359)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:968)
at org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:323)
at org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:250)
What am I doing wrong ?. I can run the streaming part fine in notebook but not as a python script.
Thanks in advance!