pyspark streaming from kinesis kills heap - apache-spark

Question

Running a sample application streaming data from kinesis. I did not get why this application uses so much heap and crashes.
Here is the code :
from __future__ import print_function
import sys
from pyspark.streaming import StreamingContext
from pyspark.streaming.kinesis import KinesisUtils, InitialPositionInStream
from pyspark.sql.session import SparkSession
from datetime import datetime
# function declaration
def isDfEmpty(df):
try:
if not df.take(1) :
return True
except Exception as e:
return True
return False
# function declaration
def mergeTable(df):
print("b:mergeTable")
print(str(datetime.now()))
try:
global refDf
if isDfEmpty(df) :
print("no record, waiting !")
else :
if(isDfEmpty(refDf)) :
refDf = df
else :
print(" before count %s" % refDf.count())
refDf = df.unionAll(refDf)
print(" after count %s" % refDf.count())
except Exception as e:
print(e)
print(str(datetime.now()))
print("e:mergeTable")
# function declaration
def doWork(df):
print("b:doWork")
print(str(datetime.now()))
try:
mergeTable(df)
except Exception as e:
print(e)
print(str(datetime.now()))
print("e:doWork")
# function declaration
def sensorFilter(sensorType, rdd):
df = spark.read.json(rdd.filter(lambda x : sensorType in x))
doWork(df)
def printRecord(rdd):
print("========================================================")
print("Starting new RDD")
print("========================================================")
sensorFilter("SensorData", rdd)
refDf = None
if __name__ == "__main__":
reload(sys)
# sys.setdefaultencoding('utf-8')
if len(sys.argv) != 5:
print( "Usage: dump.py <app-name> <stream-name> <endpoint-url> <region-name>", file=sys.stderr)
sys.exit(-1)
spark = SparkSession.builder.master("local[*]").getOrCreate()
sc = spark.sparkContext
# sc = SparkContext(appName="PythonStreamingKinesisWordCountAsl")
ssc = StreamingContext(sc, 10)
appName, streamName, endpointUrl, regionName = sys.argv[1:]
dstream = KinesisUtils.createStream(ssc, appName, streamName, endpointUrl, regionName, InitialPositionInStream.LATEST, 10)
dstream.foreachRDD(printRecord)
ssc.start()
ssc.awaitTermination()
After a time the spark application slowed down due to heap usage. But when i comment out the lines, heap usage decrease to normal levels.(According to SparkUI)
print(" before count %s" % refDf.count())
print(" after count %s" % refDf.count())
I am really new with pyspark and trying to get what is going on.
Merging on data frame continuously may explode the memory of course but the problem of heap occurs very beginning.
EDIT
Environment : Tried on single ubuntu and on cents VM hosted by macOS nothing changed.