deleted spark checkpoint directory and now it doesn't work - apache-spark

Question

I deleted the checkpoint directory for my spark stream.
Now, there are no errors, but the stream doesn't pick up any files.
How can I fix my stupid mistake? :)
I have tried to create a new checkpoint directory & changing the queryname but it's not helped
Below is the code that I have implemented.
I don't understand why it doesn't just make a new directory?
CODE
#!/usr/bin/env python
#nohup spark-submit --master local --driver-memory 1g --executor-memory 1g streaming_log_monitor.py >streammon.log 2>stderr.log &
from pyspark.sql import SparkSession
import pyspark.sql.functions as sqlfunc
import argparse, sys
from pyspark.sql import *
from pyspark.sql.functions import *
from datetime import datetime
from pyspark.sql.functions import lit
from pyspark.sql.types import *
from pyspark.sql.functions import udf
from pyspark.sql.types import *
from pyspark.sql.functions import udf, input_file_name, lower
from pyspark.streaming import StreamingContext
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
now = datetime.now()
#create a contexit that supports hive
def create_session(appname):
spark_session = SparkSession\
.builder\
.appName(appname)\
.master('local')\
.enableHiveSupport()\
.getOrCreate()
return spark_session
### START MAIN ###
if __name__ == '__main__':
spark_session = create_session('streaming_monitor')
ssc = StreamingContext(spark_session, 1)
print('start')
print(datetime.now())
myschema = StructType([
StructField('text', StringType())
])
#only files after stream starts
df = spark_session\
.readStream\
.option('newFilesOnly', 'true')\
.option('header', 'true')\
.schema(myschema)\
.text('file:///home/keenek1/analytics/logs/')\
.withColumn("FileName", input_file_name())
def errorcapture(text):
try:
text = str(text).lower()
if 'cannot obtain block length for locatedblock' in text:
return 'error: Cannot obtain block length for LocatedBlock'
elif 'outofmemoryerror' in text:
return 'error: OutOfMemoryError'
elif 'gc overhead limit exceeded' in text:
return 'error: OutOfMemoryError (GC Overhead Limit Exceeded)'
elif 'o3' in text:
return 'error o3: an UnsupportedEncodingException occurred when setting up stdout and stderr streams.'
elif 'o10' in text:
return 'error o10: an uncaught exception occurred'
elif 'o11' in text:
return 'error o11: more than spark.yarn.scheduler.reporterThread.maxFailures executor failures occurred'
elif 'o13' in text:
return 'error o13: the program terminated before the user had initialized the spark context or if the spark context did not initialize before a timeout.'
elif 'o14' in text:
return 'error o14: This is declared as EXIT_SECURITY but never used'
elif 'o15' in text:
return 'error o15: a user class threw an exception'
elif 'o16' in text:
return 'error o16: the shutdown hook called before final status was reported.'
elif 'o52' in text:
return 'error o52: The default uncaught exception handler was reached, and the uncaught exception was an OutOfMemoryError'
elif 'o53' in text:
return 'error o53: DiskStore failed to create local temporary directory after many attempts (bad spark.local.dir?)'
elif 'o54' in text:
return 'error o54: ExternalBlockStore failed to initialize after many attempts'
elif 'o55' in text:
return 'error o55: ExternalBlockStore failed to create a local temporary directory after many attempts'
elif 'o56' in text:
return 'error o56: Executor is unable to send heartbeats to the driver more than "spark.executor.heartbeat.maxFailures" times.'
elif 'array index out of bounds' in text:
return 'error Array Index Out of Bounds'
elif 'string index out of bounds' in text:
return 'error Array Index Out of Bounds'
elif 'error' in text:
return 'Unidentified Error'
else:
return 'Success'
except AttributeError:
return text
except UnicodeEncodeError:
return text
def errorsolution(text):
try:
text = str(text).lower()
if 'cannot obtain block length for locatedblock' in text:
return 'Find and resolve block issue'
elif 'outofmemoryerror' in text:
return 'Increase memory limit using --driver-memory 10g --executor-memory 10g in spark-submit'
elif 'gc overhead limit exceeded' in text:
return 'Increase memory limit using --driver-memory 10g --executor-memory 10g in spark-submit'
elif 'total size of serialized results of' in text:
return 'use this parameter in Spark-Submit --conf spark.driver.maxResultSize=0'
else:
return 'Unknown Solution'
except AttributeError:
return text
except UnicodeEncodeError:
return text
udfdict = udf(errorcapture, StringType())
errorsolutionudf = udf(errorsolution, StringType())
df = df.withColumn('did_it_error',udfdict(df.text))
df = df.withColumn('solution',errorsolutionudf(df.text))
from datetime import datetime
now = datetime.now()
output = df.createOrReplaceTempView('log')
hive_dump = spark_session.sql("select '" + str(now) + "' as timestamp, FileName, did_it_error, solution, text from log")
output = hive_dump\
.writeStream\
.format("csv")\
.queryName('logsmonitor')\
.option("checkpointLocation", "file:///home/keenek1/analytics/logs/chkpoint_dir")\
.start('/user/hive/warehouse/design.db/streaming_log_monitor')\
.awaitTermination()

Answer 1

Check your cluster has trash is enabled or not. If it enabled a file or directory is deleted that file or directory will be moved to the .Trash directory in the user’s home directory instead of being deleted.
Try using below path to check if your checkpoint directory is available or not.
hdfs://<host>/user/<username>/.Trash/Current/<your_checkpoint_path>