Having fatal error with rdd.first( ) command in pyspark - apache-spark

Question

Having trouble executing the following code, encountering error with first() command only. If, I run df1.rdd.collect(), it runs without any error:
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import urllib
from pyspark.sql.types import StructType,StructField,DoubleType
from pyspark import SparkContext
spark = SparkSession \
.builder \
.getOrCreate()
df = spark.read.csv('subset2009_5000.csv',header=True,inferSchema=True)
df1 = df.select(['Start_Lat','Start_Lon']).dropDuplicates(['Start_Lat','Start_Lon'])
df1.rdd.first() # Fatal Error
I'm getting a fatal error. Running pypspark in local mode. Driver memory is 3GB and subset2009_5000.csv size is 3.3MB.
I don't understand why I'm getting this error with .first() only? What is wrong?
#
# A fatal error has been detected by the Java Runtime Environment:
#
# SIGSEGV (0xb) at pc=0x00007fb4a33c37b4, pid=23702, tid=23966
#
# JRE version: OpenJDK Runtime Environment (11.0.13+8) (build 11.0.13+8-Ubuntu-0ubuntu1.20.04)
# Java VM: OpenJDK 64-Bit Server VM (11.0.13+8-Ubuntu-0ubuntu1.20.04, mixed mode, sharing, tiered, compressed oops, g1 gc, linux-amd64)
# Problematic frame:
# V [libjvm.so+0xe4f7b4]
#
# Core dump will be written. Default location: Core dumps may be processed with "/usr/share/apport/apport %p %s %c %d %P %E" (or dumping to /home/agam/core.23702)
#
# An error report file with more information is saved as:
# /home/agam/hs_err_pid23702.log
Compiled method (nm) 181573 760 n 0 jdk.internal.misc.Unsafe::getInt (native)
total in heap [0x00007fb48c370a90,0x00007fb48c370e40] = 944
relocation [0x00007fb48c370c08,0x00007fb48c370c38] = 48
main code [0x00007fb48c370c40,0x00007fb48c370e40] = 512
Compiled method (nm) 181573 760 n 0 jdk.internal.misc.Unsafe::getInt (native)
total in heap [0x00007fb48c370a90,0x00007fb48c370e40] = 944
relocation [0x00007fb48c370c08,0x00007fb48c370c38] = 48
main code [0x00007fb48c370c40,0x00007fb48c370e40] = 512
Compiled method (nm) 181576 760 n 0 jdk.internal.misc.Unsafe::getInt (native)
total in heap [0x00007fb48c370a90,0x00007fb48c370e40] = 944
relocation [0x00007fb48c370c08,0x00007fb48c370c38] = 48
main code [0x00007fb48c370c40,0x00007fb48c370e40] = 512
#
# If you would like to submit a bug report, please visit:
# https://bugs.launchpad.net/ubuntu/+source/openjdk-lts
#
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 50652)
Traceback (most recent call last):
File "/usr/lib/python3.8/socketserver.py", line 316, in _handle_request_noblock
self.process_request(request, client_address)
File "/usr/lib/python3.8/socketserver.py", line 347, in process_request
self.finish_request(request, client_address)
File "/usr/lib/python3.8/socketserver.py", line 360, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python3.8/socketserver.py", line 747, in __init__
self.handle()
File "/home/agam/.local/lib/python3.8/site-packages/pyspark/accumulators.py", line 260, in handle
poll(authenticate_and_accum_updates)
File "/home/agam/.local/lib/python3.8/site-packages/pyspark/accumulators.py", line 235, in poll
if func():
File "/home/agam/.local/lib/python3.8/site-packages/pyspark/accumulators.py", line 256, in authenticate_and_accum_updates
raise ValueError(
ValueError: The value of the provided token to the AccumulatorServer is not correct.
----------------------------------------
ERROR:root:Exception while sending command.
Traceback (most recent call last):
File "/home/agam/.local/lib/python3.8/site-packages/py4j/clientserver.py", line 480, in send_command
raise Py4JNetworkError("Answer from Java side is empty")
py4j.protocol.Py4JNetworkError: Answer from Java side is empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/agam/.local/lib/python3.8/site-packages/py4j/java_gateway.py", line 1038, in send_command
response = connection.send_command(command)
File "/home/agam/.local/lib/python3.8/site-packages/py4j/clientserver.py", line 503, in send_command
raise Py4JNetworkError(
py4j.protocol.Py4JNetworkError: Error while sending or receiving
---------------------------------------------------------------------------
Py4JError Traceback (most recent call last)
/tmp/ipykernel_23686/3841479094.py in <module>
1 df1 = df.select(['Start_Lat','Start_Lon']).dropDuplicates(['Start_Lat','Start_Lon'])
----> 2 df1.rdd.first()
~/.local/lib/python3.8/site-packages/pyspark/rdd.py in first(self)
1586 ValueError: RDD is empty
1587 """
-> 1588 rs = self.take(1)
1589 if rs:
1590 return rs[0]
~/.local/lib/python3.8/site-packages/pyspark/rdd.py in take(self, num)
1566
1567 p = range(partsScanned, min(partsScanned + numPartsToTry, totalParts))
-> 1568 res = self.context.runJob(self, takeUpToNumLeft, p)
1569
1570 items += res
~/.local/lib/python3.8/site-packages/pyspark/context.py in runJob(self, rdd, partitionFunc, partitions, allowLocal)
1225 # SparkContext#runJob.
1226 mappedRDD = rdd.mapPartitions(partitionFunc)
-> 1227 sock_info = self._jvm.PythonRDD.runJob(self._jsc.sc(), mappedRDD._jrdd, partitions)
1228 return list(_load_from_socket(sock_info, mappedRDD._jrdd_deserializer))
1229
~/.local/lib/python3.8/site-packages/py4j/java_gateway.py in __call__(self, *args)
1307
1308 answer = self.gateway_client.send_command(command)
-> 1309 return_value = get_return_value(
1310 answer, self.gateway_client, self.target_id, self.name)
1311
~/.local/lib/python3.8/site-packages/pyspark/sql/utils.py in deco(*a, **kw)
109 def deco(*a, **kw):
110 try:
--> 111 return f(*a, **kw)
112 except py4j.protocol.Py4JJavaError as e:
113 converted = convert_exception(e.java_exception)
~/.local/lib/python3.8/site-packages/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
332 format(target_id, ".", name, value))
333 else:
--> 334 raise Py4JError(
335 "An error occurred while calling {0}{1}{2}".
336 format(target_id, ".", name))
Py4JError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.runJob