This SparkContext may be an existing one. error - apache-spark

Question

I'm using spark2.0 in notebook, this is the initial set up:
spark = SparkSession.builder \
.appName("NewApp") \
.config("spark.driver.maxResultSize", "600g") \
.config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") \
.config("spark.rpc.message.maxSize",10737418240) \
.config("spark.executor.heartbeatInterval",10000000) \
.getOrCreate()
/usr/local/spark-2.0.1/python/pyspark/sql/session.py in getOrCreate(self)
167 for key, value in self._options.items():
168 sparkConf.set(key, value)
--> 169 sc = SparkContext.getOrCreate(sparkConf)
170 # This SparkContext may be an existing one.
171 for key, value in self._options.items():
/usr/local/spark-2.0.1/python/pyspark/context.py in getOrCreate(cls, conf)
292 with SparkContext._lock:
293 if SparkContext._active_spark_context is None:
--> 294 SparkContext(conf=conf or SparkConf())
295 return SparkContext._active_spark_context
296
/usr/local/spark-2.0.1/python/pyspark/context.py in __init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
113 try:
114 self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,
--> 115 conf, jsc, profiler_cls)
116 except:
117 # If an error occurs, clean up in order to allow future SparkContext creation:
/usr/local/spark-2.0.1/python/pyspark/context.py in _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls)
166
167 # Create the Java SparkContext through Py4J
--> 168 self._jsc = jsc or self._initialize_context(self._conf._jconf)
169 # Reset the SparkConf to the one actually used by the SparkContext in JVM.
170 self._conf = SparkConf(_jconf=self._jsc.sc().conf())
/usr/local/spark-2.0.1/python/pyspark/context.py in _initialize_context(self, jconf)
231 Initialize SparkContext in function to allow subclass specific initialization
232 """
--> 233 return self._jvm.JavaSparkContext(jconf)
234
235 #classmethod
/usr/local/spark-2.0.1/python/lib/py4j-0.10.3-src.zip/py4j/java_gateway.py in __call__(self, *args)
1399 answer = self._gateway_client.send_command(command)
1400 return_value = get_return_value(
-> 1401 answer, self._gateway_client, None, self._fqn)
1402
1403 for temp_arg in temp_args:
/usr/local/spark-2.0.1/python/lib/py4j-0.10.3-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
317 raise Py4JJavaError(
318 "An error occurred while calling {0}{1}{2}.\n".
--> 319 format(target_id, ".", name), value)
320 else:
321 raise Py4JError(
Py4JJavaError: An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.
: java.lang.NumberFormatException: For input string: "10737418240"
at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
at java.lang.Integer.parseInt(Integer.java:583)
at java.lang.Integer.parseInt(Integer.java:615)
at scala.collection.immutable.StringLike$class.toInt(StringLike.scala:272)
at scala.collection.immutable.StringOps.toInt(StringOps.scala:29)
at org.apache.spark.SparkConf$$anonfun$getInt$2.apply(SparkConf.scala:375)
at org.apache.spark.SparkConf$$anonfun$getInt$2.apply(SparkConf.scala:375)
at scala.Option.map(Option.scala:146)
at org.apache.spark.SparkConf.getInt(SparkConf.scala:375)
at org.apache.spark.util.RpcUtils$.maxMessageSizeBytes(RpcUtils.scala:61)
at org.apache.spark.MapOutputTrackerMaster.<init>(MapOutputTracker.scala:293)
at org.apache.spark.SparkEnv$.create(SparkEnv.scala:284)
at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:165)
at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:256)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:420)
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
how could I solve this problem? I tried SparkContext.stop(), but it gives:
TypeError: stop() missing 1 required positional argument: 'self'
Another one question is my initial set up is getOrCreate() to my understanding if there is one then get it, if not create it, it still give this problem.

Answer 1

Here is the source of the error:
: java.lang.NumberFormatException: For input string: "10737418240"
at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
at java.lang.Integer.parseInt(Integer.java:583)
10737418240 is larger than Int.MaxValue(2147483647). Use smaller value when calling:
.config("spark.rpc.message.maxSize", ...) \

Answer 2

Try to fix JAVA_HOME and restart cmd.