spark cluster on kubernetes - apache-spark

Question

I'm trying to deploy spark on kubernetes.
Before that I've started Spark on standalone mode and it works, but now I've some troubles.
I'm using the same images to cluster mode as I used to standalone mode.
These images have been build based on miniconda because I needed to customize pacakges
Dockerfile spark master
---
FROM continuumio/miniconda3:latest
COPY master.sh /
COPY elasticsearch-hadoop-8.1.1/dist/* /spark/jars/
ENV PYSPARK_DRIVER_PYTHON=/opt/conda/bin/python
ENV PYSPARK_PYTHON=/opt/conda/bin/python
ENV PYTHONPATH=/opt/conda/bin/python
ENV SPARK_MASTER_PORT 7077
ENV SPARK_MASTER_WEBUI_PORT 8080
ENV SPARK_MASTER_LOG /spark/logs
RUN conda install -c conda-forge findspark
RUN conda install -c conda-forge openjdk
RUN conda install -c conda-forge elasticsearch==8.1.0
RUN conda install -c conda-forge pyspark==3.2.1
RUN conda install -c conda-forge pyarrow==8.0.0
RUN conda install -c conda-forge prophet
RUN conda install -c conda-forge pandas
EXPOSE 8080 7077 6066
CMD ["/bin/bash", "/master.sh"]
and the second one for worker... very similar
master.sh
export SPARK_MASTER_HOST=${SPARK_MASTER_HOST:-`hostname -i`}
export SPARK_HOME=/spark
. "/spark/sbin/spark-config.sh"
. "/spark/bin/load-spark-env.sh"
mkdir -p $SPARK_MASTER_LOG
ln -sf /dev/stdout $SPARK_MASTER_LOG/spark-master.out
cd /spark/bin && /spark/sbin/../bin/spark-class org.apache.spark.deploy.master.Master \
--ip $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT --deploy-mode cluster >> $SPARK_MASTER_LOG/spark-master.out
my yaml for making pod
kind: Deployment
apiVersion: apps/v1
metadata:
name: spark-master-deployment
namespace: spark
spec:
replicas: 1
selector:
matchLabels:
app: spark-master
template:
metadata:
labels:
app: spark-master
spec:
containers:
- name: spark-master
image: spark:master_conda_pyspark_pyarrow_script_v3
imagePullPolicy: Always
command: ["bash"]
args: ["-c",
"source /opt/conda/lib/python3.9/site-packages/pyspark/sbin/spark-config.sh &&\
source /opt/conda/lib/python3.9/site-packages/pyspark/bin/load-spark-env.sh &&\
mkdir -p $SPARK_MASTER_LOG &&\
ln -sf /dev/stdout $SPARK_MASTER_LOG/spark-master.out &&\
cd /opt/conda/lib/python3.9/site-packages/pyspark/bin &&\
echo $SPARK_MASTER_HOST $SPARK_MASTER_PORT $SPARK_MASTER_WEBUI_PORT &&\
/opt/conda/lib/python3.9/site-packages/pyspark/sbin/../bin/spark-class org.apache.spark.deploy.master.Master \
--host $SPARK_MASTER_HOST \
--port $SPARK_MASTER_PORT \
--webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_MASTER_LOG/spark-master.out"
]
ports:
- containerPort: 8080
- containerPort: 7077
resources:
requests:
memory: '512Mi'
cpu: '300m'
limits:
memory: '1Gi'
cpu: '600m'
env:
- name: INIT_DAEMON_STEP
value: 'setup_spark'
- name: SPARK_LOCAL_HOSTNAME
value: 'spark-master'
- name: SPARK_WORKLOAD
value: 'master'
#- name: SPARK_PUBLIC_DNS
#value: '10.242.130.225'
- name: SPARK_MASTER_HOST
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: SPARK_MASTER_PORT
value: '7077'
- name: SPARK_MASTER_WEBUI_PORT
value: '8080'
- name: SPARK_HOME
value: '/opt/conda/lib/python3.9/site-packages/pyspark'
- name: JAVA_HOME
value: '/opt/conda'
---
apiVersion: v1
kind: Service
metadata:
name: spark-master-service
namespace: spark
spec:
selector:
app: spark-master
ports:
- name: http
port: 8080
protocol: TCP
- name: http-cos
port: 7077
protocol: TCP
I can establish spark session with spark master from jupyter netbook
#import
import pyspark
from pyspark.sql import SparkSession
from time import time
import numpy as np
import operator
import os
import mlflow
from random import random
from operator import add
from pyspark import SparkContext
from pyspark import StorageLevel
from pyspark.sql.types import StructType, StructField, TimestampType, StringType, FloatType
#set pyspark enviroment
os.environ["JAVA_HOME"]="/usr/lib/jvm/java-11-openjdk-amd64"
os.environ['PYSPARK_PYTHON'] = '/opt/conda/bin/python'
os.environ['PYSPARK_DRIVER_PYTHON'] = '/opt/conda/bin/python'
os.environ["PYSPARK_SUBMIT_ARGS"] = '--master spark://spark-master-service:7077 pyspark-shell'
#SK start spark session
spark = SparkSession.builder.master("spark://spark-master-service:7077").config("spark.driver.extraJavaOptions", "-Dio.netty.tryReflectionSetAccessible=true").config("spark.executor.extraJavaOptions", "-Dio.netty.tryReflectionSetAccessible=true").config("spark.jars", '/usr/local/bin/elasticsearch-spark-30_2.12-8.1.0.jar').config("spark.sql.execution.arrow.pyspark.enabled", "true").appName('SparkTest').getOrCreate()
I got the bellow error
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
Input In [8], in <cell line: 2>()
1 #SK start spark session
----> 2 spark = SparkSession.builder.master("spark://spark-master-service:7077").config("spark.driver.extraJavaOptions", "-Dio.netty.tryReflectionSetAccessible=true").config("spark.executor.extraJavaOptions", "-Dio.netty.tryReflectionSetAccessible=true").config("spark.jars", '/usr/local/bin/elasticsearch-spark-30_2.12-8.1.0.jar').config("spark.sql.execution.arrow.pyspark.enabled", "true").appName('SparkTestarrow').getOrCreate()
File /usr/local/spark/python/pyspark/sql/session.py:228, in SparkSession.Builder.getOrCreate(self)
226 sparkConf.set(key, value)
227 # This SparkContext may be an existing one.
--> 228 sc = SparkContext.getOrCreate(sparkConf)
229 # Do not update `SparkConf` for existing `SparkContext`, as it's shared
230 # by all sessions.
231 session = SparkSession(sc)
File /usr/local/spark/python/pyspark/context.py:392, in SparkContext.getOrCreate(cls, conf)
390 with SparkContext._lock:
391 if SparkContext._active_spark_context is None:
--> 392 SparkContext(conf=conf or SparkConf())
393 return SparkContext._active_spark_context
File /usr/local/spark/python/pyspark/context.py:144, in SparkContext.__init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
139 if gateway is not None and gateway.gateway_parameters.auth_token is None:
140 raise ValueError(
141 "You are trying to pass an insecure Py4j gateway to Spark. This"
142 " is not allowed as it is a security risk.")
--> 144 SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
145 try:
146 self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,
147 conf, jsc, profiler_cls)
File /usr/local/spark/python/pyspark/context.py:339, in SparkContext._ensure_initialized(cls, instance, gateway, conf)
337 with SparkContext._lock:
338 if not SparkContext._gateway:
--> 339 SparkContext._gateway = gateway or launch_gateway(conf)
340 SparkContext._jvm = SparkContext._gateway.jvm
342 if instance:
File /usr/local/spark/python/pyspark/java_gateway.py:108, in launch_gateway(conf, popen_kwargs)
105 time.sleep(0.1)
107 if not os.path.isfile(conn_info_file):
--> 108 raise RuntimeError("Java gateway process exited before sending its port number")
110 with open(conn_info_file, "rb") as info:
111 gateway_port = read_int(info)
RuntimeError: Java gateway process exited before sending its port number
Can You keep on eye what's wrong in my configuration?