Creating K8S only image

Author: denglishcollibra

docker_templates/Dockerfile.k8sOnly

@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM eclipse-temurin:17-jammy
+
+ARG spark_uid=185
+ARG SPARK_TGZ_FILENAME
+
+RUN groupadd --system --gid=${spark_uid} spark && \
+    useradd --system --uid=${spark_uid} --gid=spark spark
+
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y bash gnupg2 gosu krb5-user libc6 libnss-wrapper libnss3 libpam-modules net-tools procps tini; \
+    mkdir -p /opt/spark; \
+    mkdir /opt/spark/python; \
+    mkdir -p /opt/spark/examples; \
+    mkdir -p /opt/spark/work-dir; \
+    chmod g+w /opt/spark/work-dir; \
+    touch /opt/spark/RELEASE; \
+    chown -R spark:spark /opt/spark; \
+    echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \
+    rm -rf /var/lib/apt/lists/*
+
+COPY ${SPARK_TGZ_FILENAME} /tmp/spark.tgz
+
+RUN set -ex; \
+    export SPARK_TMP="$(mktemp -d)"; \
+    cd "$SPARK_TMP"; \
+    tar -xf /tmp/spark.tgz --strip-components=1; \
+    chown -R spark:spark .; \
+    mv jars /opt/spark/; \
+    mv RELEASE /opt/spark/; \
+    mv bin /opt/spark/; \
+    mv sbin /opt/spark/; \
+    mv examples /opt/spark/; \
+    ln -s "$(basename /opt/spark/examples/jars/spark-examples_*.jar)" /opt/spark/examples/jars/spark-examples.jar; \
+    mv python/pyspark /opt/spark/python/pyspark/; \
+    mv python/lib /opt/spark/python/lib/; \
+    cd ..; \
+    rm -rf "$SPARK_TMP";
+
+COPY entrypoint.sh /opt/
+RUN chmod +x /opt/entrypoint.sh
+
+ENV SPARK_HOME=/opt/spark
+
+WORKDIR /opt/spark/work-dir
+
+USER spark
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]

makeSparkK8sOnlyImage.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+#==============================================================================
+# Setup
+#==============================================================================
+set -e -x
+trap 'handle_error' ERR
+
+handle_error() {
+    set +e +x
+    echo "An error occurred. Cleaning up and exiting..."
+    exit 1
+}
+
+function banner {
+    set +x
+    echo "+----------------------------------------------------"
+    echo "| $1"
+    echo "+----------------------------------------------------"
+    set -x
+}
+
+#==============================================================================
+# Configuration Variables (SET THESE!)
+#==============================================================================
+: "${VERSION:?ERROR: VERSION is not set (e.g., 3.5.5)}"
+: "${IMG_NAME:?ERROR: IMG_NAME is not set (e.g., my-spark-image)}"
+: "${DOCKER_IMG_ROOT:?ERROR: DOCKER_IMG_ROOT is not set (e.g., docker.io/myuser/...)}"
+PROJ_HOME="/Users/david.english/workspace/spark" # Your Spark source directory
+
+# Define the base Dockerfile to fetch from apache/spark-docker (for entrypoint.sh etc.)
+SPARK_DOCKER_ORG="https://github.com/apache/spark-docker"
+SPARK_DOCKER_SUBDIR="master/${VERSION}/scala2.12-java17-ubuntu" # Common subdirectory for Dockerfile and entrypoint
+
+# Location of your Dockerfile template (create this file as described above)
+DOCKERFILE_TEMPLATE_PATH="${PROJ_HOME}/docker_templates/Dockerfile.k8sOnly"
+
+# Temporary directory for Docker build context
+DOCKER_BUILD_DIR="${PROJ_HOME}/docker_build_temp"
+
+
+#==============================================================================
+# Body
+#==============================================================================
+banner "Starting Spark Custom Image Build for $VERSION"
+
+if [[ "$(pwd)" != "$PROJ_HOME" ]]; then
+  echo "Error: You must run this script from $PROJ_HOME"
+  exit 1
+fi
+
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+if [ "$CURRENT_BRANCH" = "master" ]; then
+  echo "Error: Cannot run this script on the 'master' branch. Please switch to a different branch."
+  exit 1
+fi
+
+# 1. Build Spark distribution LOCALLY
+banner "Building custom Spark distribution..."
+./dev/make-distribution.sh --name "$IMG_NAME-$VERSION" \
+--tgz \
+-Pkubernetes
+
+SPARK_DIST_TGZ="$PROJ_HOME/spark-$VERSION-bin-$IMG_NAME-$VERSION.tgz"
+SPARK_TGZ_FILENAME=$(basename "$SPARK_DIST_TGZ") # Get just the filename
+
+banner "Setting up temporary Docker build directory: $DOCKER_BUILD_DIR"
+rm -rf "$DOCKER_BUILD_DIR" # Clean up previous build dir
+mkdir -p "$DOCKER_BUILD_DIR"
+
+# 2. Copy your pre-modified Dockerfile template to the build directory
+banner "Copying pre-modified Dockerfile template..."
+cp "$DOCKERFILE_TEMPLATE_PATH" "${DOCKER_BUILD_DIR}/Dockerfile"
+
+# 3. Fetch necessary supporting files (like entrypoint.sh) from apache/spark-docker
+banner "Fetching official Spark entrypoint.sh and other supporting files..."
+curl -sSL "${SPARK_DOCKER_ORG}/raw/${SPARK_DOCKER_SUBDIR}/entrypoint.sh" -o "${DOCKER_BUILD_DIR}/entrypoint.sh"
+
+
+# 4. Copy your custom-built TGZ into the Docker build context
+banner "Copying custom Spark TGZ into Docker build context..."
+cp "$SPARK_DIST_TGZ" "$DOCKER_BUILD_DIR/"
+
+# 5. Build the Docker image
+banner "Building Docker image..."
+docker buildx build --platform linux/amd64,linux/arm64 \
+  --build-arg spark_uid=185 \
+  --build-arg java_image_tag=17-jammy \
+  --build-arg SPARK_TGZ_FILENAME="$SPARK_TGZ_FILENAME" \
+  -t "$DOCKER_IMG_ROOT/$IMG_NAME:$VERSION" \
+  --push "$DOCKER_BUILD_DIR" # Set the build context to your new directory
+
+#==============================================================================
+# Cleanup
+#==============================================================================
+banner "Cleaning up temporary Docker build directory..."
+rm -rf "$DOCKER_BUILD_DIR" # Clean up the temporary directory
+
+set +e +x
+banner 'Script completed successfully'