Apache Kyuubi on CDH 在竞技世界大数据平台实践

export HADOOP_CONF_DIR=/opt/spark/conf:/opt/spark/confexport YARN_CONF_DIR=/opt/spark/conf:/opt/spark/conf

#方便后面解析CDH集群ipecho " ***.***.***.***  "  >> /etc/hosts#kerberos认证需要的配置文件echo "***" > /etc/krb5.conf#在镜像中进行认证操作kinit -kt /opt/spark/work-dir/hive.keytab hive/***@****.****.****

#!/bin/bash## Licensed to the Apache Software Foundation (ASF) under one or more# contributor license agreements.  See the NOTICE file distributed with# this work for additional information regarding copyright ownership.# The ASF licenses this file to You under the Apache License, Version 2.0# (the "License"); you may not use this file except in compliance with# the License.  You may obtain a copy of the License at##    http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.#
# echo commands to the terminal outputset -ex
# Check whether there is a passwd entry for the container UID#myuid=$(id -u)myuid=0mygid=$(id -g)# turn off -e for getent because it will return error code in anonymous uid caseset +euidentry=$(getent passwd $myuid)set -e
# If there is no passwd entry for the container UID, attempt to create oneif [ -z "$uidentry" ] ; then    if [ -w /etc/passwd ] ; then        echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd    else        echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"    fifi
if [ -z "$JAVA_HOME" ]; then  JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')fi
SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=(.*)/1/g' > /tmp/java_opts.txtreadarray -t SPARK_EXECUTOR_JAVA_OPTS 
if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then  SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"fi
if ! [ -z ${PYSPARK_PYTHON+x} ]; then    export PYSPARK_PYTHONfiif ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then    export PYSPARK_DRIVER_PYTHONfi
# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.if [ -n "${HADOOP_HOME}"  ] && [ -z "${SPARK_DIST_CLASSPATH}"  ]; then  export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"fi
if ! [ -z ${HADOOP_CONF_DIR+x} ]; then  SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";fi
if ! [ -z ${SPARK_CONF_DIR+x} ]; then  SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";elif ! [ -z ${SPARK_HOME+x} ]; then  SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";fi


case "$1" in  driver)    shift 1    chmod 777 /opt/spark/work-dir/run.sh    /bin/bash /opt/spark/work-dir/run.sh    cat /etc/hosts    CMD=(      "$SPARK_HOME/bin/spark-submit"      --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"      --deploy-mode client      "$@"    )    ;;  executor)    shift 1    chmod 777 /opt/spark/work-dir/run.sh    /bin/bash /opt/spark/work-dir/run.sh    cat /etc/hosts    CMD=(      ${JAVA_HOME}/bin/java      "${SPARK_EXECUTOR_JAVA_OPTS[@]}"      -Xms$SPARK_EXECUTOR_MEMORY      -Xmx$SPARK_EXECUTOR_MEMORY      -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"      org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend      --driver-url $SPARK_DRIVER_URL      --executor-id $SPARK_EXECUTOR_ID      --cores $SPARK_EXECUTOR_CORES      --app-id $SPARK_APPLICATION_ID      --hostname $SPARK_EXECUTOR_POD_IP      --resourceProfileId $SPARK_RESOURCE_PROFILE_ID      --podName $SPARK_EXECUTOR_POD_NAME    )    ;;
  *)    echo "Non-spark-on-k8s command provided, proceeding in pass-through mode..."    CMD=("$@")    ;;esac
# Execute the container CMD under tini for better hygieneexec /usr/bin/tini -s -- "${CMD[@]}"

# contributor license agreements.  See the NOTICE file distributed with# this work for additional information regarding copyright ownership.# The ASF licenses this file to You under the Apache License, Version 2.0# (the "License"); you may not use this file except in compliance with# the License.  You may obtain a copy of the License at##    http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.#ARG java_image_tag=8-jre-slim
FROM ***.***.***.***/bigdata/openjdk:${java_image_tag}
#ARG spark_uid=185ARG spark_uid=0
# Before building the docker image, first build and make a Spark distribution following# the instructions in https://spark.apache.org/docs/latest/building-spark.html.# If this docker file is being used in the context of building your images from a Spark# distribution, the docker build command should be invoked from the top level directory# of the Spark distribution. E.g.:# docker build -t spark:latest -f kubernetes/dockerfiles/spark/Dockerfile .
RUN set -ex &&     sed -i 's/http://deb.(.*)/https://deb.1/g' /etc/apt/sources.list &&     sed -i 's/http://security.(.*)/https://security.1/g' /etc/apt/sources.list &&     sed -i s@/security.debian.org/@/mirrors.aliyun.com/@g /etc/apt/sources.list &&     sed -i s@/deb.debian.org/@/mirrors.aliyun.com/@g /etc/apt/sources.list &&     apt-get update &&     ln -s /lib /lib64 &&     apt-get install -y vim sudo net-tools lsof bash tini libc6 libpam-modules krb5-user libpam-krb5 libpam-ccreds libkrb5-dev libnss3 procps &&     mkdir -p /opt/spark &&     mkdir -p /opt/spark/examples &&     mkdir -p /opt/spark/work-dir &&     mkdir -p /opt/hadoop &&     touch /opt/spark/RELEASE &&     rm /bin/sh &&     ln -sv /bin/bash /bin/sh &&     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su &&     chgrp root /etc/passwd && chmod ug+rw /etc/passwd &&     rm -rf /var/cache/apt/*
COPY jars /opt/spark/jarsCOPY bin /opt/spark/binCOPY sbin /opt/spark/sbinCOPY kubernetes/dockerfiles/spark/entrypoint.sh /opt/COPY kubernetes/dockerfiles/spark/decom.sh /opt/COPY examples /opt/spark/examplesCOPY kubernetes/tests /opt/spark/tests#COPY hadoop/conf /opt/hadoop/confCOPY conf /opt/spark/confCOPY data /opt/spark/dataCOPY hive.keytab /opt/spark/work-dirCOPY run.sh /opt/spark/work-dir
ENV SPARK_HOME /opt/spark
WORKDIR /opt/spark/work-dirRUN chmod 777 /opt/spark/work-dirRUN chmod a+x /opt/decom.shRUN chmod 777 /opt/spark/work-dir/run.shENTRYPOINT [ "/opt/entrypoint.sh" ]
# Specify the User that the actual main process will run asUSER ${spark_uid}

#创建镜像./bin/docker-image-tool.sh -t v3.3.0 build#修改镜像tagdocker tag spark:v3.3.0 ***.***.***.***/bigdata/spark:v3.3.0#将镜像push到内部库中（公司内部自建）docker push ***.***.***.***/bigdata/spark:v3.3.0

mkdir /etc/.kubechmod 777 /root/.kubecp /opt/kyuubi/config /root/.kube#kubectl可用的重要一步echo "export  KUBECONFIG=/etc/.kube/config" >> /etc/profileexport  KUBECONFIG=/etc/.kube/configsource /etc/profile
#将kubectl放入内网方便下载使用wget http://***.***.***.***/yum/k8s/kubectlchmod +x ./kubectlmv ./kubectl /usr/bin/#查看kubectl是否安装成功kubectl version --client
echo "***"  >> /etc/hosts
echo "***" > /etc/krb5.conf
kinit -kt /opt/kyuubi/hive.keytab hive/***@HADOOP.****.***

chmod 777 /opt/kyuubi/run.sh/bin/bash /opt/kyuubi/run.sh

#!/usr/bin/env bash## Licensed to the Apache Software Foundation (ASF) under one or more# contributor license agreements.  See the NOTICE file distributed with# this work for additional information regarding copyright ownership.# The ASF licenses this file to You under the Apache License, Version 2.0# (the "License"); you may not use this file except in compliance with# the License.  You may obtain a copy of the License at##    http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.#
## Kyuubi Server Main EntranceCLASS="org.apache.kyuubi.server.KyuubiServer"
function usage() {  echo "Usage: bin/kyuubi command"  echo "  commands:"  echo "    start        - Run a Kyuubi server as a daemon"  echo "    restart      - Restart Kyuubi server as a daemon"  echo "    run          - Run a Kyuubi server in the foreground"  echo "    stop         - Stop the Kyuubi daemon"  echo "    status       - Show status of the Kyuubi daemon"  echo "    -h | --help  - Show this help message"}
if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then  usage  exit 0fi
function kyuubi_logo() {  source ${KYUUBI_HOME}/bin/kyuubi-logo}
function kyuubi_rotate_log() {  log=$1;
  if [[ -z ${KYUUBI_MAX_LOG_FILES} ]]; then    num=5  elif [[ ${KYUUBI_MAX_LOG_FILES} -gt 0 ]]; then    num=${KYUUBI_MAX_LOG_FILES}  else    echo "Error: KYUUBI_MAX_LOG_FILES must be a positive number, but got ${KYUUBI_MAX_LOG_FILES}"    exit -1  fi
  if [ -f "$log" ]; then # rotate logs  while [ ${num} -gt 1 ]; do    prev=expr ${num} - 1    [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"    num=${prev}  done  mv "$log" "$log.$num";  fi}
export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"
if [[ $1 == "start" ]] || [[ $1 == "run" ]]; then  . "${KYUUBI_HOME}/bin/load-kyuubi-env.sh"else  . "${KYUUBI_HOME}/bin/load-kyuubi-env.sh" -sfi
if [[ -z ${JAVA_HOME} ]]; then  echo "Error: JAVA_HOME IS NOT SET! CANNOT PROCEED."  exit 1fi
RUNNER="${JAVA_HOME}/bin/java"
## Find the Kyuubi Jarif [[ -z "$KYUUBI_JAR_DIR" ]]; then  KYUUBI_JAR_DIR="$KYUUBI_HOME/jars"  if [[ ! -d ${KYUUBI_JAR_DIR} ]]; then  echo -e "nCandidate Kyuubi lib $KYUUBI_JAR_DIR doesn't exist, searching development environment..."    KYUUBI_JAR_DIR="$KYUUBI_HOME/kyuubi-assembly/target/scala-${KYUUBI_SCALA_VERSION}/jars"  fifi
if [[ -z ${YARN_CONF_DIR} ]]; then  KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}"else  KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}:${YARN_CONF_DIR}"fi
cmd="${RUNNER} ${KYUUBI_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS"
pid="${KYUUBI_PID_DIR}/kyuubi-$USER-$CLASS.pid"
function start_kyuubi() {  if [[ ! -w ${KYUUBI_PID_DIR} ]]; then    echo "${USER} does not have 'w' permission to ${KYUUBI_PID_DIR}"    exit 1  fi
  if [[ ! -w ${KYUUBI_LOG_DIR} ]]; then    echo "${USER} does not have 'w' permission to ${KYUUBI_LOG_DIR}"    exit 1  fi
  if [ -f "$pid" ]; then    TARGET_ID="$(cat "$pid")"    if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then      echo "$CLASS running as process $TARGET_ID  Stop it first."      exit 1    fi  fi
  log="${KYUUBI_LOG_DIR}/kyuubi-$USER-$CLASS-$HOSTNAME.out"  kyuubi_rotate_log ${log}
  echo "Starting $CLASS, logging to $log"  nohup nice -n "${KYUUBI_NICENESS:-0}" ${cmd} >> ${log} 2>&1 /dev/null &  newpid="$!"
  echo "$newpid" > "$pid"
  # Poll for up to 5 seconds for the java process to start  for i in {1..10}  do    if [[ $(ps -p "$newpid" -o comm=) =~ "java" ]]; then       break    fi    sleep 0.5  done
  sleep 2  # Check if the process has died; in that case we'll tail the log so the user can see  if [[ ! $(ps -p "$newpid" -o comm=) =~ "java" ]]; then    echo "Failed to launch: ${cmd}"    tail -2 "$log" | sed 's/^/  /'    echo "Full log in $log"  else    echo "Welcome to"    kyuubi_logo  fi}
function run_kyuubi() {  echo "Starting $CLASS"  nice -n "${KYUUBI_NICENESS:-0}" ${cmd}}
function stop_kyuubi() {  if [ -f ${pid} ]; then    TARGET_ID="$(cat "$pid")"    if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then      echo "Stopping $CLASS"      kill "$TARGET_ID" && rm -f "$pid"      for i in {1..20}      do        sleep 0.5        if [[ ! $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then          break        fi      done
      if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then        echo "Failed to stop kyuubi after 10 seconds, try 'kill -9 ${TARGET_ID}' forcefully "      else        kyuubi_logo        echo "Bye!"      fi    else      echo "no $CLASS to stop"    fi  else    echo "no $CLASS to stop"  fi}
function check_kyuubi() {  if [[ -f ${pid} ]]; then    TARGET_ID="$(cat "$pid")"    if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then      echo "Kyuubi is running (pid: $TARGET_ID)"    else      echo "Kyuubi is not running"    fi  else    echo "Kyuubi is not running"  fi}
case $1 in  (start | "")    start_kyuubi    ;;
  (restart)    echo "Restarting Kyuubi"    stop_kyuubi    start_kyuubi    ;;
  (run)    chmod 777 /opt/kyuubi/run.sh    /bin/bash /opt/kyuubi/run.sh    run_kyuubi    ;;
  (stop)    stop_kyuubi    ;;
  (status)    check_kyuubi    ;;
  (*)    usage    ;;esac