diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 121970bf5a..eeea20d482 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -44,11 +44,10 @@ COPY ./startup-init-conf.sh /root/startup-init-conf.sh COPY ./startup.sh /root/startup.sh COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ -COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ +COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/ RUN dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ - dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ rm -rf /bin/sh && \ diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl index a3ccde7c61..36bc5b8de8 100644 --- a/docker/build/conf/dolphinscheduler/common.properties.tpl +++ b/docker/build/conf/dolphinscheduler/common.properties.tpl @@ -64,7 +64,7 @@ yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s # system env path, If you want to set your own path, you need to set this env file to an absolute path -dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} +#dolphinscheduler.env.path=env/dolphinscheduler_env.sh development.state=false # kerberos tgt expire time, unit is hours diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl similarity index 72% rename from docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh rename to docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl index 7fd39335ae..b09e4972cd 100755 --- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh +++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl @@ -15,14 +15,14 @@ # limitations under the License. # -export HADOOP_HOME=/opt/soft/hadoop -export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop -export SPARK_HOME1=/opt/soft/spark1 -export SPARK_HOME2=/opt/soft/spark2 -export PYTHON_HOME=/usr/bin/python -export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink -export DATAX_HOME=/opt/soft/datax/bin/datax.py +export HADOOP_HOME=$HADOOP_HOME +export HADOOP_CONF_DIR=$HADOOP_CONF_DIR +export SPARK_HOME1=$SPARK_HOME1 +export SPARK_HOME2=$SPARK_HOME2 +export PYTHON_HOME=$PYTHON_HOME +export JAVA_HOME=$JAVA_HOME +export HIVE_HOME=$HIVE_HOME +export FLINK_HOME=$FLINK_HOME +export DATAX_HOME=$DATAX_HOME export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 52004d7e6c..89be05eaf6 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -37,7 +37,17 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} #============================================================================ # Common #============================================================================ -export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} +# dolphinscheduler env +export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"} +export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"} +export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"} +export PYTHON_HOME=${PYTHON_HOME:-"/usr/bin/python"} +export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"} +export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"} +export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"} +export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax/bin/datax.py"} +# common env export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"} @@ -83,9 +93,10 @@ export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"} export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"} echo "generate app config" -ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do +find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.tpl" | while read file; do eval "cat << EOF -$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) +$(cat ${file}) EOF -" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} +" > ${file%.*} done +find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.sh" -exec chmod +x {} \; diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 9c45e5b5df..a4a221c56e 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -162,6 +162,15 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUPS: "default" WORKER_WEIGHT: "100" + HADOOP_HOME: "/opt/soft/hadoop" + HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" + SPARK_HOME1: "/opt/soft/spark1" + SPARK_HOME2: "/opt/soft/spark2" + PYTHON_HOME: "/usr/bin/python" + JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" + HIVE_HOME: "/opt/soft/hive" + FLINK_HOME: "/opt/soft/flink" + DATAX_HOME: "/opt/soft/datax/bin/datax.py" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler ALERT_LISTEN_HOST: dolphinscheduler-alert DATABASE_HOST: dolphinscheduler-postgresql @@ -183,7 +192,6 @@ services: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - - ./dolphinscheduler_env.sh:/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-resource-local:/dolphinscheduler diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index da5b8cba16..7206e4e678 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -156,6 +156,15 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUPS: "default" WORKER_WEIGHT: "100" + HADOOP_HOME: "/opt/soft/hadoop" + HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" + SPARK_HOME1: "/opt/soft/spark1" + SPARK_HOME2: "/opt/soft/spark2" + PYTHON_HOME: "/usr/bin/python" + JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" + HIVE_HOME: "/opt/soft/hive" + FLINK_HOME: "/opt/soft/flink" + DATAX_HOME: "/opt/soft/datax/bin/datax.py" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler ALERT_LISTEN_HOST: dolphinscheduler-alert DATABASE_HOST: dolphinscheduler-postgresql @@ -173,9 +182,6 @@ services: timeout: 5s retries: 3 start_period: 30s - configs: - - source: dolphinscheduler-worker-task-env - target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh volumes: - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs @@ -193,8 +199,4 @@ volumes: dolphinscheduler-postgresql: dolphinscheduler-zookeeper: dolphinscheduler-worker-data: - dolphinscheduler-logs: - -configs: - dolphinscheduler-worker-task-env: - file: ./dolphinscheduler_env.sh \ No newline at end of file + dolphinscheduler-logs: \ No newline at end of file diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh deleted file mode 100755 index 7fd39335ae..0000000000 --- a/docker/docker-swarm/dolphinscheduler_env.sh +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export HADOOP_HOME=/opt/soft/hadoop -export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop -export SPARK_HOME1=/opt/soft/spark1 -export SPARK_HOME2=/opt/soft/spark2 -export PYTHON_HOME=/usr/bin/python -export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink -export DATAX_HOME=/opt/soft/datax/bin/datax.py - -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH