From 7eecfe45eaa34536c4c67f6f95871d97b4ecc4ce Mon Sep 17 00:00:00 2001 From: liwenhe1993 <411934049@qq.com> Date: Sun, 26 Apr 2020 14:13:07 +0800 Subject: [PATCH] Modify docker-compose and kubernetes config 1. Move kubernetes to docker directory 2. Change docker-compose directory's name, and move docker-compose to docker directory 3. Modify docker-compose config and kubernetes config 4. Modify startup.sh in dockerfile --- .../docker-swarm}/docker-compose.yml | 16 +++++++++--- docker/docker-swarm/dolphinscheduler_env.sh | 26 +++++++++++++++++++ .../kubernetes}/dolphinscheduler/Chart.yaml | 0 .../kubernetes}/dolphinscheduler/README.md | 0 .../dolphinscheduler/requirements.yaml | 0 .../dolphinscheduler/templates/NOTES.txt | 0 .../dolphinscheduler/templates/_helpers.tpl | 0 .../configmap-dolphinscheduler-alert.yaml | 0 .../configmap-dolphinscheduler-master.yaml | 0 .../configmap-dolphinscheduler-worker.yaml | 0 .../deployment-dolphinscheduler-alert.yaml | 14 +++++++--- .../deployment-dolphinscheduler-api.yaml | 12 ++++++--- .../deployment-dolphinscheduler-frontend.yaml | 0 .../dolphinscheduler/templates/ingress.yaml | 0 .../templates/pvc-dolphinscheduler-alert.yaml | 0 .../templates/pvc-dolphinscheduler-api.yaml | 0 .../pvc-dolphinscheduler-frontend.yaml | 0 .../templates/secret-external-postgresql.yaml | 0 .../statefulset-dolphinscheduler-master.yaml | 6 +++++ .../statefulset-dolphinscheduler-worker.yaml | 8 +++++- .../templates/svc-dolphinscheduler-api.yaml | 0 .../svc-dolphinscheduler-frontend.yaml | 0 .../svc-dolphinscheduler-master-headless.yaml | 0 .../svc-dolphinscheduler-worker-headless.yaml | 0 .../kubernetes}/dolphinscheduler/values.yaml | 0 dockerfile/startup.sh | 4 +-- 26 files changed, 72 insertions(+), 14 deletions(-) rename {docker-compose => docker/docker-swarm}/docker-compose.yml (95%) create mode 100644 docker/docker-swarm/dolphinscheduler_env.sh rename {kubernetes => docker/kubernetes}/dolphinscheduler/Chart.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/README.md (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/requirements.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/NOTES.txt (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/_helpers.tpl (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml (96%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml (94%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/ingress.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/secret-external-postgresql.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml (97%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml (97%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/values.yaml (100%) diff --git a/docker-compose/docker-compose.yml b/docker/docker-swarm/docker-compose.yml similarity index 95% rename from docker-compose/docker-compose.yml rename to docker/docker-swarm/docker-compose.yml index 804f39a5d4..20fb0cced3 100644 --- a/docker-compose/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" +version: "3.4" networks: dolphinscheduler-postgresql: @@ -41,7 +41,10 @@ volumes: dolphinscheduler-master: dolphinscheduler-worker-data: dolphinscheduler-worker-logs: + +configs: dolphinscheduler-worker-task-env: + file: ./dolphinscheduler_env.sh services: @@ -136,6 +139,7 @@ services: volumes: - dolphinscheduler-frontend:/var/log/nginx networks: + - dolphinscheduler-frontend - dolphinscheduler-api dolphinscheduler-alert: @@ -209,7 +213,8 @@ services: - dolphinscheduler-zookeeper volumes: - dolphinscheduler-master:/opt/dolphinscheduler/logs - networks: + networks: + - dolphinscheduler-master - dolphinscheduler-postgresql - dolphinscheduler-zookeeper @@ -247,7 +252,10 @@ services: volumes: - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-worker-task-env:/opt/dolphinscheduler/conf/env/ - networks: + configs: + - source: dolphinscheduler-worker-task-env + target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + networks: + - dolphinscheduler-worker - dolphinscheduler-postgresql - dolphinscheduler-zookeeper \ No newline at end of file diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh new file mode 100644 index 0000000000..790e30636e --- /dev/null +++ b/docker/docker-swarm/dolphinscheduler_env.sh @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/opt/soft/python +export JAVA_HOME=/opt/soft/java +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH \ No newline at end of file diff --git a/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml similarity index 100% rename from kubernetes/dolphinscheduler/Chart.yaml rename to docker/kubernetes/dolphinscheduler/Chart.yaml diff --git a/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md similarity index 100% rename from kubernetes/dolphinscheduler/README.md rename to docker/kubernetes/dolphinscheduler/README.md diff --git a/kubernetes/dolphinscheduler/requirements.yaml b/docker/kubernetes/dolphinscheduler/requirements.yaml similarity index 100% rename from kubernetes/dolphinscheduler/requirements.yaml rename to docker/kubernetes/dolphinscheduler/requirements.yaml diff --git a/kubernetes/dolphinscheduler/templates/NOTES.txt b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt similarity index 100% rename from kubernetes/dolphinscheduler/templates/NOTES.txt rename to docker/kubernetes/dolphinscheduler/templates/NOTES.txt diff --git a/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl similarity index 100% rename from kubernetes/dolphinscheduler/templates/_helpers.tpl rename to docker/kubernetes/dolphinscheduler/templates/_helpers.tpl diff --git a/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml diff --git a/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml diff --git a/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml diff --git a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml similarity index 96% rename from kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml rename to docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index f3bb1dd880..69662e95d9 100644 --- a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -166,19 +166,19 @@ spec: value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} value: {{ .Values.externalDatabase.port }} - {{- end }} + {{- end }} - name: POSTGRESQL_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PASSWORD valueFrom: secretKeyRef: @@ -188,7 +188,13 @@ spec: {{- else }} name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password - {{- end }} + {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} {{- if .Values.alert.livenessProbe.enabled }} livenessProbe: exec: diff --git a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml similarity index 94% rename from kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml rename to docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 926ce3c062..487ede0b8f 100644 --- a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -99,19 +99,19 @@ spec: value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} value: {{ .Values.externalDatabase.port }} - {{- end }} + {{- end }} - name: POSTGRESQL_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PASSWORD valueFrom: secretKeyRef: @@ -122,6 +122,12 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" diff --git a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml diff --git a/kubernetes/dolphinscheduler/templates/ingress.yaml b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/ingress.yaml rename to docker/kubernetes/dolphinscheduler/templates/ingress.yaml diff --git a/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml rename to docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml diff --git a/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml rename to docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml diff --git a/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml diff --git a/kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml b/docker/kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml rename to docker/kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml diff --git a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml similarity index 97% rename from kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml rename to docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index adc35ce804..b142fac4fe 100644 --- a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -188,6 +188,12 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: {{ template "dolphinscheduler.zookeeper.quorum" . }} diff --git a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml similarity index 97% rename from kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml rename to docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 5a03c52775..198cef43b6 100644 --- a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -184,7 +184,13 @@ spec: {{- else }} name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password - {{- end }} + {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml diff --git a/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml similarity index 100% rename from kubernetes/dolphinscheduler/values.yaml rename to docker/kubernetes/dolphinscheduler/values.yaml diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index 30439c2321..af3c456116 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -70,10 +70,10 @@ initZK() { while ! nc -z ${line%:*} ${line#*:}; do counter=$((counter+1)) if [ $counter == 30 ]; then - log "Error: Couldn't connect to zookeeper." + echo "Error: Couldn't connect to zookeeper." exit 1 fi - log "Trying to connect to zookeeper at ${line}. Attempt $counter." + echo "Trying to connect to zookeeper at ${line}. Attempt $counter." sleep 5 done done