diff --git a/docker-compose/docker-compose.yml b/docker/docker-swarm/docker-compose.yml similarity index 95% rename from docker-compose/docker-compose.yml rename to docker/docker-swarm/docker-compose.yml index 804f39a5d4..20fb0cced3 100644 --- a/docker-compose/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" +version: "3.4" networks: dolphinscheduler-postgresql: @@ -41,7 +41,10 @@ volumes: dolphinscheduler-master: dolphinscheduler-worker-data: dolphinscheduler-worker-logs: + +configs: dolphinscheduler-worker-task-env: + file: ./dolphinscheduler_env.sh services: @@ -136,6 +139,7 @@ services: volumes: - dolphinscheduler-frontend:/var/log/nginx networks: + - dolphinscheduler-frontend - dolphinscheduler-api dolphinscheduler-alert: @@ -209,7 +213,8 @@ services: - dolphinscheduler-zookeeper volumes: - dolphinscheduler-master:/opt/dolphinscheduler/logs - networks: + networks: + - dolphinscheduler-master - dolphinscheduler-postgresql - dolphinscheduler-zookeeper @@ -247,7 +252,10 @@ services: volumes: - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-worker-task-env:/opt/dolphinscheduler/conf/env/ - networks: + configs: + - source: dolphinscheduler-worker-task-env + target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + networks: + - dolphinscheduler-worker - dolphinscheduler-postgresql - dolphinscheduler-zookeeper \ No newline at end of file diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh new file mode 100644 index 0000000000..790e30636e --- /dev/null +++ b/docker/docker-swarm/dolphinscheduler_env.sh @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/opt/soft/python +export JAVA_HOME=/opt/soft/java +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH \ No newline at end of file diff --git a/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml similarity index 100% rename from kubernetes/dolphinscheduler/Chart.yaml rename to docker/kubernetes/dolphinscheduler/Chart.yaml diff --git a/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md similarity index 100% rename from kubernetes/dolphinscheduler/README.md rename to docker/kubernetes/dolphinscheduler/README.md diff --git a/kubernetes/dolphinscheduler/requirements.yaml b/docker/kubernetes/dolphinscheduler/requirements.yaml similarity index 100% rename from kubernetes/dolphinscheduler/requirements.yaml rename to docker/kubernetes/dolphinscheduler/requirements.yaml diff --git a/kubernetes/dolphinscheduler/templates/NOTES.txt b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt similarity index 100% rename from kubernetes/dolphinscheduler/templates/NOTES.txt rename to docker/kubernetes/dolphinscheduler/templates/NOTES.txt diff --git a/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl similarity index 100% rename from kubernetes/dolphinscheduler/templates/_helpers.tpl rename to docker/kubernetes/dolphinscheduler/templates/_helpers.tpl diff --git a/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml diff --git a/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml diff --git a/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml diff --git a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml similarity index 96% rename from kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml rename to docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index f3bb1dd880..69662e95d9 100644 --- a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -166,19 +166,19 @@ spec: value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} value: {{ .Values.externalDatabase.port }} - {{- end }} + {{- end }} - name: POSTGRESQL_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PASSWORD valueFrom: secretKeyRef: @@ -188,7 +188,13 @@ spec: {{- else }} name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password - {{- end }} + {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} {{- if .Values.alert.livenessProbe.enabled }} livenessProbe: exec: diff --git a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml similarity index 94% rename from kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml rename to docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 926ce3c062..487ede0b8f 100644 --- a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -99,19 +99,19 @@ spec: value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} value: {{ .Values.externalDatabase.port }} - {{- end }} + {{- end }} - name: POSTGRESQL_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PASSWORD valueFrom: secretKeyRef: @@ -122,6 +122,12 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" diff --git a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml diff --git a/kubernetes/dolphinscheduler/templates/ingress.yaml b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/ingress.yaml rename to docker/kubernetes/dolphinscheduler/templates/ingress.yaml diff --git a/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml rename to docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml diff --git a/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml rename to docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml diff --git a/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml diff --git a/kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml b/docker/kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml rename to docker/kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml diff --git a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml similarity index 97% rename from kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml rename to docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index adc35ce804..b142fac4fe 100644 --- a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -188,6 +188,12 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: {{ template "dolphinscheduler.zookeeper.quorum" . }} diff --git a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml similarity index 97% rename from kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml rename to docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 5a03c52775..198cef43b6 100644 --- a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -184,7 +184,13 @@ spec: {{- else }} name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password - {{- end }} + {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml diff --git a/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml similarity index 100% rename from kubernetes/dolphinscheduler/values.yaml rename to docker/kubernetes/dolphinscheduler/values.yaml diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index 30439c2321..af3c456116 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -70,10 +70,10 @@ initZK() { while ! nc -z ${line%:*} ${line#*:}; do counter=$((counter+1)) if [ $counter == 30 ]; then - log "Error: Couldn't connect to zookeeper." + echo "Error: Couldn't connect to zookeeper." exit 1 fi - log "Trying to connect to zookeeper at ${line}. Attempt $counter." + echo "Trying to connect to zookeeper at ${line}. Attempt $counter." sleep 5 done done