Modify docker-compose and kubernetes config

1. Move kubernetes to docker directory
2. Change docker-compose directory's name, and move docker-compose to docker directory
3. Modify docker-compose config and kubernetes config
4. Modify startup.sh in dockerfile

docker/docker-swarm/docker-compose.yml

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version: "3"
+version: "3.4"
 
 networks:
   dolphinscheduler-postgresql:
@@ -41,7 +41,10 @@ volumes:
   dolphinscheduler-master:
   dolphinscheduler-worker-data:
   dolphinscheduler-worker-logs:
+
+configs:
   dolphinscheduler-worker-task-env:
+    file: ./dolphinscheduler_env.sh
 
 services:
 
@@ -136,6 +139,7 @@ services:
     volumes:
     - dolphinscheduler-frontend:/var/log/nginx
     networks:
+    - dolphinscheduler-frontend
     - dolphinscheduler-api
 
   dolphinscheduler-alert:
@@ -209,7 +213,8 @@ services:
     - dolphinscheduler-zookeeper
     volumes: 
     - dolphinscheduler-master:/opt/dolphinscheduler/logs
-    networks: 
+    networks:
+    - dolphinscheduler-master
     - dolphinscheduler-postgresql
     - dolphinscheduler-zookeeper
 
@@ -247,7 +252,10 @@ services:
     volumes: 
     - dolphinscheduler-worker-data:/tmp/dolphinscheduler
     - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs
-    - dolphinscheduler-worker-task-env:/opt/dolphinscheduler/conf/env/
-    networks: 
+    configs:
+    - source: dolphinscheduler-worker-task-env
+      target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
+    networks:
+    - dolphinscheduler-worker
     - dolphinscheduler-postgresql
     - dolphinscheduler-zookeeper

docker/docker-swarm/dolphinscheduler_env.sh

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+export HADOOP_HOME=/opt/soft/hadoop
+export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
+export SPARK_HOME1=/opt/soft/spark1
+export SPARK_HOME2=/opt/soft/spark2
+export PYTHON_HOME=/opt/soft/python
+export JAVA_HOME=/opt/soft/java
+export HIVE_HOME=/opt/soft/hive
+export FLINK_HOME=/opt/soft/flink
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH

docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@@ -166,19 +166,19 @@ spec:
               value: {{ template "dolphinscheduler.postgresql.fullname" . }}
               {{- else }}
               value: {{ .Values.externalDatabase.host | quote }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_PORT
               {{- if .Values.postgresql.enabled }}
               value: "5432"
               {{- else }}
               value: {{ .Values.externalDatabase.port }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_USERNAME
               {{- if .Values.postgresql.enabled }}
               value: {{ .Values.postgresql.postgresqlUsername }}
               {{- else }}
               value: {{ .Values.externalDatabase.username | quote }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -188,7 +188,13 @@ spec:
                   {{- else }}
                   name: {{ printf "%s-%s" .Release.Name "externaldb"  }}
                   key: db-password
-          {{- end }}
+                  {{- end }}
+            - name: POSTGRESQL_DATABASE
+              {{- if .Values.postgresql.enabled }}
+              value: {{ .Values.postgresql.postgresqlDatabase }}
+              {{- else }}
+              value: {{ .Values.externalDatabase.database | quote }}
+              {{- end }}
           {{- if .Values.alert.livenessProbe.enabled }}
           livenessProbe:
             exec:

docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@@ -99,19 +99,19 @@ spec:
               value: {{ template "dolphinscheduler.postgresql.fullname" . }}
               {{- else }}
               value: {{ .Values.externalDatabase.host | quote }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_PORT
               {{- if .Values.postgresql.enabled }}
               value: "5432"
               {{- else }}
               value: {{ .Values.externalDatabase.port }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_USERNAME
               {{- if .Values.postgresql.enabled }}
               value: {{ .Values.postgresql.postgresqlUsername }}
               {{- else }}
               value: {{ .Values.externalDatabase.username | quote }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -122,6 +122,12 @@ spec:
                   name: {{ printf "%s-%s" .Release.Name "externaldb"  }}
                   key: db-password
                   {{- end }}
+            - name: POSTGRESQL_DATABASE
+              {{- if .Values.postgresql.enabled }}
+              value: {{ .Values.postgresql.postgresqlDatabase }}
+              {{- else }}
+              value: {{ .Values.externalDatabase.database | quote }}
+              {{- end }}
             - name: ZOOKEEPER_QUORUM
               {{- if .Values.zookeeper.enabled }}
               value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"

docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@@ -188,6 +188,12 @@ spec:
                   name: {{ printf "%s-%s" .Release.Name "externaldb"  }}
                   key: db-password
                   {{- end }}
+            - name: POSTGRESQL_DATABASE
+              {{- if .Values.postgresql.enabled }}
+              value: {{ .Values.postgresql.postgresqlDatabase }}
+              {{- else }}
+              value: {{ .Values.externalDatabase.database | quote }}
+              {{- end }}
             - name: ZOOKEEPER_QUORUM
               {{- if .Values.zookeeper.enabled }}
               value: {{ template "dolphinscheduler.zookeeper.quorum" . }}

docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -184,7 +184,13 @@ spec:
                   {{- else }}
                   name: {{ printf "%s-%s" .Release.Name "externaldb"  }}
                   key: db-password
-            {{- end }}
+                  {{- end }}
+            - name: POSTGRESQL_DATABASE
+              {{- if .Values.postgresql.enabled }}
+              value: {{ .Values.postgresql.postgresqlDatabase }}
+              {{- else }}
+              value: {{ .Values.externalDatabase.database | quote }}
+              {{- end }}
             - name: ZOOKEEPER_QUORUM
               {{- if .Values.zookeeper.enabled }}
               value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"

dockerfile/startup.sh

@@ -70,10 +70,10 @@ initZK() {
             while ! nc -z ${line%:*} ${line#*:}; do
                 counter=$((counter+1))
                 if [ $counter == 30 ]; then
-                    log "Error: Couldn't connect to zookeeper."
+                    echo "Error: Couldn't connect to zookeeper."
                     exit 1
                 fi
-                log "Trying to connect to zookeeper at ${line}. Attempt $counter."
+                echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
                 sleep 5
             done
         done