[Improvement-5405][Docker/K8s] Adapt to registry SPI (#5681)

docker/build/conf/dolphinscheduler/registry.properties.tpl

@@ -15,9 +15,13 @@
 # limitations under the License.
 #
 
-registry.plugin.name=${REGISTRY_PLUGIN_NAME}
+#registry.plugin.dir config the Registry Plugin dir.
 registry.plugin.dir=${REGISTRY_PLUGIN_DIR}
-#registry.plugin.binding=registry
+
+registry.plugin.name=${REGISTRY_PLUGIN_NAME}
 registry.servers=${REGISTRY_SERVERS}
 
+#maven.local.repository=/usr/local/localRepository
 
+#registry.plugin.binding config the Registry Plugin need be load when development and run in IDE
+#registry.plugin.binding=./dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml

docker/build/startup-init-conf.sh

@@ -37,7 +37,7 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
 #============================================================================
 # Registry
 #============================================================================
-export REGISTRY_PLUGIN_DIR=${REGISTRY_PLUGIN_DIR:-"lib/plugin/registry/zookeeper"}
+export REGISTRY_PLUGIN_DIR=${REGISTRY_PLUGIN_DIR:-"lib/plugin/registry"}
 export REGISTRY_PLUGIN_NAME=${REGISTRY_PLUGIN_NAME:-"zookeeper"}
 export REGISTRY_SERVERS=${REGISTRY_SERVERS:-"127.0.0.1:2181"}
 

docker/docker-swarm/config.env.sh

@@ -37,9 +37,8 @@ DATABASE_PARAMS=characterEncoding=utf8
 # DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8
 
 #============================================================================
-# ZooKeeper
+# Registry
 #============================================================================
-
 REGISTRY_PLUGIN_DIR=lib/plugin/registry
 REGISTRY_PLUGIN_NAME=zookeeper
 REGISTRY_SERVERS=dolphinscheduler-zookeeper:2181

docker/kubernetes/dolphinscheduler/templates/_helpers.tpl

@@ -162,21 +162,27 @@ Create a database environment variables.
   {{- end }}
 {{- end -}}
 
-{{/* todo
-Create a rregistry environment variables.
+{{/*
+Create a registry environment variables.
 */}}
-{{- define "dolphinscheduler.zookeeper.env_vars" -}}
-- name: ZOOKEEPER_QUORUM
+{{- define "dolphinscheduler.registry.env_vars" -}}
+- name: REGISTRY_PLUGIN_DIR
   {{- if .Values.zookeeper.enabled }}
-  value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
+  value: "lib/plugin/registry"
+  {{- else }}
+  value: {{ .Values.externalRegistry.registryPluginDir }}
+  {{- end }}
+- name: REGISTRY_PLUGIN_NAME
+  {{- if .Values.zookeeper.enabled }}
+  value: "zookeeper"
   {{- else }}
-  value: {{ .Values.externalZookeeper.zookeeperQuorum }}
+  value: {{ .Values.externalRegistry.registryPluginName }}
   {{- end }}
-- name: ZOOKEEPER_ROOT
+- name: REGISTRY_SERVERS
   {{- if .Values.zookeeper.enabled }}
-  value: {{ .Values.zookeeper.zookeeperRoot }}
+  value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
   {{- else }}
-  value: {{ .Values.externalZookeeper.zookeeperRoot }}
+  value: {{ .Values.externalRegistry.registryServers }}
   {{- end }}
 {{- end -}}
 

docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@@ -68,7 +68,7 @@ spec:
             - name: TZ
               value: {{ .Values.timezone }}
             {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
-            {{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
+            {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
           envFrom:
             - configMapRef:

docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@@ -65,7 +65,7 @@ spec:
             - name: TZ
               value: {{ .Values.timezone }}
             {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
-            {{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
+            {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
           envFrom:
             - configMapRef:

docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -69,7 +69,7 @@ spec:
             - name: ALERT_LISTEN_HOST
               value: {{ include "dolphinscheduler.fullname" . }}-alert
             {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
-            {{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
+            {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
           envFrom:
             - configMapRef:

docker/kubernetes/dolphinscheduler/values.yaml

@@ -27,7 +27,7 @@ image:
   pullPolicy: "IfNotPresent"
   pullSecret: ""
 
-## If not exists external database, by default, Dolphinscheduler's database will use it.
+## If not exists external database, the postgresql database will be used by default.
 postgresql:
   enabled: true
   postgresqlUsername: "root"
@@ -38,8 +38,7 @@ postgresql:
     size: "20Gi"
     storageClass: "-"
 
-## If exists external database, and set postgresql.enable value to false.
-## external database will be used, otherwise Dolphinscheduler's database will be used.
+## If exists external database and set postgresql.enable value to false, the external database will be used.
 externalDatabase:
   type: "postgresql"
   driver: "org.postgresql.Driver"
@@ -50,7 +49,7 @@ externalDatabase:
   database: "dolphinscheduler"
   params: "characterEncoding=utf8"
 
-## If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it.
+## If not exists external registry, the zookeeper registry will be used by default.
 zookeeper:
   enabled: true
   fourlwCommandsWhitelist: "srvr,ruok,wchs,cons"
@@ -58,13 +57,12 @@ zookeeper:
     enabled: false
     size: "20Gi"
     storageClass: "-"
-  zookeeperRoot: "/dolphinscheduler"
 
-## If exists external zookeeper, and set zookeeper.enable value to false.
-## If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it.
-externalZookeeper:
-  zookeeperQuorum: "127.0.0.1:2181"
-  zookeeperRoot: "/dolphinscheduler"
+## If exists external registry and set zookeeper.enable value to false, the external registry will be used.
+externalRegistry:
+  registryPluginDir: "lib/plugin/registry"
+  registryPluginName: "zookeeper"
+  registryServers: "127.0.0.1:2181"
 
 common:
   ## Configmap

dolphinscheduler-alert/src/main/resources/logback-alert.xml

@@ -46,7 +46,6 @@
     </appender>
 
     <root level="INFO">
-        <appender-ref ref="STDOUT"/>
         <appender-ref ref="ALERTLOGFILE"/>
     </root>
 

dolphinscheduler-server/src/main/resources/logback-master.xml

@@ -30,8 +30,6 @@
         </encoder>
     </appender>
 
-
-
     <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
         <!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
             <level>INFO</level>

dolphinscheduler-service/src/main/resources/registry.properties

@@ -25,5 +25,3 @@ registry.servers=127.0.0.1:2181
 
 #registry.plugin.binding config the Registry Plugin need be load when development and run in IDE
 #registry.plugin.binding=./dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml
-
-