[Improvement][Etcd] Support SSL In Etcd And Enhance Etcd In Helm (#13924)

deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl

@@ -212,15 +212,36 @@ Create a registry environment variables.
 - name: REGISTRY_TYPE
   {{- if .Values.zookeeper.enabled }}
   value: "zookeeper"
+  {{- else if .Values.etcd.enabled }}
+  value: "etcd"
   {{- else }}
   value: {{ .Values.externalRegistry.registryPluginName }}
   {{- end }}
+{{- if .Values.etcd.enabled }}
+- name: REGISTRY_ENDPOINTS
+  value: {{ .Values.etcd.endpoints }}
+- name: REGISTRY_NAMESPACE
+  value: {{ .Values.etcd.namespace }}
+- name: REGISTRY_USER
+  value: {{ .Values.etcd.user }}
+- name: REGISTRY_PASSWORD
+  value: {{ .Values.etcd.passWord }}
+- name: REGISTRY_AUTHORITY
+  value: {{ .Values.etcd.authority }}
+- name: REGISTRY_CERT_FILE
+  value: {{ .Values.etcd.ssl.certFile }}
+- name: REGISTRY_KEY_CERT_CHAIN_FILE
+  value: {{ .Values.etcd.ssl.keyCertChainFile }}
+- name: REGISTRY_KEY_FILE
+  value: {{ .Values.etcd.ssl.keyFile }}
+{{- else }}
 - name: REGISTRY_ZOOKEEPER_CONNECT_STRING
   {{- if .Values.zookeeper.enabled }}
   value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
   {{- else }}
   value: {{ .Values.externalRegistry.registryServers }}
   {{- end }}
+{{- end }}
 {{- end -}}
 
 {{/*
@@ -264,3 +285,31 @@ Create a fsFileResourcePersistence volumeMount.
   name: {{ include "dolphinscheduler.fullname" . }}-fs-file
 {{- end -}}
 {{- end -}}
+
+{{/*
+Create a etcd ssl volume.
+*/}}
+{{- define "dolphinscheduler.etcd.ssl.volume" -}}
+{{- if .Values.etcd.ssl.enabled -}}
+- name: etcd-ssl
+  secret:
+    secretName: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create a etcd ssl volumeMount.
+*/}}
+{{- define "dolphinscheduler.etcd.ssl.volumeMount" -}}
+{{- if .Values.etcd.ssl.enabled -}}
+- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.certFile }}
+  name: etcd-ssl
+  subPath: cert-file
+- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.keyCertChainFile  }}
+  name: etcd-ssl
+  subPath: key-cert-chain-file
+- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.keyFile }}
+  name: etcd-ssl
+  subPath: key-file
+{{- end -}}
+{{- end -}}

deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@@ -111,6 +111,7 @@ spec:
               subPath: common_properties
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
+            {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
       volumes:
         - name: {{ include "dolphinscheduler.fullname" . }}-api
           {{- if .Values.api.persistentVolumeClaim.enabled }}
@@ -124,3 +125,4 @@ spec:
             name: {{ include "dolphinscheduler.fullname" . }}-configs
         {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
         {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
+        {{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }}

deploy/kubernetes/dolphinscheduler/templates/secret-external-etcd-ssl.yaml

@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if .Values.etcd.ssl.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl
+  labels:
+    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl
+    {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
+type: Opaque
+data:
+  cert-file: {{ .Files.Get .Values.etcd.ssl.certFile | b64enc | quote }}
+  key-cert-chain-file: {{ .Files.Get .Values.etcd.ssl.keyCertChainFile | b64enc | quote }}
+  key-file: {{ .Files.Get .Values.etcd.ssl.keyFile | b64enc | quote }}
+{{- end }}

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@@ -105,6 +105,7 @@ spec:
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
               subPath: common_properties
+            {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
       volumes:
         - name: {{ include "dolphinscheduler.fullname" . }}-master
           {{- if .Values.master.persistentVolumeClaim.enabled }}
@@ -117,6 +118,7 @@ spec:
         - name: config-volume
           configMap:
             name: {{ include "dolphinscheduler.fullname" . }}-configs
+        {{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }}
   {{- if .Values.master.persistentVolumeClaim.enabled }}
   volumeClaimTemplates:
     - metadata:

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -108,6 +108,7 @@ spec:
               subPath: common_properties
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
+            {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
       volumes:
         {{- if .Values.worker.persistentVolumeClaim.enabled }}
         - name: {{ include "dolphinscheduler.fullname" . }}-worker-data
@@ -135,6 +136,7 @@ spec:
             name: {{ include "dolphinscheduler.fullname" . }}-configs
         {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
         {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
+        {{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }}
   {{- if .Values.worker.persistentVolumeClaim.enabled }}
   volumeClaimTemplates:
     {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}

deploy/kubernetes/dolphinscheduler/values.yaml

@@ -94,6 +94,20 @@ zookeeper:
     size: "20Gi"
     storageClass: "-"
 
+etcd:
+  enabled: false
+  endpoints: ""
+  namespace: "dolphinscheduler"
+  user: ""
+  passWord: ""
+  authority: ""
+  # Please create a new folder: deploy/kubernetes/dolphinscheduler/etcd-certs
+  ssl:
+    enabled: false
+    certFile: "etcd-certs/ca.crt"
+    keyCertChainFile: "etcd-certs/client.crt"
+    keyFile: "etcd-certs/client.pem"
+
 ## If exists external registry and set zookeeper.enable value to false, the external registry will be used.
 externalRegistry:
   registryPluginName: "zookeeper"

dolphinscheduler-master/pom.xml

@@ -154,6 +154,10 @@
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-reload4j</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -250,6 +254,10 @@
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-reload4j</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md

@@ -12,17 +12,31 @@ registry:
   endpoints: "http://etcd0:2379, http://etcd1:2379, http://etcd2:2379"
   # The options below have default values
   namespace: dolphinscheduler
-  connectionTimeout: 9s
+  connection-timeout: 9s
   # The unit is milliseconds
-  retryDelay: 60ms
+  retry-delay: 60ms
-  retryMaxDelay: 300ms
+  retry-max-delay: 300ms
-  retryMaxDuration: 1500ms
+  retry-max-duration: 1500ms
-  # The following options are set according to personal needs
+  # The following ssl options are set according to personal needs
+  cert-file: "deploy/kubernetes/dolphinscheduler/etcd-certs/ca.crt"
+  key-cert-chain-file: "deploy/kubernetes/dolphinscheduler/etcd-certs/client.crt"
+  key-file: "deploy/kubernetes/dolphinscheduler/etcd-certs/client.pem"
+  # The following auth options are set according to personal needs
   user: ""
   password: ""
   authority: ""
-  loadBalancerPolicy: ""
+  load-balancer-policy: ""
 ```
 
+If your etcd server has configured with ssl, about certification files you can see [here](https://github.com/etcd-io/jetcd/blob/main/docs/SslConfig.md) for how to convert.
+
+> If you need ssl certification, you need to make sure your jdk version is newer than Java 8u252 (April 2020), jdk11 works well too. 
+>
+> By the way, the jdk version in docker images `FROM eclipse-temurin:8-jre` now is 8u362 works well, don't need change.
+>
+> Because after version 8u252 has native support for ALPN. Detail you can see:
+> 
+> https://github.com/grpc/grpc-java/issues/5369#issuecomment-751885384
+
 After do this config, you can start your DolphinScheduler cluster, your cluster will use etcd as registry center to
 store server metadata.

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java

@@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.registry.api.Registry;
 import org.apache.dolphinscheduler.registry.api.RegistryException;
 import org.apache.dolphinscheduler.registry.api.SubscribeListener;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
@@ -36,6 +37,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
 import javax.annotation.PostConstruct;
+import javax.net.ssl.SSLException;
 
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
@@ -60,6 +62,8 @@ import io.etcd.jetcd.options.PutOption;
 import io.etcd.jetcd.options.WatchOption;
 import io.etcd.jetcd.support.Observers;
 import io.etcd.jetcd.watch.WatchEvent;
+import io.grpc.netty.GrpcSslContexts;
+import io.netty.handler.ssl.SslContext;
 
 /**
  * This is one of the implementation of {@link Registry}, with this implementation, you need to rely on Etcd cluster to
@@ -80,7 +84,7 @@ public class EtcdRegistry implements Registry {
     private final Map<String, Watch.Watcher> watcherMap = new ConcurrentHashMap<>();
 
     private static final long TIME_TO_LIVE_SECONDS = 30L;
-    public EtcdRegistry(EtcdRegistryProperties registryProperties) {
+    public EtcdRegistry(EtcdRegistryProperties registryProperties) throws SSLException {
         ClientBuilder clientBuilder = Client.builder()
                 .endpoints(Util.toURIs(Splitter.on(",").trimResults().splitToList(registryProperties.getEndpoints())))
                 .namespace(byteSequence(registryProperties.getNamespace()))
@@ -100,6 +104,19 @@ public class EtcdRegistry implements Registry {
         if (StringUtils.hasLength(registryProperties.getAuthority())) {
             clientBuilder.authority(registryProperties.getAuthority());
         }
+        if (StringUtils.hasLength(registryProperties.getCertFile())
+                && StringUtils.hasLength(registryProperties.getKeyCertChainFile())
+                && StringUtils.hasLength(registryProperties.getKeyFile())) {
+            String userDir = System.getProperty("user.dir") + "/";
+            File certFile = new File(userDir + registryProperties.getCertFile());
+            File keyCertChainFile = new File(userDir + registryProperties.getKeyCertChainFile());
+            File keyFile = new File(userDir + registryProperties.getKeyFile());
+            SslContext context = GrpcSslContexts.forClient()
+                    .trustManager(certFile)
+                    .keyManager(keyCertChainFile, keyFile)
+                    .build();
+            clientBuilder.sslContext(context);
+        }
         client = clientBuilder.build();
         log.info("Started Etcd Registry...");
         etcdConnectionStateListener = new EtcdConnectionStateListener(client);

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java

@@ -47,4 +47,9 @@ public class EtcdRegistryProperties {
 
     // loadBalancerPolicy
     private String loadBalancerPolicy;
+
+    // ssl
+    private String certFile;
+    private String keyCertChainFile;
+    private String keyFile;
 }