diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl index 3991a198d9..37bdc7909f 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -212,15 +212,36 @@ Create a registry environment variables. - name: REGISTRY_TYPE {{- if .Values.zookeeper.enabled }} value: "zookeeper" + {{- else if .Values.etcd.enabled }} + value: "etcd" {{- else }} value: {{ .Values.externalRegistry.registryPluginName }} {{- end }} +{{- if .Values.etcd.enabled }} +- name: REGISTRY_ENDPOINTS + value: {{ .Values.etcd.endpoints }} +- name: REGISTRY_NAMESPACE + value: {{ .Values.etcd.namespace }} +- name: REGISTRY_USER + value: {{ .Values.etcd.user }} +- name: REGISTRY_PASSWORD + value: {{ .Values.etcd.passWord }} +- name: REGISTRY_AUTHORITY + value: {{ .Values.etcd.authority }} +- name: REGISTRY_CERT_FILE + value: {{ .Values.etcd.ssl.certFile }} +- name: REGISTRY_KEY_CERT_CHAIN_FILE + value: {{ .Values.etcd.ssl.keyCertChainFile }} +- name: REGISTRY_KEY_FILE + value: {{ .Values.etcd.ssl.keyFile }} +{{- else }} - name: REGISTRY_ZOOKEEPER_CONNECT_STRING {{- if .Values.zookeeper.enabled }} value: {{ template "dolphinscheduler.zookeeper.quorum" . }} {{- else }} value: {{ .Values.externalRegistry.registryServers }} {{- end }} +{{- end }} {{- end -}} {{/* @@ -264,3 +285,31 @@ Create a fsFileResourcePersistence volumeMount. name: {{ include "dolphinscheduler.fullname" . }}-fs-file {{- end -}} {{- end -}} + +{{/* +Create a etcd ssl volume. +*/}} +{{- define "dolphinscheduler.etcd.ssl.volume" -}} +{{- if .Values.etcd.ssl.enabled -}} +- name: etcd-ssl + secret: + secretName: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl +{{- end -}} +{{- end -}} + +{{/* +Create a etcd ssl volumeMount. +*/}} +{{- define "dolphinscheduler.etcd.ssl.volumeMount" -}} +{{- if .Values.etcd.ssl.enabled -}} +- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.certFile }} + name: etcd-ssl + subPath: cert-file +- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.keyCertChainFile }} + name: etcd-ssl + subPath: key-cert-chain-file +- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.keyFile }} + name: etcd-ssl + subPath: key-file +{{- end -}} +{{- end -}} diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 159d893ac3..bce1c2e8d9 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -111,6 +111,7 @@ spec: subPath: common_properties {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }} + {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }} volumes: - name: {{ include "dolphinscheduler.fullname" . }}-api {{- if .Values.api.persistentVolumeClaim.enabled }} @@ -124,3 +125,4 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-configs {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }} + {{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/secret-external-etcd-ssl.yaml b/deploy/kubernetes/dolphinscheduler/templates/secret-external-etcd-ssl.yaml new file mode 100644 index 0000000000..f2ab7474b2 --- /dev/null +++ b/deploy/kubernetes/dolphinscheduler/templates/secret-external-etcd-ssl.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.etcd.ssl.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} +type: Opaque +data: + cert-file: {{ .Files.Get .Values.etcd.ssl.certFile | b64enc | quote }} + key-cert-chain-file: {{ .Files.Get .Values.etcd.ssl.keyCertChainFile | b64enc | quote }} + key-file: {{ .Files.Get .Values.etcd.ssl.keyFile | b64enc | quote }} +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index 3c1c507cd2..1314889468 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -105,6 +105,7 @@ spec: - name: config-volume mountPath: /opt/dolphinscheduler/conf/common.properties subPath: common_properties + {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }} volumes: - name: {{ include "dolphinscheduler.fullname" . }}-master {{- if .Values.master.persistentVolumeClaim.enabled }} @@ -117,6 +118,7 @@ spec: - name: config-volume configMap: name: {{ include "dolphinscheduler.fullname" . }}-configs + {{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }} {{- if .Values.master.persistentVolumeClaim.enabled }} volumeClaimTemplates: - metadata: diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index bae447c2cd..1be6f55a5b 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -108,6 +108,7 @@ spec: subPath: common_properties {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }} + {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }} volumes: {{- if .Values.worker.persistentVolumeClaim.enabled }} - name: {{ include "dolphinscheduler.fullname" . }}-worker-data @@ -135,6 +136,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-configs {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }} + {{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }} {{- if .Values.worker.persistentVolumeClaim.enabled }} volumeClaimTemplates: {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 27eaab94f2..49859676ee 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -94,6 +94,20 @@ zookeeper: size: "20Gi" storageClass: "-" +etcd: + enabled: false + endpoints: "" + namespace: "dolphinscheduler" + user: "" + passWord: "" + authority: "" + # Please create a new folder: deploy/kubernetes/dolphinscheduler/etcd-certs + ssl: + enabled: false + certFile: "etcd-certs/ca.crt" + keyCertChainFile: "etcd-certs/client.crt" + keyFile: "etcd-certs/client.pem" + ## If exists external registry and set zookeeper.enable value to false, the external registry will be used. externalRegistry: registryPluginName: "zookeeper" diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index be975407e9..b5b3c09cb5 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -154,6 +154,10 @@ org.slf4j slf4j-reload4j + + com.google.protobuf + protobuf-java + @@ -250,6 +254,10 @@ org.slf4j slf4j-reload4j + + com.google.protobuf + protobuf-java + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md index 156dbfba74..cd5795fc65 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md @@ -12,17 +12,31 @@ registry: endpoints: "http://etcd0:2379, http://etcd1:2379, http://etcd2:2379" # The options below have default values namespace: dolphinscheduler - connectionTimeout: 9s + connection-timeout: 9s # The unit is milliseconds - retryDelay: 60ms - retryMaxDelay: 300ms - retryMaxDuration: 1500ms - # The following options are set according to personal needs + retry-delay: 60ms + retry-max-delay: 300ms + retry-max-duration: 1500ms + # The following ssl options are set according to personal needs + cert-file: "deploy/kubernetes/dolphinscheduler/etcd-certs/ca.crt" + key-cert-chain-file: "deploy/kubernetes/dolphinscheduler/etcd-certs/client.crt" + key-file: "deploy/kubernetes/dolphinscheduler/etcd-certs/client.pem" + # The following auth options are set according to personal needs user: "" password: "" authority: "" - loadBalancerPolicy: "" + load-balancer-policy: "" ``` +If your etcd server has configured with ssl, about certification files you can see [here](https://github.com/etcd-io/jetcd/blob/main/docs/SslConfig.md) for how to convert. + +> If you need ssl certification, you need to make sure your jdk version is newer than Java 8u252 (April 2020), jdk11 works well too. +> +> By the way, the jdk version in docker images `FROM eclipse-temurin:8-jre` now is 8u362 works well, don't need change. +> +> Because after version 8u252 has native support for ALPN. Detail you can see: +> +> https://github.com/grpc/grpc-java/issues/5369#issuecomment-751885384 + After do this config, you can start your DolphinScheduler cluster, your cluster will use etcd as registry center to store server metadata. diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java index ff6afdeea4..10d3dfc226 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.registry.api.Registry; import org.apache.dolphinscheduler.registry.api.RegistryException; import org.apache.dolphinscheduler.registry.api.SubscribeListener; +import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -36,6 +37,7 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import javax.annotation.PostConstruct; +import javax.net.ssl.SSLException; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -60,6 +62,8 @@ import io.etcd.jetcd.options.PutOption; import io.etcd.jetcd.options.WatchOption; import io.etcd.jetcd.support.Observers; import io.etcd.jetcd.watch.WatchEvent; +import io.grpc.netty.GrpcSslContexts; +import io.netty.handler.ssl.SslContext; /** * This is one of the implementation of {@link Registry}, with this implementation, you need to rely on Etcd cluster to @@ -80,7 +84,7 @@ public class EtcdRegistry implements Registry { private final Map watcherMap = new ConcurrentHashMap<>(); private static final long TIME_TO_LIVE_SECONDS = 30L; - public EtcdRegistry(EtcdRegistryProperties registryProperties) { + public EtcdRegistry(EtcdRegistryProperties registryProperties) throws SSLException { ClientBuilder clientBuilder = Client.builder() .endpoints(Util.toURIs(Splitter.on(",").trimResults().splitToList(registryProperties.getEndpoints()))) .namespace(byteSequence(registryProperties.getNamespace())) @@ -100,6 +104,19 @@ public class EtcdRegistry implements Registry { if (StringUtils.hasLength(registryProperties.getAuthority())) { clientBuilder.authority(registryProperties.getAuthority()); } + if (StringUtils.hasLength(registryProperties.getCertFile()) + && StringUtils.hasLength(registryProperties.getKeyCertChainFile()) + && StringUtils.hasLength(registryProperties.getKeyFile())) { + String userDir = System.getProperty("user.dir") + "/"; + File certFile = new File(userDir + registryProperties.getCertFile()); + File keyCertChainFile = new File(userDir + registryProperties.getKeyCertChainFile()); + File keyFile = new File(userDir + registryProperties.getKeyFile()); + SslContext context = GrpcSslContexts.forClient() + .trustManager(certFile) + .keyManager(keyCertChainFile, keyFile) + .build(); + clientBuilder.sslContext(context); + } client = clientBuilder.build(); log.info("Started Etcd Registry..."); etcdConnectionStateListener = new EtcdConnectionStateListener(client); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java index 593752a7c0..faded2a506 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java @@ -47,4 +47,9 @@ public class EtcdRegistryProperties { // loadBalancerPolicy private String loadBalancerPolicy; + + // ssl + private String certFile; + private String keyCertChainFile; + private String keyFile; }