[Feature][Api] LDAP SSL supports&& Add LDAP K8s configuration (#14349)

deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl

@@ -177,6 +177,38 @@ Create a database environment variables.
       {{- end }}
 {{- end -}}
 
+{{/*
+Create a security environment variables.
+*/}}
+{{- define "dolphinscheduler.security.env_vars" -}}
+- name: SECURITY_AUTHENTICATION_TYPE
+  value: {{ .Values.security.authentication.type | quote }}
+{{- if eq .Values.security.authentication.type "LDAP" }}
+- name: SECURITY_AUTHENTICATION_LDAP_URLS
+  value: {{ .Values.security.authentication.ldap.urls | quote }}
+- name: SECURITY_AUTHENTICATION_LDAP_BASE_DN
+  value: {{ .Values.security.authentication.ldap.basedn | quote }}
+- name: SECURITY_AUTHENTICATION_LDAP_USERNAME
+  value: {{ .Values.security.authentication.ldap.username | quote }}
+- name: SECURITY_AUTHENTICATION_LDAP_PASSWORD
+  value: {{ .Values.security.authentication.ldap.password | quote }}
+- name: SECURITY_AUTHENTICATION_LDAP_USER_ADMIN
+  value: {{ .Values.security.authentication.ldap.user.admin | quote }}
+- name: SECURITY_AUTHENTICATION_LDAP_USER_IDENTITY_ATTRIBUTE
+  value: {{ .Values.security.authentication.ldap.user.identityattribute | quote }}
+- name: SECURITY_AUTHENTICATION_LDAP_USER_EMAIL_ATTRIBUTE
+  value: {{ .Values.security.authentication.ldap.user.emailattribute | quote }}
+- name: SECURITY_AUTHENTICATION_LDAP_USER_NOT_EXIST_ACTION
+  value: {{ .Values.security.authentication.ldap.user.notexistaction | quote }}
+- name: SECURITY_AUTHENTICATION_LDAP_SSL_ENABLE
+  value: {{ .Values.security.authentication.ldap.ssl.enable | quote }}
+- name: SECURITY_AUTHENTICATION_LDAP_SSL_TRUST_STORE
+  value: {{ .Values.security.authentication.ldap.ssl.truststore | quote }}
+- name: SECURITY_AUTHENTICATION_LDAP_SSL_TRUST_STORE_PASSWORD
+  value: {{ .Values.security.authentication.ldap.ssl.truststorepassword | quote }}
+{{- end }}
+{{- end -}}
+
 {{/*
 Wait for database to be ready.
 */}}
@@ -313,3 +345,25 @@ Create a etcd ssl volumeMount.
   subPath: key-file
 {{- end -}}
 {{- end -}}
+
+{{/*
+Create a ldap ssl volume.
+*/}}
+{{- define "dolphinscheduler.ldap.ssl.volume" -}}
+{{- if .Values.security.authentication.ldap.ssl.enable -}}
+- name: jks-file
+  secret:
+    secretName: {{ include "dolphinscheduler.fullname" . }}-ldap-ssl
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create a ldap ssl volumeMount.
+*/}}
+{{- define "dolphinscheduler.ldap.ssl.volumeMount" -}}
+{{- if .Values.security.authentication.ldap.ssl.enable -}}
+- mountPath: {{ .Values.security.authentication.ldap.ssl.truststore }}
+  name: jks-file
+  subPath: jks-file
+{{- end -}}
+{{- end -}}

deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@@ -72,6 +72,7 @@ spec:
               value: {{ .Values.timezone }}
             {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
             {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }}
+            {{- include "dolphinscheduler.security.env_vars" . | nindent 12 }}
             {{ range $key, $value :=  .Values.api.env }}
             - name: {{ $key }}
               value: {{ $value | quote }}
@@ -111,6 +112,7 @@ spec:
               subPath: common_properties
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
+            {{- include "dolphinscheduler.ldap.ssl.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
       volumes:
         - name: {{ include "dolphinscheduler.fullname" . }}-api
@@ -125,4 +127,5 @@ spec:
             name: {{ include "dolphinscheduler.fullname" . }}-configs
         {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
         {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
+        {{- include "dolphinscheduler.ldap.ssl.volume" . | nindent 8 }}
         {{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }}

deploy/kubernetes/dolphinscheduler/templates/secret-external-ldap-ssl.yaml

@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if .Values.security.authentication.ldap.ssl.enable }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-ldap-ssl
+  labels:
+    app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-ldap-ssl
+    {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
+type: Opaque
+data:
+  jks-file: {{ .Values.security.authentication.ldap.ssl.jksbase64content | quote }}
+{{- end }}

deploy/kubernetes/dolphinscheduler/values.yaml

@@ -113,6 +113,30 @@ externalRegistry:
   registryPluginName: "zookeeper"
   registryServers: "127.0.0.1:2181"
 
+security:
+  authentication:
+    type: PASSWORD
+    ldap:
+      urls: ldap://ldap.forumsys.com:389/
+      basedn: dc=example,dc=com
+      username: cn=read-only-admin,dc=example,dc=com
+      password: password
+      user:
+        admin: read-only-admin
+        identityattribute: uid
+        emailattribute: mail
+        notexistaction: CREATE
+      ssl:
+        enable: false
+        # do not change this value
+        truststore: "/opt/ldapkeystore.jks"
+        # if you use macOS, please run `base64 -b 0 -i /path/to/your.jks`
+        # if you use Linux, please run `base64 -w 0 /path/to/your.jks`
+        # if you use Windows, please run `certutil -f -encode /path/to/your.jks`
+        # Then copy the base64 content to below field in one line
+        jksbase64content: ""
+        truststorepassword: ""
+
 conf:
   common:
     # user data local directory path, please make sure the directory exists and have read write permissions

docs/docs/en/architecture/configuration.md

@@ -251,6 +251,9 @@ Location: `api-server/conf/application.yaml`
 |security.authentication.ldap.user.identity-attribute|uid|LDAP user identity attribute|
 |security.authentication.ldap.user.email-attribute|mail|LDAP user email attribute|
 |security.authentication.ldap.user.not-exist-action|CREATE|action when ldap user is not exist,default value: CREATE. Optional values include(CREATE,DENY)|
+|security.authentication.ldap.ssl.enable|false|LDAP switch|
+|security.authentication.ldap.ssl.trust-store|ldapkeystore.jks|LDAP jks file absolute path|
+|security.authentication.ldap.ssl.trust-store-password|password|LDAP jks password|
 |traffic.control.global.switch|false|traffic control global switch|
 |traffic.control.max-global-qps-rate|300|global max request number per second|
 |traffic.control.tenant-switch|false|traffic control tenant switch|

docs/docs/en/guide/security/authentication-type.md

@@ -25,6 +25,11 @@ security:
         email-attribute: mail
         # action when ldap user is not exist (supported types: CREATE,DENY)
         not-exist-action: CREATE
+      ssl:
+        enable: false
+        # jks file absolute path && password
+        trust-store: "/ldapkeystore.jks"
+        trust-store-password: "password"
 ```
 
 For detailed explanation of specific fields, please see: [Api-server related configuration](../../architecture/configuration.md)
@@ -41,6 +46,17 @@ You can follow guide below：
 - Change the expected email to the return value you expect in the `ldapLogin` method.
 - Run`ldapLogin`method and determine whether the LDAP login result is expected.
 
+If you want to enable ssl, please change configuration in `TestPropertySource` like below:
+
+```
+security.authentication.ldap.ssl.enable=false
+// absolute path
+security.authentication.ldap.ssl.trust-store=/ldapkeystore.jks
+security.authentication.ldap.ssl.trust-store-password=yourpassword
+```
+
+Then run`ldapLoginSSL`method and determine whether the LDAP login result is expected.
+
 ## Casdoor SSO
 
 [Casdoor](https://casdoor.org/) is a UI-first Identity Access Management (IAM) / Single-Sign-On (SSO) platform based on OAuth 2.0, OIDC, SAML and CAS. You can add SSO capability to Dolphinscheduler through Casdoor by following these steps:

docs/docs/zh/architecture/configuration.md

@@ -247,6 +247,9 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
 |security.authentication.ldap.user.identity-attribute|uid|LDAP用户身份标识字段名|
 |security.authentication.ldap.user.email-attribute|mail|LDAP邮箱字段名|
 |security.authentication.ldap.user.not-exist-action|CREATE|当通过LDAP登陆时用户不存在的操作，默认值是: CREATE，可选值:CREATE、DENY|
+|security.authentication.ldap.ssl.enable|false|LDAP ssl开关|
+|security.authentication.ldap.ssl.trust-store|ldapkeystore.jks|LDAP jks文件绝对路径|
+|security.authentication.ldap.ssl.trust-store-password|password|LDAP jks密码|
 |traffic.control.global.switch|false|流量控制全局开关|
 |traffic.control.max-global-qps-rate|300|全局最大请求数/秒|
 |traffic.control.tenant-switch|false|流量控制租户开关|

docs/docs/zh/guide/security/authentication-type.md

@@ -25,11 +25,16 @@ security:
         email-attribute: mail
         # action when ldap user is not exist (supported types: CREATE,DENY)
         not-exist-action: CREATE
+      ssl:
+        enable: false
+        # jks file absolute path && password
+        trust-store: "/ldapkeystore.jks"
+        trust-store-password: "password"
 ```
 
 具体字段解释详见：[Api-server相关配置](../../architecture/configuration.md)
 
-## LDAP测试
+## 开发者LDAP测试
 
 我们提供了一个单元测试类，可以在不启动项目的情况下测试DolphinScheduler与LDAP的集成。
 
@@ -41,6 +46,17 @@ security:
 - 修改`ldapLogin`方法中的expected email为正常登陆的返回值;
 - 执行`ldapLogin`方法，判断LDAP登陆结果是否为预期;
 
+如果你要启用ssl，请修改`TestPropertySource`配置中ssl相关参数为：
+
+```
+security.authentication.ldap.ssl.enable=false
+// absolute path
+security.authentication.ldap.ssl.trust-store=/ldapkeystore.jks 
+security.authentication.ldap.ssl.trust-store-password=yourpassword
+```
+
+运行`ldapLoginSSL`方法，判断email是否为预期的返回值。
+
 ## 通过 Casdoor 实现 SSO 登录
 
 Casdoor 是基于 OAuth 2.0、OIDC、SAML 和 CAS 的面向 UI 的身份访问管理（IAM）/单点登录（SSO）平台。您可以通过以下步骤通过 Casdoor 为 Dolphinscheduler 添加 SSO 功能：

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapService.java

@@ -70,6 +70,15 @@ public class LdapService {
     @Value("${security.authentication.ldap.user.not-exist-action:CREATE}")
     private String ldapUserNotExistAction;
 
+    @Value("${security.authentication.ldap.ssl.enable:false}")
+    private Boolean sslEnable;
+
+    @Value("${security.authentication.ldap.ssl.trust-store:#{null}}")
+    private String trustStore;
+
+    @Value("${security.authentication.ldap.ssl.trust-store-password:#{null}}")
+    private String trustStorePassword;
+
     /***
      * get user type by configured admin userId
      * @param userId login userId
@@ -144,6 +153,14 @@ public class LdapService {
         env.put(Context.SECURITY_PRINCIPAL, ldapSecurityPrincipal);
         env.put(Context.SECURITY_CREDENTIALS, ldapPrincipalPassword);
         env.put(Context.PROVIDER_URL, ldapUrls);
+
+        if (sslEnable) {
+            env.put(Context.SECURITY_PROTOCOL, "ssl");
+            System.setProperty("javax.net.ssl.trustStore", trustStore);
+            if (StringUtils.isNotEmpty(trustStorePassword)) {
+                System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword);
+            }
+        }
         return env;
     }
 

dolphinscheduler-api/src/main/resources/application.yaml

@@ -162,6 +162,11 @@ security:
         email-attribute: mail
         # action when ldap user is not exist (supported types: CREATE,DENY)
         not-exist-action: CREATE
+      ssl:
+        enable: false
+        # jks file absolute path && password
+        trust-store: "/ldapkeystore.jks"
+        trust-store-password: "password"
 
 # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
 # global max request number per second

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapAuthenticatorTest.java

@@ -58,6 +58,9 @@ import org.springframework.test.context.TestPropertySource;
         "security.authentication.ldap.user.identity-attribute=uid",
         "security.authentication.ldap.user.email-attribute=mail",
         "security.authentication.ldap.user.not-exist-action=CREATE",
+        "security.authentication.ldap.ssl.enable=false",
+        "security.authentication.ldap.ssl.trust-store=",
+        "security.authentication.ldap.ssl.trust-store-password=",
 })
 public class LdapAuthenticatorTest extends AbstractControllerTest {
 

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapServiceTest.java

@@ -21,6 +21,8 @@ import org.apache.dolphinscheduler.api.ApiApplicationServer;
 import org.apache.dolphinscheduler.common.enums.ProfileType;
 import org.apache.dolphinscheduler.common.enums.UserType;
 
+import java.lang.reflect.Field;
+
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
@@ -44,6 +46,9 @@ import org.springframework.test.context.TestPropertySource;
         "security.authentication.ldap.user.identity-attribute=uid",
         "security.authentication.ldap.user.email-attribute=mail",
         "security.authentication.ldap.user.not-exist-action=CREATE",
+        "security.authentication.ldap.ssl.enable=false",
+        "security.authentication.ldap.ssl.trust-store=",
+        "security.authentication.ldap.ssl.trust-store-password=",
 })
 public class LdapServiceTest {
 
@@ -52,6 +57,9 @@ public class LdapServiceTest {
 
     private LdapService ldapService;
 
+    private final String username = "tesla";
+    private final String correctPassword = "password";
+
     @BeforeEach
     public void setUp() {
         ldapService = new LdapService();
@@ -65,14 +73,35 @@ public class LdapServiceTest {
     }
 
     @Test
-    public void ldapLogin() {
+    public void ldapLogin() throws NoSuchFieldException, IllegalAccessException {
-        String email = ldapService.ldapLogin("tesla", "password");
+        changeSslEnable(false);
+        String email = ldapService.ldapLogin(username, correctPassword);
         Assertions.assertEquals("tesla@ldap.forumsys.com", email);
     }
 
     @Test
-    public void ldapLoginError() {
+    public void ldapLoginError() throws NoSuchFieldException, IllegalAccessException {
-        String email2 = ldapService.ldapLogin("tesla", "error password");
+        changeSslEnable(false);
+        String email2 = ldapService.ldapLogin(username, "error password");
         Assertions.assertNull(email2);
     }
+
+    @Test
+    public void ldapLoginSSL() throws NoSuchFieldException, IllegalAccessException {
+        changeSslEnable(true);
+        String email = ldapService.ldapLogin(username, correctPassword);
+        Assertions.assertNull(email);
+    }
+
+    private void changeSslEnable(boolean sslEnable) throws NoSuchFieldException, IllegalAccessException {
+        Class<LdapService> ldapServiceClass = LdapService.class;
+        Field sslEnableField = ldapServiceClass.getDeclaredField("sslEnable");
+        sslEnableField.setAccessible(true);
+        sslEnableField.set(ldapService, sslEnable);
+        if (sslEnable) {
+            Field trustStorePasswordField = ldapServiceClass.getDeclaredField("trustStorePassword");
+            trustStorePasswordField.setAccessible(true);
+            trustStorePasswordField.set(ldapService, "trustStorePassword");
+        }
+    }
 }

dolphinscheduler-standalone-server/src/main/resources/application.yaml

@@ -106,6 +106,11 @@ security:
         email-attribute: mail
         # action when ldap user is not exist (supported types: CREATE,DENY)
         not-exist-action: CREATE
+      ssl:
+        enable: false
+        # jks file absolute path && password
+        trust-store: "/ldapkeystore.jks"
+        trust-store-password: ""
 
 # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
 # global max request number per second