From 1c15a7d60536f0b02f1f285a083600088fd54278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=88=B1=E7=A9=BF=E6=A0=BC=E5=AD=90=E8=A3=A4?= <30542109+yutianaiqingtian@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:06:30 +0800 Subject: [PATCH] [fix-11404]: make the common.properties to configurable on values.yaml (#11441) and set the resource.storage.type default to hdfs --- .../resources/config/common.properties | 21 +++++ .../dolphinscheduler/templates/configmap.yaml | 26 +++++++ .../deployment-dolphinscheduler-alert.yaml | 6 ++ .../deployment-dolphinscheduler-api.yaml | 6 ++ .../statefulset-dolphinscheduler-master.yaml | 6 ++ .../statefulset-dolphinscheduler-worker.yaml | 6 ++ .../kubernetes/dolphinscheduler/values.yaml | 78 +++++++++++++++++++ 7 files changed, 149 insertions(+) create mode 100644 deploy/kubernetes/dolphinscheduler/resources/config/common.properties create mode 100644 deploy/kubernetes/dolphinscheduler/templates/configmap.yaml diff --git a/deploy/kubernetes/dolphinscheduler/resources/config/common.properties b/deploy/kubernetes/dolphinscheduler/resources/config/common.properties new file mode 100644 index 0000000000..a152640c0f --- /dev/null +++ b/deploy/kubernetes/dolphinscheduler/resources/config/common.properties @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if index .Values.conf "common" }} +{{- range $key, $value := index .Values.conf "common" }} +{{ $key }}={{ $value }} +{{- end }} +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml new file mode 100644 index 0000000000..57d5155c37 --- /dev/null +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-configs + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} +data: + common_properties: |- +{{ tpl (.Files.Get "resources/config/common.properties") . | indent 4 }} \ No newline at end of file diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index 49ff69ac83..3f7f2796d7 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -104,6 +104,9 @@ spec: volumeMounts: - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: config-volume + mountPath: /opt/dolphinscheduler/conf/common.properties + subPath: common_properties volumes: - name: {{ include "dolphinscheduler.fullname" . }}-alert {{- if .Values.alert.persistentVolumeClaim.enabled }} @@ -112,3 +115,6 @@ spec: {{- else }} emptyDir: {} {{- end }} + - name: config-volume + configMap: + name: {{ include "dolphinscheduler.fullname" . }}-configs \ No newline at end of file diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 41cf6a55ed..789c8eaaf5 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -106,6 +106,9 @@ spec: volumeMounts: - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-api + - name: config-volume + mountPath: /opt/dolphinscheduler/conf/common.properties + subPath: common_properties {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }} volumes: @@ -116,5 +119,8 @@ spec: {{- else }} emptyDir: {} {{- end }} + - name: config-volume + configMap: + name: {{ include "dolphinscheduler.fullname" . }}-configs {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index f1f8f4b86e..1dc6ee7afd 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -102,6 +102,9 @@ spec: - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-master {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} + - name: config-volume + mountPath: /opt/dolphinscheduler/conf/common.properties + subPath: common_properties volumes: - name: {{ include "dolphinscheduler.fullname" . }}-master {{- if .Values.master.persistentVolumeClaim.enabled }} @@ -111,6 +114,9 @@ spec: emptyDir: {} {{- end }} {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} + - name: config-volume + configMap: + name: {{ include "dolphinscheduler.fullname" . }}-configs {{- if .Values.master.persistentVolumeClaim.enabled }} volumeClaimTemplates: - metadata: diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index ebab09f279..ce266d1eaf 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -105,6 +105,9 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-worker-data - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-worker-logs + - name: config-volume + mountPath: /opt/dolphinscheduler/conf/common.properties + subPath: common_properties {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }} volumes: @@ -122,6 +125,9 @@ spec: {{- else }} emptyDir: {} {{- end }} + - name: config-volume + configMap: + name: {{ include "dolphinscheduler.fullname" . }}-configs {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }} {{- if .Values.worker.persistentVolumeClaim.enabled }} diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 6fe4bdd99c..791bd65a4d 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -66,6 +66,84 @@ externalRegistry: registryPluginName: "zookeeper" registryServers: "127.0.0.1:2181" +conf: + common: + # user data local directory path, please make sure the directory exists and have read write permissions + data.basedir.path: /tmp/dolphinscheduler + + # resource storage type: HDFS, S3, NONE + resource.storage.type: HDFS + + # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended + resource.upload.path: /dolphinscheduler + + # whether to startup kerberos + hadoop.security.authentication.startup.state: false + + # java.security.krb5.conf path + java.security.krb5.conf.path: /opt/krb5.conf + + # login user from keytab username + login.user.keytab.username: hdfs-mycluster@ESZ.COM + + # login user from keytab path + login.user.keytab.path: /opt/hdfs.headless.keytab + + # kerberos expire time, the unit is hour + kerberos.expire.time: 2 + # resource view suffixs + #resource.view.suffixs: txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js + # if resource.storage.type: HDFS, the user must have the permission to create directories under the HDFS root path + hdfs.root.user: hdfs + # if resource.storage.type: S3, the value like: s3a://dolphinscheduler; if resource.storage.type: HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir + fs.defaultFS: file:/// + aws.access.key.id: minioadmin + aws.secret.access.key: minioadmin + aws.region: us-east-1 + aws.endpoint: http://localhost:9000 + # resourcemanager port, the default value is 8088 if not specified + resource.manager.httpaddress.port: 8088 + # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty + yarn.resourcemanager.ha.rm.ids: 192.168.xx.xx,192.168.xx.xx + # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname + yarn.application.status.address: http://ds1:%s/ws/v1/cluster/apps/%s + # job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) + yarn.job.history.status.address: http://ds1:19888/ws/v1/history/mapreduce/jobs/%s + + # datasource encryption enable + datasource.encryption.enable: false + + # datasource encryption salt + datasource.encryption.salt: '!@#$%^&*' + + # data quality option + data-quality.jar.name: dolphinscheduler-data-quality-dev-SNAPSHOT.jar + + #data-quality.error.output.path: /tmp/data-quality-error-data + + # Network IP gets priority, default inner outer + + # Whether hive SQL is executed in the same session + support.hive.oneSession: false + + # use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions + sudo.enable: true + + # network interface preferred like eth0, default: empty + #dolphin.scheduler.network.interface.preferred: + + # network IP gets priority, default: inner outer + #dolphin.scheduler.network.priority.strategy: default + + # system env path + #dolphinscheduler.env.path: dolphinscheduler_env.sh + # development state + development.state: false + # rpc port + alert.rpc.port: 50052 + # Url endpoint for zeppelin RESTful API + zeppelin.rest.url: http://localhost:8080 + common: ## Configmap configmap: