From f166de1798fb593cb395d59dcbd207f47cd12473 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sun, 21 Feb 2021 22:44:55 +0800 Subject: [PATCH] [Improvement][K8s] Support resource storage when fs.defaultFS=file:/// (#4777) --- .../dolphinscheduler/templates/_helpers.tpl | 9 ----- .../configmap-dolphinscheduler-common.yaml | 2 +- .../deployment-dolphinscheduler-api.yaml | 11 +++++- .../pvc-dolphinscheduler-fs-file.yaml | 37 +++++++++++++++++++ .../statefulset-dolphinscheduler-master.yaml | 2 +- .../statefulset-dolphinscheduler-worker.yaml | 13 ++++++- .../kubernetes/dolphinscheduler/values.yaml | 8 ++++ 7 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl index ccfc746645..542bcb2f66 100644 --- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -129,13 +129,4 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- define "dolphinscheduler.zookeeper.quorum" -}} {{- $port := default "2181" (.Values.zookeeper.service.port | toString) -}} {{- printf "%s:%s" (include "dolphinscheduler.zookeeper.fullname" .) $port | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default dolphinscheduler worker base dir. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "dolphinscheduler.data.basedir.path" -}} -{{- $name := default "/tmp/dolphinscheduler" .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH -}} -{{- printf "%s" $name | trunc 63 | trimSuffix "/" -}} {{- end -}} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml index 1fcb5b5992..f350055a3d 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml @@ -28,7 +28,7 @@ data: {{- range .Values.common.configmap.DOLPHINSCHEDULER_ENV }} {{ . }} {{- end }} - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.data.basedir.path" . | quote }} + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }} RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }} RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 563f9ebbf9..f4b4795c28 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -206,6 +206,10 @@ spec: volumeMounts: - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-api + {{- if .Values.common.fsFileResourcePersistence.enabled }} + - mountPath: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} + name: {{ include "dolphinscheduler.fullname" . }}-fs-file + {{- end }} volumes: - name: {{ include "dolphinscheduler.fullname" . }}-api {{- if .Values.api.persistentVolumeClaim.enabled }} @@ -213,4 +217,9 @@ spec: claimName: {{ include "dolphinscheduler.fullname" . }}-api {{- else }} emptyDir: {} - {{- end }} \ No newline at end of file + {{- end }} + {{- if .Values.common.fsFileResourcePersistence.enabled }} + - name: {{ include "dolphinscheduler.fullname" . }}-fs-file + persistentVolumeClaim: + claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file + {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml new file mode 100644 index 0000000000..1e277eb060 --- /dev/null +++ b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.common.fsFileResourcePersistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-fs-file + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-file + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + "helm.sh/resource-policy": keep +spec: + accessModes: + {{- range .Values.common.fsFileResourcePersistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.common.fsFileResourcePersistence.storageClassName | quote }} + resources: + requests: + storage: {{ .Values.common.fsFileResourcePersistence.storage | quote }} +{{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index 85633a9f33..55d13d73ad 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -247,4 +247,4 @@ spec: resources: requests: storage: {{ .Values.master.persistentVolumeClaim.storage | quote }} - {{- end }} + {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index b8a62fa6ae..76527b79e2 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -324,13 +324,17 @@ spec: failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} {{- end }} volumeMounts: - - mountPath: {{ include "dolphinscheduler.data.basedir.path" . | quote }} + - mountPath: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }} name: {{ include "dolphinscheduler.fullname" . }}-worker-data - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-worker-logs - mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh" subPath: "dolphinscheduler_env.sh" name: {{ include "dolphinscheduler.fullname" . }}-common-env + {{- if .Values.common.fsFileResourcePersistence.enabled }} + - mountPath: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} + name: {{ include "dolphinscheduler.fullname" . }}-fs-file + {{- end }} volumes: - name: {{ include "dolphinscheduler.fullname" . }}-worker-data {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} @@ -353,6 +357,11 @@ spec: items: - key: DOLPHINSCHEDULER_ENV path: dolphinscheduler_env.sh + {{- if .Values.common.fsFileResourcePersistence.enabled }} + - name: {{ include "dolphinscheduler.fullname" . }}-fs-file + persistentVolumeClaim: + claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file + {{- end }} {{- if .Values.worker.persistentVolumeClaim.enabled }} volumeClaimTemplates: {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} @@ -389,4 +398,4 @@ spec: requests: storage: {{ .Values.worker.persistentVolumeClaim.logsPersistentVolume.storage | quote }} {{- end }} - {{- end }} + {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index 61201dfaa7..b99b553b4b 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -94,6 +94,14 @@ common: FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com" FS_S3A_ACCESS_KEY: "xxxxxxx" FS_S3A_SECRET_KEY: "xxxxxxx" + ## If RESOURCE_STORAGE_TYPE is HDFS and FS_DEFAULT_FS is file:///, fsFileResourcePersistence should be enabled for resource storage + fsFileResourcePersistence: + enabled: false + accessModes: + - "ReadWriteMany" + ## storageClassName must support the access mode: ReadWriteMany + storageClassName: "-" + storage: "20Gi" master: ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down.