Browse Source

[1.3.5-prepare][Improvement][K8s] Support resource storage when fs.defaultFS=file:/// (#4700)

Shiwen Cheng 4 years ago committed by GitHub
parent
commit
263e94d177
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      docker/kubernetes/dolphinscheduler/README.md
  2. 9
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
  3. 37
      docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-resource.yaml
  4. 9
      docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
  5. 8
      docker/kubernetes/dolphinscheduler/values.yaml

4
docker/kubernetes/dolphinscheduler/README.md

@ -80,6 +80,10 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `resource.fsS3aEndpoint` | S3 endpoint when `resource.resourceStorageType` be seted to `S3` | `s3.xxx.amazonaws.com` |
| `resource.fsS3aAccessKey` | S3 access key when `resource.resourceStorageType` be seted to `S3` | `xxxxxxx` |
| `resource.fsS3aSecretKey` | S3 secret key when `resource.resourceStorageType` be seted to `S3` | `xxxxxxx` |
| `resource.persistentVolumeClaim.enabled` | Set `resource.persistentVolumeClaim.enabled` to `true` to mount a new resource volume for `api` and `worker` | `false` |
| `resource.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes, must be `ReadWriteMany` | `[ReadWriteMany]` |
| `resource.persistentVolumeClaim.storageClassName` | Resource Persistent Volume Storage Class, must support the access mode: ReadWriteMany | `-` |
| `resource.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | |
| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` |

9
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@ -166,6 +166,10 @@ spec:
volumeMounts:
- mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-api
{{- if .Values.resource.persistentVolumeClaim.enabled }}
- mountPath: {{ .Values.resource.resourceUploadPath }}
name: {{ include "dolphinscheduler.fullname" . }}-resource
{{- end }}
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-api
{{- if .Values.api.persistentVolumeClaim.enabled }}
@ -174,3 +178,8 @@ spec:
{{- else }}
emptyDir: {}
{{- end }}
{{- if .Values.resource.persistentVolumeClaim.enabled }}
- name: {{ include "dolphinscheduler.fullname" . }}-resource
persistentVolumeClaim:
claimName: {{ include "dolphinscheduler.fullname" . }}-resource
{{- end }}

37
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-resource.yaml

@ -0,0 +1,37 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if .Values.resource.persistentVolumeClaim.enabled }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-resource
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-resource
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
annotations:
"helm.sh/resource-policy": keep
spec:
accessModes:
{{- range .Values.resource.persistentVolumeClaim.accessModes }}
- {{ . | quote }}
{{- end }}
storageClassName: {{ .Values.resource.persistentVolumeClaim.storageClassName | quote }}
resources:
requests:
storage: {{ .Values.resource.persistentVolumeClaim.storage | quote }}
{{- end }}

9
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@ -211,6 +211,10 @@ spec:
- mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"
subPath: "dolphinscheduler_env.sh"
name: {{ include "dolphinscheduler.fullname" . }}-worker-configmap
{{- if .Values.resource.persistentVolumeClaim.enabled }}
- mountPath: {{ .Values.resource.resourceUploadPath }}
name: {{ include "dolphinscheduler.fullname" . }}-resource
{{- end }}
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-worker-data
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
@ -233,6 +237,11 @@ spec:
items:
- key: dolphinscheduler_env.sh
path: dolphinscheduler_env.sh
{{- if .Values.resource.persistentVolumeClaim.enabled }}
- name: {{ include "dolphinscheduler.fullname" . }}-resource
persistentVolumeClaim:
claimName: {{ include "dolphinscheduler.fullname" . }}-resource
{{- end }}
{{- if .Values.worker.persistentVolumeClaim.enabled }}
volumeClaimTemplates:
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}

8
docker/kubernetes/dolphinscheduler/values.yaml

@ -77,6 +77,14 @@ resource:
fsS3aEndpoint: "s3.xxx.amazonaws.com"
fsS3aAccessKey: "xxxxxxx"
fsS3aSecretKey: "xxxxxxx"
# If resourceStorageType is HDFS and fsDefaultFS is file:///, persistentVolumeClaim should be enabled for resource storage
persistentVolumeClaim:
enabled: false
accessModes:
- "ReadWriteMany"
# storageClassName must support the access mode: ReadWriteMany
storageClassName: "-"
storage: "20Gi"
master:
podManagementPolicy: "Parallel"

Loading…
Cancel
Save