Browse Source

Add minio as demo storage for helm charts (#12861)

3.2.0-release
kezhenxu94 2 years ago committed by GitHub
parent
commit
454f1303a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      deploy/kubernetes/dolphinscheduler/Chart.yaml
  2. 21
      deploy/kubernetes/dolphinscheduler/resources/config/common.properties
  3. 32
      deploy/kubernetes/dolphinscheduler/templates/NOTES.txt
  4. 21
      deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
  5. 10
      deploy/kubernetes/dolphinscheduler/templates/configmap.yaml
  6. 15
      deploy/kubernetes/dolphinscheduler/values.yaml
  7. 1
      docs/docs/en/guide/installation/kubernetes.md

4
deploy/kubernetes/dolphinscheduler/Chart.yaml

@ -60,3 +60,7 @@ dependencies:
version: 9.4.1
repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
condition: mysql.enabled
- name: minio
version: 2022.10.29
repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
condition: minio.enabled

21
deploy/kubernetes/dolphinscheduler/resources/config/common.properties

@ -1,21 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if index .Values.conf "common" }}
{{- range $key, $value := index .Values.conf "common" }}
{{ $key }}={{ $value }}
{{- end }}
{{- end }}

32
deploy/kubernetes/dolphinscheduler/templates/NOTES.txt

@ -1,19 +1,19 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
** Please be patient while the chart DolphinScheduler {{ .Chart.AppVersion }} is being deployed **

21
deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl

@ -117,6 +117,15 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified minio name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
*/}}
{{- define "dolphinscheduler.minio.fullname" -}}
{{- $name := default "minio" .Values.minio.nameOverride -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified zookkeeper quorum.
*/}}
@ -184,6 +193,18 @@ Wait for database to be ready.
{{- end }}
{{- end -}}
{{/*
Wait for minio to be ready.
*/}}
{{- define "dolphinscheduler.minio.wait-for-ready" -}}
{{- if .Values.minio.enabled }}
- name: wait-for-minio
image: busybox:1.30
imagePullPolicy: IfNotPresent
command: ['sh', '-xc', 'for i in $(seq 1 180); do nc -z -w3 {{ template "dolphinscheduler.minio.fullname" . }} 9000 && exit 0 || sleep 5; done; exit 1']
{{- end }}
{{- end -}}
{{/*
Create a registry environment variables.
*/}}

10
deploy/kubernetes/dolphinscheduler/templates/configmap.yaml

@ -23,4 +23,12 @@ metadata:
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
data:
common_properties: |-
{{ tpl (.Files.Get "resources/config/common.properties") . | indent 4 }}
{{- if index .Values.conf "common" }}
{{- range $key, $value := index .Values.conf "common" }}
{{- if and $.Values.minio.enabled }}
{{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }}
{{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
{{- end }}
{{ $key }}={{ $value }}
{{- end }}
{{- end }}

15
deploy/kubernetes/dolphinscheduler/values.yaml

@ -57,6 +57,15 @@ mysql:
size: "20Gi"
storageClass: "-"
minio:
enabled: false
auth:
rootUser: minioadmin
rootPassword: minioadmin
persistence:
enabled: false
defaultBuckets: "dolphinscheduler"
## If exists external database, and set postgresql.enable value to false.
## external database will be used, otherwise Dolphinscheduler's database will be used.
externalDatabase:
@ -91,7 +100,7 @@ conf:
data.basedir.path: /tmp/dolphinscheduler
# resource storage type: HDFS, S3, NONE
resource.storage.type: HDFS
resource.storage.type: NONE
# resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.storage.upload.base.path: /dolphinscheduler
@ -103,13 +112,13 @@ conf:
resource.aws.secret.access.key: minioadmin
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
resource.aws.region: cn-north-1
resource.aws.region: ca-central-1
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
resource.aws.s3.bucket.name: dolphinscheduler
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
resource.aws.s3.endpoint: http://localhost:9000
resource.aws.s3.endpoint: http://minio:9000
# alibaba cloud access key id, required if you set resource.storage.type=OSS
resource.alibaba.cloud.access.key.id: <your-access-key-id>

1
docs/docs/en/guide/installation/kubernetes.md

@ -495,6 +495,7 @@ common:
| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` |
| `postgresql.persistence.size` | `PersistentVolumeClaim` size | `20Gi` |
| `postgresql.persistence.storageClass` | PostgreSQL data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `minio.enabled` | Deploy minio and configure it as the default storage for DolphinScheduler, note this is for demo only, not for production. | `false` |
| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database type will use it | `postgresql` |
| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database driver will use it | `org.postgresql.Driver` |
| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database host will use it | `localhost` |

Loading…
Cancel
Save