From 454f1303a099736663eed62e327229a05573aacf Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Fri, 11 Nov 2022 10:26:57 +0800 Subject: [PATCH] Add minio as demo storage for helm charts (#12861) --- deploy/kubernetes/dolphinscheduler/Chart.yaml | 4 +++ .../resources/config/common.properties | 21 ------------ .../dolphinscheduler/templates/NOTES.txt | 34 +++++++++---------- .../dolphinscheduler/templates/_helpers.tpl | 21 ++++++++++++ .../dolphinscheduler/templates/configmap.yaml | 10 +++++- .../kubernetes/dolphinscheduler/values.yaml | 15 ++++++-- docs/docs/en/guide/installation/kubernetes.md | 1 + 7 files changed, 64 insertions(+), 42 deletions(-) delete mode 100644 deploy/kubernetes/dolphinscheduler/resources/config/common.properties diff --git a/deploy/kubernetes/dolphinscheduler/Chart.yaml b/deploy/kubernetes/dolphinscheduler/Chart.yaml index 94e5b312ec..2f93e51d81 100644 --- a/deploy/kubernetes/dolphinscheduler/Chart.yaml +++ b/deploy/kubernetes/dolphinscheduler/Chart.yaml @@ -60,3 +60,7 @@ dependencies: version: 9.4.1 repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami condition: mysql.enabled +- name: minio + version: 2022.10.29 + repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + condition: minio.enabled diff --git a/deploy/kubernetes/dolphinscheduler/resources/config/common.properties b/deploy/kubernetes/dolphinscheduler/resources/config/common.properties deleted file mode 100644 index a152640c0f..0000000000 --- a/deploy/kubernetes/dolphinscheduler/resources/config/common.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -{{- if index .Values.conf "common" }} -{{- range $key, $value := index .Values.conf "common" }} -{{ $key }}={{ $value }} -{{- end }} -{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt b/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt index ac15c07b53..ab42cf2e12 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt +++ b/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt @@ -1,19 +1,19 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} ** Please be patient while the chart DolphinScheduler {{ .Chart.AppVersion }} is being deployed ** @@ -47,4 +47,4 @@ Access DolphinScheduler UI URL by: DolphinScheduler UI URL: http://$SERVICE_IP:12345/dolphinscheduler -{{- end }} \ No newline at end of file +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl index 0f36875d56..dd23b8d184 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -117,6 +117,15 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create a default fully qualified minio name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.minio.fullname" -}} +{{- $name := default "minio" .Values.minio.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Create a default fully qualified zookkeeper quorum. */}} @@ -184,6 +193,18 @@ Wait for database to be ready. {{- end }} {{- end -}} +{{/* +Wait for minio to be ready. +*/}} +{{- define "dolphinscheduler.minio.wait-for-ready" -}} +{{- if .Values.minio.enabled }} +- name: wait-for-minio + image: busybox:1.30 + imagePullPolicy: IfNotPresent + command: ['sh', '-xc', 'for i in $(seq 1 180); do nc -z -w3 {{ template "dolphinscheduler.minio.fullname" . }} 9000 && exit 0 || sleep 5; done; exit 1'] +{{- end }} +{{- end -}} + {{/* Create a registry environment variables. */}} diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml index 57d5155c37..52c2730341 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml @@ -23,4 +23,12 @@ metadata: {{- include "dolphinscheduler.common.labels" . | nindent 4 }} data: common_properties: |- -{{ tpl (.Files.Get "resources/config/common.properties") . | indent 4 }} \ No newline at end of file + {{- if index .Values.conf "common" }} + {{- range $key, $value := index .Values.conf "common" }} + {{- if and $.Values.minio.enabled }} + {{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }} + {{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }} + {{- end }} + {{ $key }}={{ $value }} + {{- end }} + {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 46b015644d..555ceefef3 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -57,6 +57,15 @@ mysql: size: "20Gi" storageClass: "-" +minio: + enabled: false + auth: + rootUser: minioadmin + rootPassword: minioadmin + persistence: + enabled: false + defaultBuckets: "dolphinscheduler" + ## If exists external database, and set postgresql.enable value to false. ## external database will be used, otherwise Dolphinscheduler's database will be used. externalDatabase: @@ -91,7 +100,7 @@ conf: data.basedir.path: /tmp/dolphinscheduler # resource storage type: HDFS, S3, NONE - resource.storage.type: HDFS + resource.storage.type: NONE # resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended resource.storage.upload.base.path: /dolphinscheduler @@ -103,13 +112,13 @@ conf: resource.aws.secret.access.key: minioadmin # The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required - resource.aws.region: cn-north-1 + resource.aws.region: ca-central-1 # The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. resource.aws.s3.bucket.name: dolphinscheduler # You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn - resource.aws.s3.endpoint: http://localhost:9000 + resource.aws.s3.endpoint: http://minio:9000 # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id: diff --git a/docs/docs/en/guide/installation/kubernetes.md b/docs/docs/en/guide/installation/kubernetes.md index 8671e65a4e..cbe2233f6d 100644 --- a/docs/docs/en/guide/installation/kubernetes.md +++ b/docs/docs/en/guide/installation/kubernetes.md @@ -495,6 +495,7 @@ common: | `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` | | `postgresql.persistence.size` | `PersistentVolumeClaim` size | `20Gi` | | `postgresql.persistence.storageClass` | PostgreSQL data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `minio.enabled` | Deploy minio and configure it as the default storage for DolphinScheduler, note this is for demo only, not for production. | `false` | | `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database type will use it | `postgresql` | | `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database driver will use it | `org.postgresql.Driver` | | `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database host will use it | `localhost` |