Add minio as demo storage for helm charts (#12861)

deploy/kubernetes/dolphinscheduler/Chart.yaml

@@ -60,3 +60,7 @@ dependencies:
   version: 9.4.1
   repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
   condition: mysql.enabled
+- name: minio
+  version: 2022.10.29
+  repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
+  condition: minio.enabled

deploy/kubernetes/dolphinscheduler/resources/config/common.properties

@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-{{- if index .Values.conf "common" }}
-{{- range $key, $value := index .Values.conf "common" }}
-{{ $key }}={{ $value }}
-{{- end }}
-{{- end }}

deploy/kubernetes/dolphinscheduler/templates/NOTES.txt

@@ -1,19 +1,19 @@
-#
+{{/*
-# Licensed to the Apache Software Foundation (ASF) under one or more
+ Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
+ contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
+ this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
+ The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
+ (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
+ the License.  You may obtain a copy of the License at
-#
+
-#     http://www.apache.org/licenses/LICENSE-2.0
+     http://www.apache.org/licenses/LICENSE-2.0
-#
+
-# Unless required by applicable law or agreed to in writing, software
+ Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
+ distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
+ See the License for the specific language governing permissions and
-# limitations under the License.
+ limitations under the License.
-#
+*/}}
 
 ** Please be patient while the chart DolphinScheduler {{ .Chart.AppVersion }} is being deployed **
 

deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl

@@ -117,6 +117,15 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Create a default fully qualified minio name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "dolphinscheduler.minio.fullname" -}}
+{{- $name := default "minio" .Values.minio.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{/*
 Create a default fully qualified zookkeeper quorum.
 */}}
@@ -184,6 +193,18 @@ Wait for database to be ready.
 {{- end }}
 {{- end -}}
 
+{{/*
+Wait for minio to be ready.
+*/}}
+{{- define "dolphinscheduler.minio.wait-for-ready" -}}
+{{- if .Values.minio.enabled }}
+- name: wait-for-minio
+  image: busybox:1.30
+  imagePullPolicy: IfNotPresent
+  command: ['sh', '-xc', 'for i in $(seq 1 180); do nc -z -w3 {{ template "dolphinscheduler.minio.fullname" . }} 9000 && exit 0 || sleep 5; done; exit 1']
+{{- end }}
+{{- end -}}
+
 {{/*
 Create a registry environment variables.
 */}}

deploy/kubernetes/dolphinscheduler/templates/configmap.yaml

@@ -23,4 +23,12 @@ metadata:
     {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
 data:
   common_properties: |-
-{{ tpl (.Files.Get "resources/config/common.properties") . | indent 4 }}
+  {{- if index .Values.conf "common" }}
+  {{- range $key, $value := index .Values.conf "common" }}
+    {{- if and $.Values.minio.enabled }}
+    {{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }}
+    {{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
+    {{- end }}
+    {{ $key }}={{ $value }}
+  {{- end }}
+  {{- end }}

deploy/kubernetes/dolphinscheduler/values.yaml

@@ -57,6 +57,15 @@ mysql:
       size: "20Gi"
       storageClass: "-"
 
+minio:
+  enabled: false
+  auth:
+    rootUser: minioadmin
+    rootPassword: minioadmin
+  persistence:
+    enabled: false
+  defaultBuckets: "dolphinscheduler"
+
 ## If exists external database, and set postgresql.enable value to false.
 ## external database will be used, otherwise Dolphinscheduler's database will be used.
 externalDatabase:
@@ -91,7 +100,7 @@ conf:
     data.basedir.path: /tmp/dolphinscheduler
 
     # resource storage type: HDFS, S3, NONE
-    resource.storage.type: HDFS
+    resource.storage.type: NONE
 
     # resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
     resource.storage.upload.base.path: /dolphinscheduler
@@ -103,13 +112,13 @@ conf:
     resource.aws.secret.access.key: minioadmin
 
     # The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
-    resource.aws.region: cn-north-1
+    resource.aws.region: ca-central-1
 
     # The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
     resource.aws.s3.bucket.name: dolphinscheduler
 
     # You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
-    resource.aws.s3.endpoint: http://localhost:9000
+    resource.aws.s3.endpoint: http://minio:9000
 
     # alibaba cloud access key id, required if you set resource.storage.type=OSS
     resource.alibaba.cloud.access.key.id: <your-access-key-id>

docs/docs/en/guide/installation/kubernetes.md

@@ -495,6 +495,7 @@ common:
 | `postgresql.persistence.enabled`                                     | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL                                  | `false`                               |
 | `postgresql.persistence.size`                                        | `PersistentVolumeClaim` size                                                                                                  | `20Gi`                                |
 | `postgresql.persistence.storageClass`                                | PostgreSQL data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning     | `-`                                   |
+| `minio.enabled`                                                      | Deploy minio and configure it as the default storage for DolphinScheduler, note this is for demo only, not for production.    | `false`                               |
 | `externalDatabase.type`                                              | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database type will use it      | `postgresql`                          |
 | `externalDatabase.driver`                                            | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database driver will use it    | `org.postgresql.Driver`               |
 | `externalDatabase.host`                                              | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database host will use it      | `localhost`                           |