From 366f999167feb5da65b86b48a010b754572857ef Mon Sep 17 00:00:00 2001 From: Eric Gao Date: Sat, 14 Jan 2023 13:51:15 +0800 Subject: [PATCH] [Feature][Deployment] Add KEDA autoscaler support for worker deployment when deployed in K8S cluster (#13367) * Add keda autoscaler support for worker deployment when deployed in K8S cluster * Add mysql scaler for worker autoscaling * Add docs for worker autoscaler * Add support for external postgresql db --- .../templates/keda-autoscaler-worker.yaml | 102 ++++++++++++++++++ .../kubernetes/dolphinscheduler/values.yaml | 33 +++++- docs/docs/en/guide/installation/kubernetes.md | 37 +++++++ docs/docs/zh/guide/installation/kubernetes.md | 36 +++++++ 4 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 deploy/kubernetes/dolphinscheduler/templates/keda-autoscaler-worker.yaml diff --git a/deploy/kubernetes/dolphinscheduler/templates/keda-autoscaler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/keda-autoscaler-worker.yaml new file mode 100644 index 0000000000..3251e0e04a --- /dev/null +++ b/deploy/kubernetes/dolphinscheduler/templates/keda-autoscaler-worker.yaml @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## DolphinScheduler Worker KEDA Scaler +################################# +{{- if and .Values.worker.keda.enabled }} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-worker + labels: + component: worker-horizontalpodautoscaler + deploymentName: {{ include "dolphinscheduler.fullname" . }}-worker +spec: + scaleTargetRef: + kind: StatefulSet + name: {{ include "dolphinscheduler.fullname" . }}-worker + pollingInterval: {{ .Values.worker.keda.pollingInterval }} + cooldownPeriod: {{ .Values.worker.keda.cooldownPeriod }} + minReplicaCount: {{ .Values.worker.keda.minReplicaCount }} + maxReplicaCount: {{ .Values.worker.keda.maxReplicaCount }} + {{- if .Values.worker.keda.advanced }} + advanced: + {{ toYaml .Values.worker.keda.advanced | indent 4 }} + {{- end }} + # This is just an example, you could customize the trigger rule. + # FYI, check TaskExecutionStatus.java for the human-readable meaning of state values below. + triggers: + {{- if .Values.postgresql.enabled }} + - type: postgresql + metadata: + host: {{ template "dolphinscheduler.postgresql.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local + port: "5432" + dbName: {{ .Values.postgresql.postgresqlDatabase }} + userName: {{ .Values.postgresql.postgresqlUsername }} + passwordFromEnv: SPRING_DATASOURCE_PASSWORD + sslmode: "disable" + targetQueryValue: "1" + query: >- + SELECT ceil(COUNT(*)::decimal / {{ .Values.worker.env.WORKER_EXEC_THREADS }}) + FROM t_ds_task_instance + WHERE state IN (0, 1, 8, 12, 17) + {{- else if .Values.mysql.enabled }} + - type: mysql + metadata: + host: {{ template "dolphinscheduler.mysql.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local + port: "3306" + dbName: {{ .Values.mysql.auth.database }} + username: {{ .Values.mysql.auth.username }} + passwordFromEnv: SPRING_DATASOURCE_PASSWORD + queryValue: "1" + query: >- + SELECT CEIL(COUNT(*) / {{ .Values.worker.env.WORKER_EXEC_THREADS }}) + FROM t_ds_task_instance + WHERE state IN (0, 1, 8, 12, 17) + {{- else if .Values.externalDatabase.enabled }} + {{- if eq .Values.externalDatabase.type "mysql" }} + - type: mysql + metadata: + host: {{ .Values.externalDatabase.host }} + # mysql scaler requests port in string format + port: "{{ .Values.externalDatabase.port }}" + dbName: {{ .Values.externalDatabase.database }} + username: {{ .Values.externalDatabase.username }} + passwordFromEnv: SPRING_DATASOURCE_PASSWORD + queryValue: "1" + query: >- + SELECT CEIL(COUNT(*) / {{ .Values.worker.env.WORKER_EXEC_THREADS }}) + FROM t_ds_task_instance + WHERE state IN (0, 1, 8, 12, 17) + {{- else if eq .Values.externalDatabase.type "postgresql" }} + - type: postgresql + metadata: + host: {{ .Values.externalDatabase.host }} + port: "{{ .Values.externalDatabase.port }}" + dbName: {{ .Values.externalDatabase.database }} + userName: {{ .Values.externalDatabase.username }} + passwordFromEnv: SPRING_DATASOURCE_PASSWORD + sslmode: "disable" + targetQueryValue: "1" + query: >- + SELECT ceil(COUNT(*)::decimal / {{ .Values.worker.env.WORKER_EXEC_THREADS }}) + FROM t_ds_task_instance + WHERE state IN (0, 1, 8, 12, 17) + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index c5820cf898..4b212f90c0 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -58,7 +58,7 @@ mysql: storageClass: "-" minio: - enabled: false + enabled: true auth: rootUser: minioadmin rootPassword: minioadmin @@ -69,6 +69,7 @@ minio: ## If exists external database, and set postgresql.enable value to false. ## external database will be used, otherwise Dolphinscheduler's database will be used. externalDatabase: + enabled: false type: "postgresql" host: "localhost" port: "5432" @@ -100,7 +101,7 @@ conf: data.basedir.path: /tmp/dolphinscheduler # resource storage type: HDFS, S3, NONE - resource.storage.type: NONE + resource.storage.type: S3 # resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended resource.storage.upload.base.path: /dolphinscheduler @@ -385,6 +386,34 @@ worker: WORKER_HEART_ERROR_THRESHOLD: "5" WORKER_HOST_WEIGHT: "100" WORKER_GROUPS: "default" + keda: + enabled: false + namespaceLabels: { } + + # How often KEDA polls the DolphinScheduler DB to report new scale requests to the HPA + pollingInterval: 5 + + # How many seconds KEDA will wait before scaling to zero. + # Note that HPA has a separate cooldown period for scale-downs + cooldownPeriod: 30 + + # Minimum number of workers created by keda + minReplicaCount: 0 + + # Maximum number of workers created by keda + maxReplicaCount: 3 + + # Specify HPA related options + advanced: { } + # horizontalPodAutoscalerConfig: + # behavior: + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Percent + # value: 100 + # periodSeconds: 15 + alert: ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. diff --git a/docs/docs/en/guide/installation/kubernetes.md b/docs/docs/en/guide/installation/kubernetes.md index a2f832caa1..be32e63c8e 100644 --- a/docs/docs/en/guide/installation/kubernetes.md +++ b/docs/docs/en/guide/installation/kubernetes.md @@ -87,6 +87,43 @@ $ kubectl delete pvc -l app.kubernetes.io/instance=dolphinscheduler > **Note**: Deleting the PVC's will delete all data as well. Please be cautious before doing it. +## [Experimental] Worker Autoscaling + +> **Warning**: Currently this is an experimental feature and may not be suitable for production! + +`DolphinScheduler` uses [KEDA](https://github.com/kedacore/keda) for worker autoscaling. However, `DolphinScheduler` disables +this feature by default. To turn on worker autoscaling: + +Firstly, you need to create a namespace for `KEDA` and install it with `helm`: + +```bash +helm repo add kedacore https://kedacore.github.io/charts + +helm repo update + +kubectl create namespace keda + +helm install keda kedacore/keda \ + --namespace keda \ + --version "v2.0.0" +``` + +Secondly, you need to set `worker.keda.enabled` to `true` in `values.yaml` or install the chart by: + +```bash +helm install dolphinscheduler . --set worker.keda.enabled=true -n +``` + +Once autoscaling enabled, the number of workers will scale between `minReplicaCount` and `maxReplicaCount` based on the states +of your tasks. For example, when there is no tasks running in your `DolphinScheduler` instance, there will be no workers, +which will significantly save the resources. + +Worker autoscaling feature is compatible with `postgresql` and `mysql` shipped with `DolphinScheduler official helm chart`. If you +use external database, worker autoscaling feature only supports external `mysql` and `postgresql` databases. + +If you need to change the value of worker `WORKER_EXEC_THREADS` when using autoscaling feature, +please change `worker.env.WORKER_EXEC_THREADS` in `values.yaml` instead of through `configmap`. + ## Configuration The configuration file is `values.yaml`, and the [Appendix-Configuration](#appendix-configuration) tables lists the configurable parameters of the DolphinScheduler and their default values. diff --git a/docs/docs/zh/guide/installation/kubernetes.md b/docs/docs/zh/guide/installation/kubernetes.md index e8c00d61ca..8416f1e0ee 100644 --- a/docs/docs/zh/guide/installation/kubernetes.md +++ b/docs/docs/zh/guide/installation/kubernetes.md @@ -87,6 +87,42 @@ $ kubectl delete pvc -l app.kubernetes.io/instance=dolphinscheduler > **注意**: 删除 PVC 也会删除所有数据，请谨慎操作！ +## [试验性] worker 自动扩缩容 + +> **警告**: 目前此功能尚在试验阶段，不建议在生产环境使用！ + +`DolphinScheduler` 使用 [KEDA](https://github.com/kedacore/keda) 对 worker 进行自动扩缩容。但是 `DolphinScheduler` 默认是不启用该功能的。 +您需要做下列配置来启用该功能： + +首先您需要创建一个单独的命名空间并使用 `helm` 安装 `KEDA`： + +```bash +helm repo add kedacore https://kedacore.github.io/charts + +helm repo update + +kubectl create namespace keda + +helm install keda kedacore/keda \ + --namespace keda \ + --version "v2.0.0" +``` + +其次，您需要将 `values.yaml` 中的 `worker.keda.enabled` 配置设置成 `true`，或者您可以通过以下命令安装 chart： + +```bash +helm install dolphinscheduler . --set worker.keda.enabled=true -n +``` + +一旦自动扩缩容功能启用，worker的数量将基于任务状态在 `minReplicaCount` 和 `maxReplicaCount` 之间弹性扩缩。 +举例来说，当您的 `DolphinScheduler` 实例中没有任务在运行时，将不会有 worker。因此，这个功能会显著节约资源，降低您的使用成本。 + +自动扩缩容功能目前支持 `DolphinScheduler 官方 helm chart` 中自带的 `postgresql` and `mysql`。 +如果您要使用外部的数据库，自动扩缩容功能目前只支持 `mysql` 和 `postgresql` 类型的外部数据库。 + +如果您在使用自动扩缩容时需要改变 worker `WORKER_EXEC_THREADS` 的值，请直接在 `values.yaml` 中修改 `worker.env.WORKER_EXEC_THREADS` 的值， +而不要通过 `configmap` 来更新。 + ## 配置 配置文件为 `values.yaml`，[附录-配置](#appendix-configuration) 表格列出了 DolphinScheduler 的可配置参数及其默认值