From 93514ddf316475caf3c8498b48c31c807c29c6f1 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sun, 14 Feb 2021 21:35:23 +0800 Subject: [PATCH] [Improvement][K8s] Improve K8s for helm (#4746) --- docker/kubernetes/dolphinscheduler/Chart.yaml | 10 +- docker/kubernetes/dolphinscheduler/README.md | 37 +-- .../dolphinscheduler/requirements.yaml | 25 -- .../dolphinscheduler/requirements.yaml~HEAD | 25 -- .../dolphinscheduler/requirements.yaml~HEAD_0 | 25 -- .../dolphinscheduler/requirements.yaml~dev | 25 -- .../dolphinscheduler/requirements.yaml~dev_0 | 25 -- .../dolphinscheduler/templates/NOTES.txt | 6 +- .../dolphinscheduler/templates/_helpers.tpl | 4 +- .../configmap-dolphinscheduler-alert.yaml | 1 + ...ml => configmap-dolphinscheduler-api.yaml} | 21 +- .../configmap-dolphinscheduler-common.yaml | 8 +- .../configmap-dolphinscheduler-master.yaml | 2 +- .../configmap-dolphinscheduler-worker.yaml | 7 +- .../deployment-dolphinscheduler-alert.yaml | 68 +--- .../deployment-dolphinscheduler-api.yaml | 48 +-- .../deployment-dolphinscheduler-frontend.yaml | 119 ------- .../dolphinscheduler/templates/ingress.yaml | 2 +- .../templates/pvc-dolphinscheduler-alert.yaml | 2 +- .../templates/pvc-dolphinscheduler-api.yaml | 2 +- ...ntend.yaml => secret-external-fs-s3a.yaml} | 20 +- .../statefulset-dolphinscheduler-master.yaml | 96 +----- .../statefulset-dolphinscheduler-worker.yaml | 184 ++++------- .../svc-dolphinscheduler-worker-headless.yaml | 4 +- .../kubernetes/dolphinscheduler/values.yaml | 310 ++++++++---------- 25 files changed, 269 insertions(+), 807 deletions(-) delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD_0 delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml~dev delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml~dev_0 rename docker/kubernetes/dolphinscheduler/templates/{svc-dolphinscheduler-frontend.yaml => configmap-dolphinscheduler-api.yaml} (69%) delete mode 100644 docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml rename docker/kubernetes/dolphinscheduler/templates/{pvc-dolphinscheduler-frontend.yaml => secret-external-fs-s3a.yaml} (66%) diff --git a/docker/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml index 9d640869dd..1e0ca755b9 100644 --- a/docker/kubernetes/dolphinscheduler/Chart.yaml +++ b/docker/kubernetes/dolphinscheduler/Chart.yaml @@ -22,7 +22,7 @@ home: https://dolphinscheduler.apache.org icon: https://dolphinscheduler.apache.org/img/hlogo_colorful.svg keywords: - dolphinscheduler -- Scheduler +- scheduler # A chart can be either an 'application' or a 'library' chart. # # Application charts are a collection of templates that can be packaged into versioned archives @@ -35,18 +35,18 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 1.0.0 +version: 1.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 1.3.0 +appVersion: 1.4.0 dependencies: - name: postgresql - version: 8.x.x + version: 10.x.x repository: https://charts.bitnami.com/bitnami condition: postgresql.enabled - name: zookeeper - version: 5.x.x + version: 6.x.x repository: https://charts.bitnami.com/bitnami condition: zookeeper.enabled diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md index 0691b1ed2e..318c3a9132 100644 --- a/docker/kubernetes/dolphinscheduler/README.md +++ b/docker/kubernetes/dolphinscheduler/README.md @@ -7,19 +7,20 @@ This chart bootstraps a [Dolphin Scheduler](https://dolphinscheduler.apache.org) ## Prerequisites -- Kubernetes 1.10+ +- Helm 3.1.0+ +- Kubernetes 1.12+ - PV provisioner support in the underlying infrastructure ## Installing the Chart -To install the chart with the release name `my-release`: +To install the chart with the release name `dolphinscheduler`: ```bash $ git clone https://github.com/apache/incubator-dolphinscheduler.git -$ cd incubator-dolphinscheduler/kubernetes/dolphinscheduler +$ cd incubator-dolphinscheduler/docker/kubernetes/dolphinscheduler $ helm repo add bitnami https://charts.bitnami.com/bitnami $ helm dependency update . -$ helm install --name dolphinscheduler . +$ helm install dolphinscheduler . ``` These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. @@ -30,7 +31,7 @@ These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default To uninstall/delete the `dolphinscheduler` deployment: ```bash -$ helm delete --purge dolphinscheduler +$ helm uninstall dolphinscheduler ``` The command removes all the Kubernetes components associated with the chart and deletes the release. @@ -220,32 +221,6 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `api.persistentVolumeClaim.storageClassName` | `api` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `api.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | | | | | -| `frontend.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | -| `frontend.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | -| `frontend.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | -| `frontend.replicas` | Replicas is the desired number of replicas of the given Template | `1` | -| `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | -| `frontend.tolerations` | If specified, the pod's tolerations | `{}` | -| `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` | -| `frontend.resources` | The `resource` limit and request config for frontend server. | `{}` | -| `frontend.annotations` | The `annotations` for frontend server. | `{}` | -| `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` | -| `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | -| `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` | -| `frontend.livenessProbe.timeoutSeconds` | When the probe times out | `5` | -| `frontend.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `frontend.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `frontend.readinessProbe.enabled` | Turn on and off readiness probe | `true` | -| `frontend.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | -| `frontend.readinessProbe.periodSeconds` | How often to perform the probe | `30` | -| `frontend.readinessProbe.timeoutSeconds` | When the probe times out | `5` | -| `frontend.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `frontend.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `frontend.persistentVolumeClaim.enabled` | Set `frontend.persistentVolumeClaim.enabled` to `true` to mount a new volume for `frontend` | `false` | -| `frontend.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | -| `frontend.persistentVolumeClaim.storageClassName` | `frontend` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `frontend.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | | `ingress.enabled` | Enable ingress | `false` | | `ingress.host` | Ingress host | `dolphinscheduler.org` | | `ingress.path` | Ingress path | `/` | diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml b/docker/kubernetes/dolphinscheduler/requirements.yaml deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD b/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD_0 b/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD_0 deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD_0 +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml~dev b/docker/kubernetes/dolphinscheduler/requirements.yaml~dev deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml~dev +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml~dev_0 b/docker/kubernetes/dolphinscheduler/requirements.yaml~dev_0 deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml~dev_0 +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt index 256c53ca57..8afe766d27 100644 --- a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt +++ b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt @@ -15,9 +15,9 @@ # limitations under the License. # -** Please be patient while the chart is being deployed ** +** Please be patient while the chart Dolphinscheduler {{ .Chart.AppVersion }} is being deployed ** -1. Get the Dolphinscheduler URL by running: +Get the Dolphinscheduler URL by running: {{- if .Values.ingress.enabled }} @@ -26,6 +26,6 @@ {{- else }} - kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888 + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-api 12345:12345 {{- end }} diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl index 1f121dfea8..ccfc746645 100644 --- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -135,7 +135,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this Create a default dolphinscheduler worker base dir. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). */}} -{{- define "dolphinscheduler.worker.base.dir" -}} -{{- $name := default "/tmp/dolphinscheduler" .Values.worker.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH -}} +{{- define "dolphinscheduler.data.basedir.path" -}} +{{- $name := default "/tmp/dolphinscheduler" .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH -}} {{- printf "%s" $name | trunc 63 | trimSuffix "/" -}} {{- end -}} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml index 2c5b76c3f0..2c7dd67c57 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml @@ -24,6 +24,7 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: + DOLPHINSCHEDULER_OPTS: {{ .Values.alert.configmap.DOLPHINSCHEDULER_OPTS | quote }} ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }} XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }} MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml similarity index 69% rename from docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml index 60d0d6e7b5..3845f1f67c 100644 --- a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml @@ -14,22 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # +{{- if .Values.api.configmap }} apiVersion: v1 -kind: Service +kind: ConfigMap metadata: - name: {{ include "dolphinscheduler.fullname" . }}-frontend + name: {{ include "dolphinscheduler.fullname" . }}-api labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - ports: - - port: 8888 - targetPort: tcp-port - protocol: TCP - name: tcp-port - selector: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: frontend \ No newline at end of file +data: + DOLPHINSCHEDULER_OPTS: {{ .Values.api.configmap.DOLPHINSCHEDULER_OPTS | quote }} +{{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml index 931f6d48e4..1fcb5b5992 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml @@ -24,12 +24,14 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: - DOLPHINSCHEDULER_ENV_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_ENV_PATH | quote }} - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }} + DOLPHINSCHEDULER_ENV: |- + {{- range .Values.common.configmap.DOLPHINSCHEDULER_ENV }} + {{ . }} + {{- end }} + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.data.basedir.path" . | quote }} RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }} RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }} FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }} FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }} - FS_S3A_SECRET_KEY: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | quote }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml index da82d639cb..7c1be7717e 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml @@ -24,6 +24,7 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: + DOLPHINSCHEDULER_OPTS: {{ .Values.master.configmap.DOLPHINSCHEDULER_OPTS | quote }} MASTER_EXEC_THREADS: {{ .Values.master.configmap.MASTER_EXEC_THREADS | quote }} MASTER_EXEC_TASK_NUM: {{ .Values.master.configmap.MASTER_EXEC_TASK_NUM | quote }} MASTER_HEARTBEAT_INTERVAL: {{ .Values.master.configmap.MASTER_HEARTBEAT_INTERVAL | quote }} @@ -32,5 +33,4 @@ data: MASTER_MAX_CPULOAD_AVG: {{ .Values.master.configmap.MASTER_MAX_CPULOAD_AVG | quote }} MASTER_RESERVED_MEMORY: {{ .Values.master.configmap.MASTER_RESERVED_MEMORY | quote }} MASTER_LISTEN_PORT: {{ .Values.master.configmap.MASTER_LISTEN_PORT | quote }} - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml index 569341c225..78b6ed1036 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml @@ -24,17 +24,12 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: + DOLPHINSCHEDULER_OPTS: {{ .Values.worker.configmap.DOLPHINSCHEDULER_OPTS | quote }} WORKER_EXEC_THREADS: {{ .Values.worker.configmap.WORKER_EXEC_THREADS | quote }} WORKER_HEARTBEAT_INTERVAL: {{ .Values.worker.configmap.WORKER_HEARTBEAT_INTERVAL | quote }} - WORKER_FETCH_TASK_NUM: {{ .Values.worker.configmap.WORKER_FETCH_TASK_NUM | quote }} WORKER_MAX_CPULOAD_AVG: {{ .Values.worker.configmap.WORKER_MAX_CPULOAD_AVG | quote }} WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }} WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }} WORKER_GROUP: {{ .Values.worker.configmap.WORKER_GROUP | quote }} WORKER_WEIGHT: {{ .Values.worker.configmap.WORKER_WEIGHT | quote }} - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }} - dolphinscheduler_env.sh: |- - {{- range .Values.worker.configmap.DOLPHINSCHEDULER_ENV }} - {{ . }} - {{- end }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index fb9a23786b..e4a2b21ffa 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -57,35 +57,6 @@ spec: {{- if .Values.alert.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - initContainers: - - name: init-database - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to database." - exit 1 - fi - echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." - sleep 60 - done - env: - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} {{- if .Values.image.pullSecrets }} imagePullSecrets: - name: {{ include "dolphinscheduler.image.pullSecrets" . }} @@ -93,14 +64,17 @@ spec: containers: - name: {{ include "dolphinscheduler.fullname" . }}-alert image: {{ include "dolphinscheduler.image.repository" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "alert-server" - imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: DOLPHINSCHEDULER_OPTS - value: {{ default "" .Values.alert.jvmOptions }} - name: TZ value: {{ .Values.timezone }} + - name: DOLPHINSCHEDULER_OPTS + valueFrom: + configMapKeyRef: + key: DOLPHINSCHEDULER_OPTS + name: {{ include "dolphinscheduler.fullname" . }}-alert - name: ALERT_PLUGIN_DIR valueFrom: configMapKeyRef: @@ -228,36 +202,6 @@ spec: {{- else }} value: {{ .Values.externalDatabase.params | quote }} {{- end }} - - name: RESOURCE_STORAGE_TYPE - valueFrom: - configMapKeyRef: - key: RESOURCE_STORAGE_TYPE - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: RESOURCE_UPLOAD_PATH - valueFrom: - configMapKeyRef: - key: RESOURCE_UPLOAD_PATH - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_DEFAULT_FS - valueFrom: - configMapKeyRef: - key: FS_DEFAULT_FS - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_ENDPOINT - valueFrom: - configMapKeyRef: - key: FS_S3A_ENDPOINT - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_ACCESS_KEY - valueFrom: - configMapKeyRef: - key: FS_S3A_ACCESS_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_SECRET_KEY - valueFrom: - configMapKeyRef: - key: FS_S3A_SECRET_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common {{- if .Values.alert.resources }} resources: limits: diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 92c2c72398..563f9ebbf9 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -57,35 +57,6 @@ spec: {{- if .Values.api.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - initContainers: - - name: init-database - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to database." - exit 1 - fi - echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." - sleep 60 - done - env: - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} {{- if .Values.image.pullSecrets }} imagePullSecrets: - name: {{ include "dolphinscheduler.image.pullSecrets" . }} @@ -93,17 +64,20 @@ spec: containers: - name: {{ include "dolphinscheduler.fullname" . }}-api image: {{ include "dolphinscheduler.image.repository" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "api-server" ports: - containerPort: 12345 name: tcp-port - imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: DOLPHINSCHEDULER_OPTS - value: {{ default "" .Values.api.jvmOptions }} - name: TZ value: {{ .Values.timezone }} + - name: DOLPHINSCHEDULER_OPTS + valueFrom: + configMapKeyRef: + key: DOLPHINSCHEDULER_OPTS + name: {{ include "dolphinscheduler.fullname" . }}-api - name: DATABASE_TYPE {{- if .Values.postgresql.enabled }} value: "postgresql" @@ -164,7 +138,7 @@ spec: {{- end }} - name: ZOOKEEPER_ROOT {{- if .Values.zookeeper.enabled }} - value: "/dolphinscheduler" + value: {{ .Values.zookeeper.zookeeperRoot }} {{- else }} value: {{ .Values.externalZookeeper.zookeeperRoot }} {{- end }} @@ -183,6 +157,7 @@ spec: configMapKeyRef: key: FS_DEFAULT_FS name: {{ include "dolphinscheduler.fullname" . }}-common + {{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }} - name: FS_S3A_ENDPOINT valueFrom: configMapKeyRef: @@ -195,9 +170,10 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-common - name: FS_S3A_SECRET_KEY valueFrom: - configMapKeyRef: - key: FS_S3A_SECRET_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common + secretKeyRef: + key: fs-s3a-secret-key + name: {{ printf "%s-%s" .Release.Name "fs-s3a" }} + {{- end }} {{- if .Values.api.resources }} resources: limits: diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml deleted file mode 100644 index 04b9408d9f..0000000000 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "dolphinscheduler.fullname" . }}-frontend - labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: frontend -spec: - replicas: {{ .Values.frontend.replicas }} - selector: - matchLabels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: frontend - strategy: - type: {{ .Values.frontend.strategy.type | quote }} - rollingUpdate: - maxSurge: {{ .Values.frontend.strategy.rollingUpdate.maxSurge | quote }} - maxUnavailable: {{ .Values.frontend.strategy.rollingUpdate.maxUnavailable | quote }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: frontend - {{- if .Values.alert.annotations }} - annotations: - {{- toYaml .Values.alert.annotations | nindent 8 }} - {{- end }} - spec: - {{- if .Values.frontend.affinity }} - affinity: {{- toYaml .Values.frontend.affinity | nindent 8 }} - {{- end }} - {{- if .Values.frontend.nodeSelector }} - nodeSelector: {{- toYaml .Values.frontend.nodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.frontend.tolerations }} - tolerations: {{- toYaml . | nindent 8 }} - {{- end }} - {{- if .Values.image.pullSecrets }} - imagePullSecrets: - - name: {{ include "dolphinscheduler.image.pullSecrets" . }} - {{- end }} - containers: - - name: {{ include "dolphinscheduler.fullname" . }}-frontend - image: {{ include "dolphinscheduler.image.repository" . | quote }} - args: - - "frontend" - ports: - - containerPort: 8888 - name: tcp-port - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: TZ - value: {{ .Values.timezone }} - - name: FRONTEND_API_SERVER_HOST - value: '{{ include "dolphinscheduler.fullname" . }}-api' - - name: FRONTEND_API_SERVER_PORT - value: "12345" - {{- if .Values.frontend.resources }} - resources: - limits: - memory: {{ .Values.frontend.resources.limits.memory | quote }} - cpu: {{ .Values.frontend.resources.limits.cpu | quote }} - requests: - memory: {{ .Values.frontend.resources.requests.memory | quote }} - cpu: {{ .Values.frontend.resources.requests.cpu | quote }} - {{- end }} - {{- if .Values.frontend.livenessProbe.enabled }} - livenessProbe: - tcpSocket: - port: 8888 - initialDelaySeconds: {{ .Values.frontend.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.frontend.livenessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.frontend.livenessProbe.timeoutSeconds }} - successThreshold: {{ .Values.frontend.livenessProbe.successThreshold }} - failureThreshold: {{ .Values.frontend.livenessProbe.failureThreshold }} - {{- end }} - {{- if .Values.frontend.readinessProbe.enabled }} - readinessProbe: - tcpSocket: - port: 8888 - initialDelaySeconds: {{ .Values.frontend.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.frontend.readinessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.frontend.readinessProbe.timeoutSeconds }} - successThreshold: {{ .Values.frontend.readinessProbe.successThreshold }} - failureThreshold: {{ .Values.frontend.readinessProbe.failureThreshold }} - {{- end }} - volumeMounts: - - mountPath: "/var/log/nginx" - name: {{ include "dolphinscheduler.fullname" . }}-frontend - volumes: - - name: {{ include "dolphinscheduler.fullname" . }}-frontend - {{- if .Values.frontend.persistentVolumeClaim.enabled }} - persistentVolumeClaim: - claimName: {{ include "dolphinscheduler.fullname" . }}-frontend - {{- else }} - emptyDir: {} - {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml index d0f923dcf1..6edc82b12a 100644 --- a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml @@ -30,7 +30,7 @@ spec: paths: - path: {{ .Values.ingress.path }} backend: - serviceName: {{ include "dolphinscheduler.fullname" . }}-frontend + serviceName: {{ include "dolphinscheduler.fullname" . }}-api servicePort: tcp-port {{- if .Values.ingress.tls.enabled }} tls: diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml index 7f74cd94ae..fd5e88afb9 100644 --- a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml @@ -25,7 +25,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} spec: accessModes: - {{- range .Values.alert.persistentVolumeClaim.accessModes }} + {{- range .Values.alert.persistentVolumeClaim.accessModes }} - {{ . | quote }} {{- end }} storageClassName: {{ .Values.alert.persistentVolumeClaim.storageClassName | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml index c1074cc2b1..48292966ad 100644 --- a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml @@ -25,7 +25,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} spec: accessModes: - {{- range .Values.api.persistentVolumeClaim.accessModes }} + {{- range .Values.api.persistentVolumeClaim.accessModes }} - {{ . | quote }} {{- end }} storageClassName: {{ .Values.api.persistentVolumeClaim.storageClassName | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml similarity index 66% rename from docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml index ac9fe02a9e..78e7440392 100644 --- a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml @@ -14,22 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # -{{- if .Values.frontend.persistentVolumeClaim.enabled }} +{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }} apiVersion: v1 -kind: PersistentVolumeClaim +kind: Secret metadata: - name: {{ include "dolphinscheduler.fullname" . }}-frontend + name: {{ printf "%s-%s" .Release.Name "fs-s3a" }} labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - accessModes: - {{- range .Values.frontend.persistentVolumeClaim.accessModes }} - - {{ . | quote }} - {{- end }} - storageClassName: {{ .Values.frontend.persistentVolumeClaim.storageClassName | quote }} - resources: - requests: - storage: {{ .Values.frontend.persistentVolumeClaim.storage | quote }} +type: Opaque +data: + fs-s3a-secret-key: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | b64enc | quote }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index e9dc7919ca..85633a9f33 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -54,59 +54,6 @@ spec: {{- if .Values.master.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - initContainers: - - name: init-zookeeper - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do - while ! nc -z ${line%:*} ${line#*:}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to zookeeper." - exit 1 - fi - echo "Trying to connect to zookeeper at ${line}. Attempt $counter." - sleep 60 - done - done - env: - - name: ZOOKEEPER_QUORUM - {{- if .Values.zookeeper.enabled }} - value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" - {{- else }} - value: {{ .Values.externalZookeeper.zookeeperQuorum }} - {{- end }} - - name: init-database - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to database." - exit 1 - fi - echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." - sleep 60 - done - env: - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} {{- if .Values.image.pullSecrets }} imagePullSecrets: - name: {{ include "dolphinscheduler.image.pullSecrets" . }} @@ -114,17 +61,20 @@ spec: containers: - name: {{ include "dolphinscheduler.fullname" . }}-master image: {{ include "dolphinscheduler.image.repository" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "master-server" ports: - containerPort: {{ .Values.master.configmap.MASTER_LISTEN_PORT }} name: "master-port" - imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: DOLPHINSCHEDULER_OPTS - value: {{ default "" .Values.master.jvmOptions }} - name: TZ value: {{ .Values.timezone }} + - name: DOLPHINSCHEDULER_OPTS + valueFrom: + configMapKeyRef: + key: DOLPHINSCHEDULER_OPTS + name: {{ include "dolphinscheduler.fullname" . }}-master - name: MASTER_EXEC_THREADS valueFrom: configMapKeyRef: @@ -168,7 +118,7 @@ spec: - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH valueFrom: configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master + name: {{ include "dolphinscheduler.fullname" . }}-common key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - name: DATABASE_TYPE {{- if .Values.postgresql.enabled }} @@ -230,40 +180,10 @@ spec: {{- end }} - name: ZOOKEEPER_ROOT {{- if .Values.zookeeper.enabled }} - value: "/dolphinscheduler" + value: {{ .Values.zookeeper.zookeeperRoot }} {{- else }} value: {{ .Values.externalZookeeper.zookeeperRoot }} {{- end }} - - name: RESOURCE_STORAGE_TYPE - valueFrom: - configMapKeyRef: - key: RESOURCE_STORAGE_TYPE - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: RESOURCE_UPLOAD_PATH - valueFrom: - configMapKeyRef: - key: RESOURCE_UPLOAD_PATH - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_DEFAULT_FS - valueFrom: - configMapKeyRef: - key: FS_DEFAULT_FS - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_ENDPOINT - valueFrom: - configMapKeyRef: - key: FS_S3A_ENDPOINT - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_ACCESS_KEY - valueFrom: - configMapKeyRef: - key: FS_S3A_ACCESS_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_SECRET_KEY - valueFrom: - configMapKeyRef: - key: FS_S3A_SECRET_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common {{- if .Values.master.resources }} resources: limits: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index ae562cc62b..b8a62fa6ae 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -54,59 +54,6 @@ spec: {{- if .Values.worker.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - initContainers: - - name: init-zookeeper - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do - while ! nc -z ${line%:*} ${line#*:}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to zookeeper." - exit 1 - fi - echo "Trying to connect to zookeeper at ${line}. Attempt $counter." - sleep 60 - done - done - env: - - name: ZOOKEEPER_QUORUM - {{- if .Values.zookeeper.enabled }} - value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" - {{- else }} - value: {{ .Values.externalZookeeper.zookeeperQuorum }} - {{- end }} - - name: init-database - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to database." - exit 1 - fi - echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." - sleep 60 - done - env: - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} {{- if .Values.image.pullSecrets }} imagePullSecrets: - name: {{ include "dolphinscheduler.image.pullSecrets" . }} @@ -114,29 +61,27 @@ spec: containers: - name: {{ include "dolphinscheduler.fullname" . }}-worker image: {{ include "dolphinscheduler.image.repository" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "worker-server" ports: - containerPort: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }} name: "worker-port" - containerPort: 50051 - name: "logs-port" - imagePullPolicy: {{ .Values.image.pullPolicy }} + name: "logger-port" env: - - name: DOLPHINSCHEDULER_OPTS - value: {{ default "" .Values.worker.jvmOptions }} - name: TZ value: {{ .Values.timezone }} - - name: WORKER_EXEC_THREADS + - name: DOLPHINSCHEDULER_OPTS valueFrom: configMapKeyRef: + key: DOLPHINSCHEDULER_OPTS name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_EXEC_THREADS - - name: WORKER_FETCH_TASK_NUM + - name: WORKER_EXEC_THREADS valueFrom: configMapKeyRef: name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_FETCH_TASK_NUM + key: WORKER_EXEC_THREADS - name: WORKER_HEARTBEAT_INTERVAL valueFrom: configMapKeyRef: @@ -170,8 +115,58 @@ spec: - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH valueFrom: configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master + name: {{ include "dolphinscheduler.fullname" . }}-common key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH + - name: ALERT_PLUGIN_DIR + valueFrom: + configMapKeyRef: + key: ALERT_PLUGIN_DIR + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: XLS_FILE_PATH + valueFrom: + configMapKeyRef: + key: XLS_FILE_PATH + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_HOST + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_HOST + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_PORT + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_PORT + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SENDER + valueFrom: + configMapKeyRef: + key: MAIL_SENDER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_USER + valueFrom: + configMapKeyRef: + key: MAIL_USER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_PASSWD + valueFrom: + configMapKeyRef: + key: MAIL_PASSWD + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_STARTTLS_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_STARTTLS_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_TRUST + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_TRUST + name: {{ include "dolphinscheduler.fullname" . }}-alert - name: DATABASE_TYPE {{- if .Values.postgresql.enabled }} value: "postgresql" @@ -232,7 +227,7 @@ spec: {{- end }} - name: ZOOKEEPER_ROOT {{- if .Values.zookeeper.enabled }} - value: "/dolphinscheduler" + value: {{ .Values.zookeeper.zookeeperRoot }} {{- else }} value: {{ .Values.externalZookeeper.zookeeperRoot }} {{- end }} @@ -251,6 +246,7 @@ spec: configMapKeyRef: key: FS_DEFAULT_FS name: {{ include "dolphinscheduler.fullname" . }}-common + {{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }} - name: FS_S3A_ENDPOINT valueFrom: configMapKeyRef: @@ -263,54 +259,10 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-common - name: FS_S3A_SECRET_KEY valueFrom: - configMapKeyRef: - key: FS_S3A_SECRET_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: XLS_FILE_PATH - valueFrom: - configMapKeyRef: - key: XLS_FILE_PATH - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SERVER_HOST - valueFrom: - configMapKeyRef: - key: MAIL_SERVER_HOST - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SERVER_PORT - valueFrom: - configMapKeyRef: - key: MAIL_SERVER_PORT - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SENDER - valueFrom: - configMapKeyRef: - key: MAIL_SENDER - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_USER - valueFrom: - configMapKeyRef: - key: MAIL_USER - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_PASSWD - valueFrom: - configMapKeyRef: - key: MAIL_PASSWD - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SMTP_STARTTLS_ENABLE - valueFrom: - configMapKeyRef: - key: MAIL_SMTP_STARTTLS_ENABLE - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SMTP_SSL_ENABLE - valueFrom: - configMapKeyRef: - key: MAIL_SMTP_SSL_ENABLE - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SMTP_SSL_TRUST - valueFrom: - configMapKeyRef: - key: MAIL_SMTP_SSL_TRUST - name: {{ include "dolphinscheduler.fullname" . }}-alert + secretKeyRef: + key: fs-s3a-secret-key + name: {{ printf "%s-%s" .Release.Name "fs-s3a" }} + {{- end }} - name: ENTERPRISE_WECHAT_ENABLE valueFrom: configMapKeyRef: @@ -372,13 +324,13 @@ spec: failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} {{- end }} volumeMounts: - - mountPath: {{ include "dolphinscheduler.worker.base.dir" . | quote }} + - mountPath: {{ include "dolphinscheduler.data.basedir.path" . | quote }} name: {{ include "dolphinscheduler.fullname" . }}-worker-data - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-worker-logs - mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh" subPath: "dolphinscheduler_env.sh" - name: {{ include "dolphinscheduler.fullname" . }}-worker-configmap + name: {{ include "dolphinscheduler.fullname" . }}-common-env volumes: - name: {{ include "dolphinscheduler.fullname" . }}-worker-data {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} @@ -394,12 +346,12 @@ spec: {{- else }} emptyDir: {} {{- end }} - - name: {{ include "dolphinscheduler.fullname" . }}-worker-configmap + - name: {{ include "dolphinscheduler.fullname" . }}-common-env configMap: defaultMode: 0777 - name: {{ include "dolphinscheduler.fullname" . }}-worker + name: {{ include "dolphinscheduler.fullname" . }}-common items: - - key: dolphinscheduler_env.sh + - key: DOLPHINSCHEDULER_ENV path: dolphinscheduler_env.sh {{- if .Values.worker.persistentVolumeClaim.enabled }} volumeClaimTemplates: diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml index fb3b85b5c3..5c4e75436f 100644 --- a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml @@ -30,9 +30,9 @@ spec: protocol: TCP name: worker-port - port: 50051 - targetPort: logs-port + targetPort: logger-port protocol: TCP - name: logs-port + name: logger-port selector: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index 3261b08401..61201dfaa7 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -58,62 +58,74 @@ externalDatabase: # If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it. zookeeper: enabled: true - taskQueue: "zookeeper" - config: null + fourlwCommandsWhitelist: srvr,ruok,wchs,cons service: port: "2181" persistence: enabled: false size: "20Gi" storageClass: "-" + zookeeperRoot: "/dolphinscheduler" # If exists external zookeeper, and set zookeeper.enable value to false. -# If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it. +# If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it. externalZookeeper: - taskQueue: "zookeeper" zookeeperQuorum: "127.0.0.1:2181" zookeeperRoot: "/dolphinscheduler" common: + ## ConfigMap configmap: - DOLPHINSCHEDULER_ENV_PATH: "/tmp/dolphinscheduler/env" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler/files" - RESOURCE_STORAGE_TYPE: "NONE" - RESOURCE_UPLOAD_PATH: "/ds" - FS_DEFAULT_FS: "s3a://xxxx" + DOLPHINSCHEDULER_ENV: + - "export HADOOP_HOME=/opt/soft/hadoop" + - "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop" + - "export SPARK_HOME1=/opt/soft/spark1" + - "export SPARK_HOME2=/opt/soft/spark2" + - "export PYTHON_HOME=/usr/bin/python" + - "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk" + - "export HIVE_HOME=/opt/soft/hive" + - "export FLINK_HOME=/opt/soft/flink" + - "export DATAX_HOME=/opt/soft/datax/bin/datax.py" + - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" + RESOURCE_STORAGE_TYPE: "HDFS" + RESOURCE_UPLOAD_PATH: "/dolphinscheduler" + FS_DEFAULT_FS: "file:///" FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com" FS_S3A_ACCESS_KEY: "xxxxxxx" FS_S3A_SECRET_KEY: "xxxxxxx" master: + ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. podManagementPolicy: "Parallel" + ## Replicas is the desired number of replicas of the given Template. replicas: "3" - # NodeSelector is a selector which must be true for the pod to fit on a node. - # Selector which must match a node's labels for the pod to be scheduled on that node. - # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## You can use annotations to attach arbitrary non-identifying metadata to objects. + ## Clients such as tools and libraries can retrieve this metadata. + annotations: {} + ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. + ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## NodeSelector is a selector which must be true for the pod to fit on a node. + ## Selector which must match a node's labels for the pod to be scheduled on that node. + ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} - # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, - # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] - # Affinity is a group of affinity scheduling rules. - # If specified, the pod's scheduling constraints. - # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core - affinity: {} - # The jvm options for java instance startup - jvmOptions: "" + ## Compute Resources required by this container. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container resources: {} - # limits: - # memory: "18Gi" - # cpu: "4" - # requests: - # memory: "2Gi" - # cpu: "500m" - # You can use annotations to attach arbitrary non-identifying metadata to objects. - # Clients such as tools and libraries can retrieve this metadata. - annotations: {} - ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. - ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + # resources: + # limits: + # memory: "8Gi" + # cpu: "4" + # requests: + # memory: "2Gi" + # cpu: "500m" + ## ConfigMap configmap: + DOLPHINSCHEDULER_OPTS: "" MASTER_EXEC_THREADS: "100" MASTER_EXEC_TASK_NUM: "20" MASTER_HEARTBEAT_INTERVAL: "10" @@ -122,6 +134,8 @@ master: MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" MASTER_LISTEN_PORT: "5678" + ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: enabled: true initialDelaySeconds: "30" @@ -138,7 +152,7 @@ master: timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" - ## volumeClaimTemplates is a list of claims that pods are allowed to reference. + ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. ## A claim in this list takes precedence over any volumes in the template, with the same name. @@ -150,31 +164,43 @@ master: storage: "20Gi" worker: + ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. podManagementPolicy: "Parallel" + ## Replicas is the desired number of replicas of the given Template. replicas: "3" - # NodeSelector is a selector which must be true for the pod to fit on a node. - # Selector which must match a node's labels for the pod to be scheduled on that node. - # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## You can use annotations to attach arbitrary non-identifying metadata to objects. + ## Clients such as tools and libraries can retrieve this metadata. + annotations: {} + ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. + ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## NodeSelector is a selector which must be true for the pod to fit on a node. + ## Selector which must match a node's labels for the pod to be scheduled on that node. + ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} - # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, - # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] - # Affinity is a group of affinity scheduling rules. - # If specified, the pod's scheduling constraints. - # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core - affinity: {} - # The jvm options for java instance startup - jvmOptions: "" + ## Compute Resources required by this container. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container resources: {} - # limits: - # memory: "18Gi" - # cpu: "4" - # requests: - # memory: "2Gi" - # cpu: "500m" - # You can use annotations to attach arbitrary non-identifying metadata to objects. - # Clients such as tools and libraries can retrieve this metadata. - annotations: {} + # resources: + # limits: + # memory: "8Gi" + # cpu: "4" + # requests: + # memory: "2Gi" + # cpu: "500m" + ## ConfigMap + configmap: + DOLPHINSCHEDULER_OPTS: "" + WORKER_EXEC_THREADS: "100" + WORKER_HEARTBEAT_INTERVAL: "10" + WORKER_MAX_CPULOAD_AVG: "100" + WORKER_RESERVED_MEMORY: "0.1" + WORKER_LISTEN_PORT: "1234" + WORKER_GROUP: "default" + WORKER_WEIGHT: "100" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -193,27 +219,7 @@ worker: timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" - configmap: - WORKER_EXEC_THREADS: "100" - WORKER_HEARTBEAT_INTERVAL: "10" - WORKER_FETCH_TASK_NUM: "3" - WORKER_MAX_CPULOAD_AVG: "100" - WORKER_RESERVED_MEMORY: "0.1" - WORKER_LISTEN_PORT: "1234" - WORKER_GROUP: "default" - WORKER_WEIGHT: "100" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" - DOLPHINSCHEDULER_ENV: - - "export HADOOP_HOME=/opt/soft/hadoop" - - "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop" - - "export SPARK_HOME1=/opt/soft/spark1" - - "export SPARK_HOME2=/opt/soft/spark2" - - "export PYTHON_HOME=/opt/soft/python" - - "export JAVA_HOME=/opt/soft/java" - - "export HIVE_HOME=/opt/soft/hive" - - "export FLINK_HOME=/opt/soft/flink" - - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH" - ## volumeClaimTemplates is a list of claims that pods are allowed to reference. + ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. ## A claim in this list takes precedence over any volumes in the template, with the same name. @@ -235,38 +241,40 @@ worker: storage: "20Gi" alert: + ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. + replicas: "1" + ## The deployment strategy to use to replace existing pods with new ones. strategy: type: "RollingUpdate" rollingUpdate: maxSurge: "25%" maxUnavailable: "25%" - replicas: "1" - # NodeSelector is a selector which must be true for the pod to fit on a node. - # Selector which must match a node's labels for the pod to be scheduled on that node. - # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## You can use annotations to attach arbitrary non-identifying metadata to objects. + ## Clients such as tools and libraries can retrieve this metadata. + annotations: {} + ## NodeSelector is a selector which must be true for the pod to fit on a node. + ## Selector which must match a node's labels for the pod to be scheduled on that node. + ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + affinity: {} + ## Compute Resources required by this container. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container nodeSelector: {} - # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, - # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] - # Affinity is a group of affinity scheduling rules. - # If specified, the pod's scheduling constraints. - # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core - affinity: {} - # The jvm options for java instance startup - jvmOptions: "" + ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. + ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core resources: {} - # limits: - # memory: "4Gi" - # cpu: "1" - # requests: - # memory: "2Gi" - # cpu: "500m" - # You can use annotations to attach arbitrary non-identifying metadata to objects. - # Clients such as tools and libraries can retrieve this metadata. - annotations: {} - ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. - ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + # resources: + # limits: + # memory: "2Gi" + # cpu: "1" + # requests: + # memory: "1Gi" + # cpu: "500m" + ## ConfigMap configmap: + DOLPHINSCHEDULER_OPTS: "" ALERT_PLUGIN_DIR: "/opt/dolphinscheduler/alert/plugin" XLS_FILE_PATH: "/tmp/xls" MAIL_SERVER_HOST: "" @@ -282,63 +290,6 @@ alert: ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_USERS: "" - livenessProbe: - enabled: true - initialDelaySeconds: "30" - periodSeconds: "30" - timeoutSeconds: "5" - failureThreshold: "3" - successThreshold: "1" - ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. - ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - readinessProbe: - enabled: true - initialDelaySeconds: "30" - periodSeconds: "30" - timeoutSeconds: "5" - failureThreshold: "3" - successThreshold: "1" - ## volumeClaimTemplates is a list of claims that pods are allowed to reference. - ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. - ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. - ## A claim in this list takes precedence over any volumes in the template, with the same name. - persistentVolumeClaim: - enabled: false - accessModes: - - "ReadWriteOnce" - storageClassName: "-" - storage: "20Gi" - -api: - strategy: - type: "RollingUpdate" - rollingUpdate: - maxSurge: "25%" - maxUnavailable: "25%" - replicas: "1" - # NodeSelector is a selector which must be true for the pod to fit on a node. - # Selector which must match a node's labels for the pod to be scheduled on that node. - # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - nodeSelector: {} - # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, - # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. - tolerations: [] - # Affinity is a group of affinity scheduling rules. - # If specified, the pod's scheduling constraints. - # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core - affinity: {} - # The jvm options for java instance startup - jvmOptions: "" - resources: {} - # limits: - # memory: "4Gi" - # cpu: "2" - # requests: - # memory: "2Gi" - # cpu: "500m" - # You can use annotations to attach arbitrary non-identifying metadata to objects. - # Clients such as tools and libraries can retrieve this metadata. - annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -357,10 +308,8 @@ api: timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" - ## volumeClaimTemplates is a list of claims that pods are allowed to reference. - ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. - ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. - ## A claim in this list takes precedence over any volumes in the template, with the same name. + ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. + ## More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims persistentVolumeClaim: enabled: false accessModes: @@ -368,34 +317,41 @@ api: storageClassName: "-" storage: "20Gi" -frontend: +api: + ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. + replicas: "1" + ## The deployment strategy to use to replace existing pods with new ones. strategy: type: "RollingUpdate" rollingUpdate: maxSurge: "25%" maxUnavailable: "25%" - replicas: "1" - # NodeSelector is a selector which must be true for the pod to fit on a node. - # Selector which must match a node's labels for the pod to be scheduled on that node. - # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## You can use annotations to attach arbitrary non-identifying metadata to objects. + ## Clients such as tools and libraries can retrieve this metadata. + annotations: {} + ## NodeSelector is a selector which must be true for the pod to fit on a node. + ## Selector which must match a node's labels for the pod to be scheduled on that node. + ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + affinity: {} + ## Compute Resources required by this container. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container nodeSelector: {} - # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, - # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] - # Affinity is a group of affinity scheduling rules. - # If specified, the pod's scheduling constraints. - # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core - affinity: {} + ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. + ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core resources: {} - # limits: - # memory: "256Mi" + # resources: + # limits: + # memory: "2Gi" # cpu: "1" # requests: - # memory: "256Mi" + # memory: "1Gi" # cpu: "500m" - # You can use annotations to attach arbitrary non-identifying metadata to objects. - # Clients such as tools and libraries can retrieve this metadata. - annotations: {} + ## ConfigMap + configmap: + DOLPHINSCHEDULER_OPTS: "" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -414,10 +370,8 @@ frontend: timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" - ## volumeClaimTemplates is a list of claims that pods are allowed to reference. - ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. - ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. - ## A claim in this list takes precedence over any volumes in the template, with the same name. + ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. + ## More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims persistentVolumeClaim: enabled: false accessModes: