From 12b46dfaa62f8f6c51ea739e0d31ca934c811994 Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Sun, 12 Dec 2021 23:17:50 +0800 Subject: [PATCH] Adjust Helm Chart templates in dev branch (#7353) --- .../dolphinscheduler/templates/_helpers.tpl | 72 ++++++++++--------- .../configmap-dolphinscheduler-alert.yaml | 29 -------- .../configmap-dolphinscheduler-api.yaml | 29 -------- .../configmap-dolphinscheduler-master.yaml | 29 -------- .../configmap-dolphinscheduler-worker.yaml | 29 -------- .../deployment-dolphinscheduler-alert.yaml | 18 ++--- .../deployment-dolphinscheduler-api.yaml | 16 ++--- ...b-dolphinscheduler-schema-initializer.yaml | 50 +++++++++++++ .../statefulset-dolphinscheduler-master.yaml | 16 ++--- .../statefulset-dolphinscheduler-worker.yaml | 18 +++-- .../templates/svc-dolphinscheduler-alert.yaml | 6 +- .../kubernetes/dolphinscheduler/values.yaml | 56 +++++++-------- dolphinscheduler-meter/pom.xml | 14 ++++ 13 files changed, 165 insertions(+), 217 deletions(-) delete mode 100644 deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml delete mode 100644 deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml delete mode 100644 deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml delete mode 100644 deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml create mode 100644 deploy/kubernetes/dolphinscheduler/templates/job-dolphinscheduler-schema-initializer.yaml diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl index 5ef83a72d9..c2684e029a 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -27,10 +27,22 @@ If release name contains chart name it will be used as a full name. {{- end -}} {{/* -Create a default docker image fullname. +Create default docker images' fullname. */}} -{{- define "dolphinscheduler.image.fullname" -}} -{{- .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion -}} +{{- define "dolphinscheduler.image.fullname.master" -}} +{{- .Values.image.registry }}/dolphinscheduler-master:{{ .Values.image.tag | default .Chart.AppVersion -}} +{{- end -}} +{{- define "dolphinscheduler.image.fullname.worker" -}} +{{- .Values.image.registry }}/dolphinscheduler-worker:{{ .Values.image.tag | default .Chart.AppVersion -}} +{{- end -}} +{{- define "dolphinscheduler.image.fullname.api" -}} +{{- .Values.image.registry }}/dolphinscheduler-api:{{ .Values.image.tag | default .Chart.AppVersion -}} +{{- end -}} +{{- define "dolphinscheduler.image.fullname.alert" -}} +{{- .Values.image.registry }}/dolphinscheduler-alert-server:{{ .Values.image.tag | default .Chart.AppVersion -}} +{{- end -}} +{{- define "dolphinscheduler.image.fullname.tools" -}} +{{- .Values.image.registry }}/dolphinscheduler-tools:{{ .Values.image.tag | default .Chart.AppVersion -}} {{- end -}} {{/* @@ -108,37 +120,25 @@ Create a default fully qualified zookkeeper quorum. Create a database environment variables. */}} {{- define "dolphinscheduler.database.env_vars" -}} -- name: DATABASE_TYPE +- name: DATABASE {{- if .Values.postgresql.enabled }} value: "postgresql" {{- else }} value: {{ .Values.externalDatabase.type | quote }} {{- end }} -- name: DATABASE_DRIVER - {{- if .Values.postgresql.enabled }} - value: "org.postgresql.Driver" - {{- else }} - value: {{ .Values.externalDatabase.driver | quote }} - {{- end }} -- name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} -- name: DATABASE_PORT +- name: SPRING_DATASOURCE_URL {{- if .Values.postgresql.enabled }} - value: "5432" + value: jdbc:postgresql://{{ template "dolphinscheduler.postgresql.fullname" . }}:5432/{{ .Values.postgresql.postgresqlDatabase }}?characterEncoding=utf8 {{- else }} - value: {{ .Values.externalDatabase.port | quote }} + value: jdbc:{{ .Values.externalDatabase.type }}://{{ .Values.externalDatabase.host }}:{{ .Values.externalDatabase.port }}/{{ .Values.externalDatabase.database }}?{{ .Values.externalDatabase.params }} {{- end }} -- name: DATABASE_USERNAME +- name: SPRING_DATASOURCE_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} {{- end }} -- name: DATABASE_PASSWORD +- name: SPRING_DATASOURCE_PASSWORD valueFrom: secretKeyRef: {{- if .Values.postgresql.enabled }} @@ -148,31 +148,33 @@ Create a database environment variables. name: {{ include "dolphinscheduler.fullname" . }}-externaldb key: database-password {{- end }} -- name: DATABASE_DATABASE - {{- if .Values.postgresql.enabled }} - value: {{ .Values.postgresql.postgresqlDatabase }} - {{- else }} - value: {{ .Values.externalDatabase.database | quote }} - {{- end }} -- name: DATABASE_PARAMS - {{- if .Values.postgresql.enabled }} - value: "characterEncoding=utf8" - {{- else }} - value: {{ .Values.externalDatabase.params | quote }} - {{- end }} +{{- end -}} + +{{/* +Wait for database to be ready. +*/}} +{{- define "dolphinscheduler.database.wait-for-ready" -}} +- name: wait-for-database + image: busybox:1.30 + imagePullPolicy: IfNotPresent +{{- if .Values.postgresql.enabled }} + command: ['sh', '-xc', 'for i in $(seq 1 180); do nc -z -w3 {{ template "dolphinscheduler.postgresql.fullname" . }} 5432 && exit 0 || sleep 5; done; exit 1'] +{{- else }} + command: ['sh', '-xc', 'for i in $(seq 1 180); do nc -z -w3 {{ .Values.externalDatabase.host }} {{ .Values.externalDatabase.port }} && exit 0 || sleep 5; done; exit 1'] +{{- end }} {{- end -}} {{/* Create a registry environment variables. */}} {{- define "dolphinscheduler.registry.env_vars" -}} -- name: REGISTRY_PLUGIN_NAME +- name: REGISTRY_TYPE {{- if .Values.zookeeper.enabled }} value: "zookeeper" {{- else }} value: {{ .Values.externalRegistry.registryPluginName }} {{- end }} -- name: REGISTRY_SERVERS +- name: REGISTRY_ZOOKEEPER_CONNECT_STRING {{- if .Values.zookeeper.enabled }} value: {{ template "dolphinscheduler.zookeeper.quorum" . }} {{- else }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml deleted file mode 100644 index 9ef96fbddc..0000000000 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -{{- if .Values.alert.configmap }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "dolphinscheduler.fullname" . }}-alert - labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert - {{- include "dolphinscheduler.common.labels" . | nindent 4 }} -data: - {{- range $key, $value := .Values.alert.configmap }} - {{ $key }}: {{ $value | quote }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml deleted file mode 100644 index 70b818fc4c..0000000000 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -{{- if .Values.api.configmap }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "dolphinscheduler.fullname" . }}-api - labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api - {{- include "dolphinscheduler.common.labels" . | nindent 4 }} -data: - {{- range $key, $value := .Values.api.configmap }} - {{ $key }}: {{ $value | quote }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml deleted file mode 100644 index 34823f66a7..0000000000 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -{{- if .Values.master.configmap }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "dolphinscheduler.fullname" . }}-master - labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master - {{- include "dolphinscheduler.common.labels" . | nindent 4 }} -data: - {{- range $key, $value := .Values.master.configmap }} - {{ $key }}: {{ $value | quote }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml deleted file mode 100644 index 68a292d61f..0000000000 --- a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -{{- if .Values.worker.configmap }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "dolphinscheduler.fullname" . }}-worker - labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker - {{- include "dolphinscheduler.common.labels" . | nindent 4 }} -data: - {{- range $key, $value := .Values.worker.configmap }} - {{ $key }}: {{ $value | quote }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index 70e292a11d..aba9e60d22 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -57,22 +57,24 @@ spec: {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-alert - image: {{ include "dolphinscheduler.image.fullname" . }} + image: {{ include "dolphinscheduler.image.fullname.alert" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - "alert-server" ports: - containerPort: 50052 name: "alert-port" + - containerPort: 50053 + name: "actuator-port" env: - name: TZ value: {{ .Values.timezone }} {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }} + {{ range $key, $value := .Values.alert.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{ end }} envFrom: - configMapRef: name: {{ include "dolphinscheduler.fullname" . }}-common - - configMapRef: - name: {{ include "dolphinscheduler.fullname" . }}-alert {{- if .Values.alert.resources }} resources: {{- toYaml .Values.alert.resources | nindent 12 }} @@ -80,7 +82,7 @@ spec: {{- if .Values.alert.livenessProbe.enabled }} livenessProbe: exec: - command: ["bash", "/root/checkpoint.sh", "AlertServer"] + command: ["curl", "-s", "http://localhost:50053/actuator/health/liveness"] initialDelaySeconds: {{ .Values.alert.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.alert.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.alert.livenessProbe.timeoutSeconds }} @@ -90,7 +92,7 @@ spec: {{- if .Values.alert.readinessProbe.enabled }} readinessProbe: exec: - command: ["bash", "/root/checkpoint.sh", "AlertServer"] + command: ["curl", "-s", "http://localhost:50053/actuator/health/readiness"] initialDelaySeconds: {{ .Values.alert.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.alert.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.alert.readinessProbe.timeoutSeconds }} @@ -107,4 +109,4 @@ spec: claimName: {{ include "dolphinscheduler.fullname" . }}-alert {{- else }} emptyDir: {} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index b2838dff47..64ff6abd2e 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -57,10 +57,8 @@ spec: {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-api - image: {{ include "dolphinscheduler.image.fullname" . }} + image: {{ include "dolphinscheduler.image.fullname.api" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - "api-server" ports: - containerPort: 12345 name: "api-port" @@ -70,11 +68,13 @@ spec: {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }} {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }} {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }} + {{ range $key, $value := .Values.api.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{ end }} envFrom: - configMapRef: name: {{ include "dolphinscheduler.fullname" . }}-common - - configMapRef: - name: {{ include "dolphinscheduler.fullname" . }}-api {{- if .Values.api.resources }} resources: {{- toYaml .Values.api.resources | nindent 12 }} @@ -82,7 +82,7 @@ spec: {{- if .Values.api.livenessProbe.enabled }} livenessProbe: exec: - command: ["bash", "/root/checkpoint.sh", "ApiApplicationServer"] + command: ["curl", "-s", "http://localhost:12345/actuator/health/liveness"] initialDelaySeconds: {{ .Values.api.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.api.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.api.livenessProbe.timeoutSeconds }} @@ -92,7 +92,7 @@ spec: {{- if .Values.api.readinessProbe.enabled }} readinessProbe: exec: - command: ["bash", "/root/checkpoint.sh", "ApiApplicationServer"] + command: ["curl", "-s", "http://localhost:12345/actuator/health/readiness"] initialDelaySeconds: {{ .Values.api.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.api.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.api.readinessProbe.timeoutSeconds }} @@ -113,4 +113,4 @@ spec: emptyDir: {} {{- end }} {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} - {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }} \ No newline at end of file + {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/job-dolphinscheduler-schema-initializer.yaml b/deploy/kubernetes/dolphinscheduler/templates/job-dolphinscheduler-schema-initializer.yaml new file mode 100644 index 0000000000..4df2ebd3ec --- /dev/null +++ b/deploy/kubernetes/dolphinscheduler/templates/job-dolphinscheduler-schema-initializer.yaml @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-db-init-job + labels: + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} +spec: + template: + metadata: + labels: + {{- include "dolphinscheduler.common.labels" . | nindent 8 }} + spec: + {{- if .Values.image.pullSecret }} + imagePullSecrets: + - name: {{ .Values.image.pullSecret }} + {{- end }} + restartPolicy: Never + initContainers: + {{- include "dolphinscheduler.database.wait-for-ready" . | nindent 6 }} + containers: + - name: {{ include "dolphinscheduler.fullname" . }}-db-init-job + image: {{ include "dolphinscheduler.image.fullname.tools" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - bin/create-schema.sh + env: + - name: TZ + value: {{ .Values.timezone }} + {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }} + {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }} + {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "dolphinscheduler.fullname" . }}-common diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index 686d260958..503ff23f84 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -54,10 +54,8 @@ spec: {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-master - image: {{ include "dolphinscheduler.image.fullname" . }} + image: {{ include "dolphinscheduler.image.fullname.master" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - "master-server" ports: - containerPort: 5678 name: "master-port" @@ -67,11 +65,13 @@ spec: {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }} {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }} {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }} + {{ range $key, $value := .Values.master.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{ end }} envFrom: - configMapRef: name: {{ include "dolphinscheduler.fullname" . }}-common - - configMapRef: - name: {{ include "dolphinscheduler.fullname" . }}-master {{- if .Values.master.resources }} resources: {{- toYaml .Values.master.resources | nindent 12 }} @@ -79,7 +79,7 @@ spec: {{- if .Values.master.livenessProbe.enabled }} livenessProbe: exec: - command: ["bash", "/root/checkpoint.sh", "MasterServer"] + command: ["curl", "-s", "http://localhost:5679/actuator/health/liveness"] initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.master.livenessProbe.timeoutSeconds }} @@ -89,7 +89,7 @@ spec: {{- if .Values.master.readinessProbe.enabled }} readinessProbe: exec: - command: ["bash", "/root/checkpoint.sh", "MasterServer"] + command: ["curl", "-s", "http://localhost:5679/actuator/health/readiness"] initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.master.readinessProbe.timeoutSeconds }} @@ -125,4 +125,4 @@ spec: resources: requests: storage: {{ .Values.master.persistentVolumeClaim.storage | quote }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 392ae0773b..44328055ff 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -54,10 +54,8 @@ spec: {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-worker - image: {{ include "dolphinscheduler.image.fullname" . }} + image: {{ include "dolphinscheduler.image.fullname.worker" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - "worker-server" ports: - containerPort: 1234 name: "worker-port" @@ -71,13 +69,13 @@ spec: {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }} {{- include "dolphinscheduler.registry.env_vars" . | nindent 12 }} {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }} + {{ range $key, $value := .Values.worker.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{ end }} envFrom: - configMapRef: name: {{ include "dolphinscheduler.fullname" . }}-common - - configMapRef: - name: {{ include "dolphinscheduler.fullname" . }}-worker - - configMapRef: - name: {{ include "dolphinscheduler.fullname" . }}-alert {{- if .Values.worker.resources }} resources: {{- toYaml .Values.worker.resources | nindent 12 }} @@ -85,7 +83,7 @@ spec: {{- if .Values.worker.livenessProbe.enabled }} livenessProbe: exec: - command: ["bash", "/root/checkpoint.sh", "WorkerServer"] + command: ["curl", "-s", "http://localhost:1235/actuator/health/liveness"] initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }} @@ -95,7 +93,7 @@ spec: {{- if .Values.worker.readinessProbe.enabled }} readinessProbe: exec: - command: ["bash", "/root/checkpoint.sh", "WorkerServer"] + command: ["curl", "-s", "http://localhost:1235/actuator/health/readiness"] initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} @@ -160,4 +158,4 @@ spec: requests: storage: {{ .Values.worker.persistentVolumeClaim.logsPersistentVolume.storage | quote }} {{- end }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml index a24dfe6f61..cb4e3ab75c 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml @@ -27,5 +27,9 @@ spec: targetPort: alert-port protocol: TCP name: alert-port + - port: 50053 + targetPort: actuator-port + protocol: TCP + name: actuator-port selector: - {{- include "dolphinscheduler.alert.labels" . | nindent 4 }} \ No newline at end of file + {{- include "dolphinscheduler.alert.labels" . | nindent 4 }} diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 03245c39b7..92d6a04bcf 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -22,7 +22,7 @@ timezone: "Asia/Shanghai" image: - repository: "dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler" + registry: "dolphinscheduler.docker.scarf.sh/apache" tag: "2.0.0-SNAPSHOT" pullPolicy: "IfNotPresent" pullSecret: "" @@ -42,7 +42,6 @@ postgresql: ## external database will be used, otherwise Dolphinscheduler's database will be used. externalDatabase: type: "postgresql" - driver: "org.postgresql.Driver" host: "localhost" port: "5432" username: "root" @@ -146,18 +145,6 @@ master: # requests: # memory: "2Gi" # cpu: "500m" - ## Configmap - configmap: - MASTER_SERVER_OPTS: "-Xms1g -Xmx1g -Xmn512m" - MASTER_EXEC_THREADS: "100" - MASTER_EXEC_TASK_NUM: "20" - MASTER_DISPATCH_TASK_NUM: "3" - MASTER_HOST_SELECTOR: "LowerWeight" - MASTER_HEARTBEAT_INTERVAL: "10" - MASTER_TASK_COMMIT_RETRYTIMES: "5" - MASTER_TASK_COMMIT_INTERVAL: "1000" - MASTER_MAX_CPULOAD_AVG: "-1" - MASTER_RESERVED_MEMORY: "0.3" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -186,6 +173,17 @@ master: - "ReadWriteOnce" storageClassName: "-" storage: "20Gi" + env: + JAVA_OPTS: "-Xms1g -Xmx1g -Xmn512m" + MASTER_EXEC_THREADS: "100" + MASTER_EXEC_TASK_NUM: "20" + MASTER_DISPATCH_TASK_NUM: "3" + MASTER_HOST_SELECTOR: "LowerWeight" + MASTER_HEARTBEAT_INTERVAL: "10" + MASTER_TASK_COMMIT_RETRYTIMES: "5" + MASTER_TASK_COMMIT_INTERVAL: "1000" + MASTER_MAX_CPU_LOAD_AVG: "-1" + MASTER_RESERVED_MEMORY: "0.3" worker: ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. @@ -215,16 +213,6 @@ worker: # requests: # memory: "2Gi" # cpu: "500m" - ## Configmap - configmap: - LOGGER_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m" - WORKER_SERVER_OPTS: "-Xms1g -Xmx1g -Xmn512m" - WORKER_EXEC_THREADS: "100" - WORKER_HEARTBEAT_INTERVAL: "10" - WORKER_HOST_WEIGHT: "100" - WORKER_MAX_CPULOAD_AVG: "-1" - WORKER_RESERVED_MEMORY: "0.3" - WORKER_GROUPS: "default" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -263,10 +251,18 @@ worker: - "ReadWriteOnce" storageClassName: "-" storage: "20Gi" + env: + WORKER_GROUPS_0: default + WORKER_MAX_CPU_LOAD_AVG: "-1" + WORKER_RESERVED_MEMORY: "0.3" + WORKER_EXEC_THREADS: "100" + WORKER_HEARTBEAT_INTERVAL: "10" + WORKER_HOST_WEIGHT: "100" + WORKER_GROUPS: "default" alert: ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. - replicas: "1" + replicas: 1 ## The deployment strategy to use to replace existing pods with new ones. strategy: type: "RollingUpdate" @@ -296,9 +292,6 @@ alert: # requests: # memory: "1Gi" # cpu: "500m" - ## Configmap - configmap: - ALERT_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -325,6 +318,8 @@ alert: - "ReadWriteOnce" storageClassName: "-" storage: "20Gi" + env: + JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m" api: ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. @@ -358,9 +353,6 @@ api: # requests: # memory: "1Gi" # cpu: "500m" - ## Configmap - configmap: - API_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -403,6 +395,8 @@ api: ## annotations may need to be set when service.type is LoadBalancer ## service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:EXAMPLE_CERT annotations: {} + env: + JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m" ingress: enabled: false diff --git a/dolphinscheduler-meter/pom.xml b/dolphinscheduler-meter/pom.xml index 5ff8e594c3..442167cd54 100644 --- a/dolphinscheduler-meter/pom.xml +++ b/dolphinscheduler-meter/pom.xml @@ -55,6 +55,20 @@ + + org.springframework.boot + spring-boot-starter-jetty + + + org.eclipse.jetty.websocket + javax-websocket-server-impl + + + org.eclipse.jetty.websocket + websocket-server + + + org.springframework.boot spring-boot-actuator