From f5be784044a191cd305c2979c52378f6115b3984 Mon Sep 17 00:00:00 2001 From: Gallardot Date: Mon, 17 Jul 2023 16:01:05 +0800 Subject: [PATCH] [Feature][K8S] add prometheus serviceMonitor (#14539) Signed-off-by: Gallardot Co-authored-by: Eric Gao --- .../statefulset-dolphinscheduler-master.yaml | 2 + .../statefulset-dolphinscheduler-worker.yaml | 2 + .../templates/svc-dolphinscheduler-alert.yaml | 34 +++++++++++ .../templates/svc-dolphinscheduler-api.yaml | 37 ++++++++++-- .../svc-dolphinscheduler-master-headless.yaml | 39 ++++++++++++- .../svc-dolphinscheduler-worker-headless.yaml | 39 ++++++++++++- .../kubernetes/dolphinscheduler/values.yaml | 58 +++++++++++++++++++ 7 files changed, 204 insertions(+), 7 deletions(-) diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index 1314889468..9eaf40ce62 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -60,6 +60,8 @@ spec: ports: - containerPort: 5678 name: "master-port" + - containerPort: 5679 + name: "actuator-port" env: - name: TZ value: {{ .Values.timezone }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 1be6f55a5b..ba8a2f49de 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -60,6 +60,8 @@ spec: ports: - containerPort: 1234 name: "worker-port" + - containerPort: 1235 + name: "actuator-port" env: - name: TZ value: {{ .Values.timezone }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml index cb4e3ab75c..88f1a5cb7a 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml @@ -18,9 +18,13 @@ apiVersion: v1 kind: Service metadata: name: {{ include "dolphinscheduler.fullname" . }}-alert + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert {{- include "dolphinscheduler.common.labels" . | nindent 4 }} + {{- if .Values.alert.service.annotations }} + annotations: {{- toYaml .Values.alert.service.annotations | nindent 4 }} + {{- end }} spec: ports: - port: 50052 @@ -33,3 +37,33 @@ spec: name: actuator-port selector: {{- include "dolphinscheduler.alert.labels" . | nindent 4 }} + +--- +{{- if .Values.alert.service.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-alert + namespace: {{ .Release.Namespace }} + labels: + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} + {{- if .Values.alert.service.serviceMonitor.labels }} + {{- toYaml .Values.alert.service.serviceMonitor.labels | nindent 4 }} + {{- end }} + {{- if .Values.alert.service.serviceMonitor.annotations }} + annotations: {{- toYaml .Values.alert.service.serviceMonitor.annotations | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + matchNames: + - {{ .Values.alert.service.serviceMonitor.namespace | default .Release.Namespace }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert + {{- include "dolphinscheduler.common.labels" . | nindent 6 }} + endpoints: + - scheme: http + targetPort: actuator-port + path: {{ .Values.alert.service.serviceMonitor.path }} + interval: {{ .Values.alert.service.serviceMonitor.interval }} +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml index 3ef36af9a0..a58bc1e20b 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml @@ -21,11 +21,8 @@ metadata: labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api {{- include "dolphinscheduler.common.labels" . | nindent 4 }} - {{- if and (eq .Values.api.service.type "LoadBalancer") .Values.api.service.annotations }} - annotations: - {{- range $key, $value := .Values.api.service.annotations }} - {{ $key }}: {{ $value | quote }} - {{- end }} + {{- if .Values.api.service.annotations }} + annotations: {{- toYaml .Values.api.service.annotations | nindent 4 }} {{- end }} spec: type: {{ .Values.api.service.type }} @@ -59,3 +56,33 @@ spec: {{- end }} selector: {{- include "dolphinscheduler.api.labels" . | nindent 4 }} + +--- +{{- if .Values.api.service.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-api + namespace: {{ .Release.Namespace }} + labels: + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} + {{- if .Values.api.service.serviceMonitor.labels }} + {{- toYaml .Values.api.service.serviceMonitor.labels | nindent 4 }} + {{- end }} + {{- if .Values.api.service.serviceMonitor.annotations }} + annotations: {{- toYaml .Values.api.service.serviceMonitor.annotations | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + matchNames: + - {{ .Values.api.service.serviceMonitor.namespace | default .Release.Namespace }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api + {{- include "dolphinscheduler.common.labels" . | nindent 6 }} + endpoints: + - scheme: http + targetPort: api-port + path: {{ .Values.api.service.serviceMonitor.path }} + interval: {{ .Values.api.service.serviceMonitor.interval }} +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml index 9fb21ca664..a4635e585d 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml @@ -21,6 +21,9 @@ metadata: labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless {{- include "dolphinscheduler.common.labels" . | nindent 4 }} + {{- if .Values.master.service.annotations }} + annotations: {{- toYaml .Values.master.service.annotations | nindent 4 }} + {{- end }} spec: clusterIP: "None" ports: @@ -28,5 +31,39 @@ spec: targetPort: master-port protocol: TCP name: master-port + - port: 5679 + targetPort: actuator-port + protocol: TCP + name: actuator-port + selector: + {{- include "dolphinscheduler.master.labels" . | nindent 4 }} + +--- +{{- if .Values.master.service.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-master + namespace: {{ .Release.Namespace }} + labels: + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} + {{- if .Values.master.service.serviceMonitor.labels }} + {{- toYaml .Values.master.service.serviceMonitor.labels | nindent 4 }} + {{- end }} + {{- if .Values.master.service.serviceMonitor.annotations }} + annotations: {{- toYaml .Values.master.service.serviceMonitor.annotations | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + matchNames: + - {{ .Values.master.service.serviceMonitor.namespace | default .Release.Namespace }} selector: - {{- include "dolphinscheduler.master.labels" . | nindent 4 }} \ No newline at end of file + matchLabels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless + {{- include "dolphinscheduler.common.labels" . | nindent 6 }} + endpoints: + - scheme: http + targetPort: actuator-port + path: {{ .Values.master.service.serviceMonitor.path }} + interval: {{ .Values.master.service.serviceMonitor.interval }} +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml index bb6baf8034..8851489707 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml +++ b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml @@ -21,6 +21,9 @@ metadata: labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless {{- include "dolphinscheduler.common.labels" . | nindent 4 }} + {{- if .Values.worker.service.annotations }} + annotations: {{- toYaml .Values.worker.service.annotations | nindent 4 }} + {{- end }} spec: clusterIP: "None" ports: @@ -28,5 +31,39 @@ spec: targetPort: worker-port protocol: TCP name: worker-port + - port: 1235 + targetPort: actuator-port + protocol: TCP + name: actuator-port + selector: + {{- include "dolphinscheduler.worker.labels" . | nindent 4 }} + +--- +{{- if .Values.worker.service.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-worker + namespace: {{ .Release.Namespace }} + labels: + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} + {{- if .Values.worker.service.serviceMonitor.labels }} + {{- toYaml .Values.worker.service.serviceMonitor.labels | nindent 4 }} + {{- end }} + {{- if .Values.worker.service.serviceMonitor.annotations }} + annotations: {{- toYaml .Values.worker.service.serviceMonitor.annotations | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + matchNames: + - {{ .Values.worker.service.serviceMonitor.namespace | default .Release.Namespace }} selector: - {{- include "dolphinscheduler.worker.labels" . | nindent 4 }} \ No newline at end of file + matchLabels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless + {{- include "dolphinscheduler.common.labels" . | nindent 6 }} + endpoints: + - scheme: http + targetPort: actuator-port + path: {{ .Values.worker.service.serviceMonitor.path }} + interval: {{ .Values.worker.service.serviceMonitor.interval }} +{{- end }} diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 4e38213c49..e24b3cb45c 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -352,6 +352,21 @@ master: MASTER_RESERVED_MEMORY: "0.3" MASTER_FAILOVER_INTERVAL: "10m" MASTER_KILL_APPLICATION_WHEN_HANDLE_FAILOVER: "true" + service: + # annotations may need to be set when want to scrapy metrics by prometheus but not install prometheus operator + annotations: {} + # serviceMonitor for prometheus operator + serviceMonitor: + # -- Enable or disable master serviceMonitor + enabled: false + # -- @param serviceMonitor.interval interval at which metrics should be scraped + interval: 15s + # -- @param serviceMonitor.path path of the metrics endpoint + path: /actuator/prometheus + # -- @param serviceMonitor.labels ServiceMonitor extra labels + labels: {} + # -- @param serviceMonitor.annotations ServiceMonitor annotations + annotations: {} worker: ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. @@ -453,6 +468,21 @@ worker: # - type: Percent # value: 100 # periodSeconds: 15 + service: + # annotations may need to be set when want to scrapy metrics by prometheus but not install prometheus operator + annotations: {} + # serviceMonitor for prometheus operator + serviceMonitor: + # -- Enable or disable worker serviceMonitor + enabled: false + # -- @param serviceMonitor.interval interval at which metrics should be scraped + interval: 15s + # -- @param serviceMonitor.path path of the metrics endpoint + path: /actuator/prometheus + # -- @param serviceMonitor.labels ServiceMonitor extra labels + labels: {} + # -- @param serviceMonitor.annotations ServiceMonitor annotations + annotations: {} alert: @@ -515,6 +545,22 @@ alert: storage: "20Gi" env: JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m" + service: + # annotations may need to be set when want to scrapy metrics by prometheus but not install prometheus operator + annotations: {} + # serviceMonitor for prometheus operator + serviceMonitor: + # -- Enable or disable alert-server serviceMonitor + enabled: false + # -- @param serviceMonitor.interval interval at which metrics should be scraped + interval: 15s + # -- @param serviceMonitor.path path of the metrics endpoint + path: /actuator/prometheus + # -- @param serviceMonitor.labels ServiceMonitor extra labels + labels: {} + # -- @param serviceMonitor.annotations ServiceMonitor annotations + annotations: {} + api: ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. @@ -592,6 +638,18 @@ api: ## annotations may need to be set when service.type is LoadBalancer ## service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:EXAMPLE_CERT annotations: {} + # serviceMonitor for prometheus operator + serviceMonitor: + # -- Enable or disable api-server serviceMonitor + enabled: false + # -- @param serviceMonitor.interval interval at which metrics should be scraped + interval: 15s + # -- @param serviceMonitor.path path of the metrics endpoint + path: /dolphinscheduler/actuator/prometheus + # -- @param serviceMonitor.labels ServiceMonitor extra labels + labels: {} + # -- @param serviceMonitor.annotations ServiceMonitor annotations + annotations: {} env: JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m"