[Feature][K8S] add prometheus serviceMonitor (#14539)

Signed-off-by: Gallardot <gallardot@apache.org>
Co-authored-by: Eric Gao <ericgao.apache@gmail.com>

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@@ -60,6 +60,8 @@ spec:
           ports:
             - containerPort: 5678
               name: "master-port"
+            - containerPort: 5679
+              name: "actuator-port"
           env:
             - name: TZ
               value: {{ .Values.timezone }}

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -60,6 +60,8 @@ spec:
           ports:
             - containerPort: 1234
               name: "worker-port"
+            - containerPort: 1235
+              name: "actuator-port"
           env:
             - name: TZ
               value: {{ .Values.timezone }}

deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml

@@ -18,9 +18,13 @@ apiVersion: v1
 kind: Service
 metadata:
   name: {{ include "dolphinscheduler.fullname" . }}-alert
+  namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
     {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
+  {{- if .Values.alert.service.annotations }}
+  annotations: {{- toYaml .Values.alert.service.annotations | nindent 4 }}
+  {{- end }}
 spec:
   ports:
     - port: 50052
@@ -33,3 +37,33 @@ spec:
       name: actuator-port
   selector:
     {{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
+
+---
+{{- if .Values.alert.service.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-alert
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
+    {{- if .Values.alert.service.serviceMonitor.labels }}
+    {{- toYaml .Values.alert.service.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+  {{- if .Values.alert.service.serviceMonitor.annotations }}
+  annotations: {{- toYaml .Values.alert.service.serviceMonitor.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  namespaceSelector:
+    matchNames:
+    - {{ .Values.alert.service.serviceMonitor.namespace | default .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
+      {{- include "dolphinscheduler.common.labels" . | nindent 6 }}
+  endpoints:
+  - scheme: http
+    targetPort: actuator-port
+    path: {{ .Values.alert.service.serviceMonitor.path }}
+    interval: {{ .Values.alert.service.serviceMonitor.interval }}
+{{- end }}

deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml

@@ -21,11 +21,8 @@ metadata:
   labels:
     app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
     {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
-  {{- if and (eq .Values.api.service.type "LoadBalancer") .Values.api.service.annotations }}
-  annotations:
-    {{- range $key, $value := .Values.api.service.annotations }}
-    {{ $key }}: {{ $value | quote }}
-    {{- end }}
+  {{- if .Values.api.service.annotations }}
+  annotations: {{- toYaml .Values.api.service.annotations | nindent 4 }}
   {{- end }}
 spec:
   type: {{ .Values.api.service.type }}
@@ -59,3 +56,33 @@ spec:
   {{- end }}
   selector:
     {{- include "dolphinscheduler.api.labels" . | nindent 4 }}
+
+---
+{{- if .Values.api.service.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-api
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
+    {{- if .Values.api.service.serviceMonitor.labels }}
+    {{- toYaml .Values.api.service.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+  {{- if .Values.api.service.serviceMonitor.annotations }}
+  annotations: {{- toYaml .Values.api.service.serviceMonitor.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  namespaceSelector:
+    matchNames:
+    - {{ .Values.api.service.serviceMonitor.namespace | default .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
+      {{- include "dolphinscheduler.common.labels" . | nindent 6 }}
+  endpoints:
+  - scheme: http
+    targetPort: api-port
+    path: {{ .Values.api.service.serviceMonitor.path }}
+    interval: {{ .Values.api.service.serviceMonitor.interval }}
+{{- end }}

deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml

@@ -21,6 +21,9 @@ metadata:
   labels:
     app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless
     {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
+  {{- if .Values.master.service.annotations }}
+  annotations: {{- toYaml .Values.master.service.annotations | nindent 4 }}
+  {{- end }}
 spec:
   clusterIP: "None"
   ports:
@@ -28,5 +31,39 @@ spec:
       targetPort: master-port
       protocol: TCP
       name: master-port
+    - port: 5679
+      targetPort: actuator-port
+      protocol: TCP
+      name: actuator-port
+  selector:
+    {{- include "dolphinscheduler.master.labels" . | nindent 4 }}
+
+---
+{{- if .Values.master.service.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-master
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
+    {{- if .Values.master.service.serviceMonitor.labels }}
+    {{- toYaml .Values.master.service.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+  {{- if .Values.master.service.serviceMonitor.annotations }}
+  annotations: {{- toYaml .Values.master.service.serviceMonitor.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  namespaceSelector:
+    matchNames:
+    - {{ .Values.master.service.serviceMonitor.namespace | default .Release.Namespace }}
   selector:
-    {{- include "dolphinscheduler.master.labels" . | nindent 4 }}
+    matchLabels:
+      app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless
+      {{- include "dolphinscheduler.common.labels" . | nindent 6 }}
+  endpoints:
+  - scheme: http
+    targetPort: actuator-port
+    path: {{ .Values.master.service.serviceMonitor.path }}
+    interval: {{ .Values.master.service.serviceMonitor.interval }}
+{{- end }}

deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml

@@ -21,6 +21,9 @@ metadata:
   labels:
     app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless
     {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
+  {{- if .Values.worker.service.annotations }}
+  annotations: {{- toYaml .Values.worker.service.annotations | nindent 4 }}
+  {{- end }}
 spec:
   clusterIP: "None"
   ports:
@@ -28,5 +31,39 @@ spec:
       targetPort: worker-port
       protocol: TCP
       name: worker-port
+    - port: 1235
+      targetPort: actuator-port
+      protocol: TCP
+      name: actuator-port
+  selector:
+    {{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
+
+---
+{{- if .Values.worker.service.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "dolphinscheduler.fullname" . }}-worker
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
+    {{- if .Values.worker.service.serviceMonitor.labels }}
+    {{- toYaml .Values.worker.service.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+  {{- if .Values.worker.service.serviceMonitor.annotations }}
+  annotations: {{- toYaml .Values.worker.service.serviceMonitor.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  namespaceSelector:
+    matchNames:
+    - {{ .Values.worker.service.serviceMonitor.namespace | default .Release.Namespace }}
   selector:
-    {{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
+    matchLabels:
+      app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless
+      {{- include "dolphinscheduler.common.labels" . | nindent 6 }}
+  endpoints:
+  - scheme: http
+    targetPort: actuator-port
+    path: {{ .Values.worker.service.serviceMonitor.path }}
+    interval: {{ .Values.worker.service.serviceMonitor.interval }}
+{{- end }}

deploy/kubernetes/dolphinscheduler/values.yaml

@@ -352,6 +352,21 @@ master:
     MASTER_RESERVED_MEMORY: "0.3"
     MASTER_FAILOVER_INTERVAL: "10m"
     MASTER_KILL_APPLICATION_WHEN_HANDLE_FAILOVER: "true"
+  service:
+    # annotations may need to be set when want to scrapy metrics by prometheus but not install prometheus operator
+    annotations: {}
+    # serviceMonitor for prometheus operator
+    serviceMonitor:
+      # -- Enable or disable master serviceMonitor
+      enabled: false
+      # -- @param serviceMonitor.interval interval at which metrics should be scraped
+      interval: 15s
+      # -- @param serviceMonitor.path path of the metrics endpoint
+      path: /actuator/prometheus
+      # -- @param serviceMonitor.labels ServiceMonitor extra labels
+      labels: {}
+      # -- @param serviceMonitor.annotations ServiceMonitor annotations
+      annotations: {}
 
 worker:
   ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down.
@@ -453,6 +468,21 @@ worker:
     #         - type: Percent
     #           value: 100
     #           periodSeconds: 15
+  service:
+    # annotations may need to be set when want to scrapy metrics by prometheus but not install prometheus operator
+    annotations: {}
+    # serviceMonitor for prometheus operator
+    serviceMonitor:
+      # -- Enable or disable worker serviceMonitor
+      enabled: false
+      # -- @param serviceMonitor.interval interval at which metrics should be scraped
+      interval: 15s
+      # -- @param serviceMonitor.path path of the metrics endpoint
+      path: /actuator/prometheus
+      # -- @param serviceMonitor.labels ServiceMonitor extra labels
+      labels: {}
+      # -- @param serviceMonitor.annotations ServiceMonitor annotations
+      annotations: {}
 
 
 alert:
@@ -515,6 +545,22 @@ alert:
     storage: "20Gi"
   env:
     JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m"
+  service:
+    # annotations may need to be set when want to scrapy metrics by prometheus but not install prometheus operator
+    annotations: {}
+    # serviceMonitor for prometheus operator
+    serviceMonitor:
+      # -- Enable or disable alert-server serviceMonitor
+      enabled: false
+      # -- @param serviceMonitor.interval interval at which metrics should be scraped
+      interval: 15s
+      # -- @param serviceMonitor.path path of the metrics endpoint
+      path: /actuator/prometheus
+      # -- @param serviceMonitor.labels ServiceMonitor extra labels
+      labels: {}
+      # -- @param serviceMonitor.annotations ServiceMonitor annotations
+      annotations: {}
+
 
 api:
   ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1.
@@ -592,6 +638,18 @@ api:
     ## annotations may need to be set when service.type is LoadBalancer
     ## service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:EXAMPLE_CERT
     annotations: {}
+    # serviceMonitor for prometheus operator
+    serviceMonitor:
+      # -- Enable or disable api-server serviceMonitor
+      enabled: false
+      # -- @param serviceMonitor.interval interval at which metrics should be scraped
+      interval: 15s
+      # -- @param serviceMonitor.path path of the metrics endpoint
+      path: /dolphinscheduler/actuator/prometheus
+      # -- @param serviceMonitor.labels ServiceMonitor extra labels
+      labels: {}
+      # -- @param serviceMonitor.annotations ServiceMonitor annotations
+      annotations: {}
   env:
     JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m"