Browse Source

[Feature][K8S] add prometheus serviceMonitor (#14539)

Signed-off-by: Gallardot <gallardot@apache.org>
Co-authored-by: Eric Gao <ericgao.apache@gmail.com>
3.2.1-prepare
Gallardot 1 year ago committed by GitHub
parent
commit
f5be784044
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
  2. 2
      deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
  3. 34
      deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml
  4. 37
      deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml
  5. 37
      deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml
  6. 37
      deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml
  7. 58
      deploy/kubernetes/dolphinscheduler/values.yaml

2
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@ -60,6 +60,8 @@ spec:
ports: ports:
- containerPort: 5678 - containerPort: 5678
name: "master-port" name: "master-port"
- containerPort: 5679
name: "actuator-port"
env: env:
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}

2
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@ -60,6 +60,8 @@ spec:
ports: ports:
- containerPort: 1234 - containerPort: 1234
name: "worker-port" name: "worker-port"
- containerPort: 1235
name: "actuator-port"
env: env:
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}

34
deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml

@ -18,9 +18,13 @@ apiVersion: v1
kind: Service kind: Service
metadata: metadata:
name: {{ include "dolphinscheduler.fullname" . }}-alert name: {{ include "dolphinscheduler.fullname" . }}-alert
namespace: {{ .Release.Namespace }}
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- include "dolphinscheduler.common.labels" . | nindent 4 }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
{{- if .Values.alert.service.annotations }}
annotations: {{- toYaml .Values.alert.service.annotations | nindent 4 }}
{{- end }}
spec: spec:
ports: ports:
- port: 50052 - port: 50052
@ -33,3 +37,33 @@ spec:
name: actuator-port name: actuator-port
selector: selector:
{{- include "dolphinscheduler.alert.labels" . | nindent 4 }} {{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
---
{{- if .Values.alert.service.serviceMonitor.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-alert
namespace: {{ .Release.Namespace }}
labels:
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
{{- if .Values.alert.service.serviceMonitor.labels }}
{{- toYaml .Values.alert.service.serviceMonitor.labels | nindent 4 }}
{{- end }}
{{- if .Values.alert.service.serviceMonitor.annotations }}
annotations: {{- toYaml .Values.alert.service.serviceMonitor.annotations | nindent 4 }}
{{- end }}
spec:
namespaceSelector:
matchNames:
- {{ .Values.alert.service.serviceMonitor.namespace | default .Release.Namespace }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- include "dolphinscheduler.common.labels" . | nindent 6 }}
endpoints:
- scheme: http
targetPort: actuator-port
path: {{ .Values.alert.service.serviceMonitor.path }}
interval: {{ .Values.alert.service.serviceMonitor.interval }}
{{- end }}

37
deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml

@ -21,11 +21,8 @@ metadata:
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
{{- include "dolphinscheduler.common.labels" . | nindent 4 }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
{{- if and (eq .Values.api.service.type "LoadBalancer") .Values.api.service.annotations }} {{- if .Values.api.service.annotations }}
annotations: annotations: {{- toYaml .Values.api.service.annotations | nindent 4 }}
{{- range $key, $value := .Values.api.service.annotations }}
{{ $key }}: {{ $value | quote }}
{{- end }}
{{- end }} {{- end }}
spec: spec:
type: {{ .Values.api.service.type }} type: {{ .Values.api.service.type }}
@ -59,3 +56,33 @@ spec:
{{- end }} {{- end }}
selector: selector:
{{- include "dolphinscheduler.api.labels" . | nindent 4 }} {{- include "dolphinscheduler.api.labels" . | nindent 4 }}
---
{{- if .Values.api.service.serviceMonitor.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-api
namespace: {{ .Release.Namespace }}
labels:
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
{{- if .Values.api.service.serviceMonitor.labels }}
{{- toYaml .Values.api.service.serviceMonitor.labels | nindent 4 }}
{{- end }}
{{- if .Values.api.service.serviceMonitor.annotations }}
annotations: {{- toYaml .Values.api.service.serviceMonitor.annotations | nindent 4 }}
{{- end }}
spec:
namespaceSelector:
matchNames:
- {{ .Values.api.service.serviceMonitor.namespace | default .Release.Namespace }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
{{- include "dolphinscheduler.common.labels" . | nindent 6 }}
endpoints:
- scheme: http
targetPort: api-port
path: {{ .Values.api.service.serviceMonitor.path }}
interval: {{ .Values.api.service.serviceMonitor.interval }}
{{- end }}

37
deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml

@ -21,6 +21,9 @@ metadata:
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless
{{- include "dolphinscheduler.common.labels" . | nindent 4 }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
{{- if .Values.master.service.annotations }}
annotations: {{- toYaml .Values.master.service.annotations | nindent 4 }}
{{- end }}
spec: spec:
clusterIP: "None" clusterIP: "None"
ports: ports:
@ -28,5 +31,39 @@ spec:
targetPort: master-port targetPort: master-port
protocol: TCP protocol: TCP
name: master-port name: master-port
- port: 5679
targetPort: actuator-port
protocol: TCP
name: actuator-port
selector: selector:
{{- include "dolphinscheduler.master.labels" . | nindent 4 }} {{- include "dolphinscheduler.master.labels" . | nindent 4 }}
---
{{- if .Values.master.service.serviceMonitor.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-master
namespace: {{ .Release.Namespace }}
labels:
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
{{- if .Values.master.service.serviceMonitor.labels }}
{{- toYaml .Values.master.service.serviceMonitor.labels | nindent 4 }}
{{- end }}
{{- if .Values.master.service.serviceMonitor.annotations }}
annotations: {{- toYaml .Values.master.service.serviceMonitor.annotations | nindent 4 }}
{{- end }}
spec:
namespaceSelector:
matchNames:
- {{ .Values.master.service.serviceMonitor.namespace | default .Release.Namespace }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless
{{- include "dolphinscheduler.common.labels" . | nindent 6 }}
endpoints:
- scheme: http
targetPort: actuator-port
path: {{ .Values.master.service.serviceMonitor.path }}
interval: {{ .Values.master.service.serviceMonitor.interval }}
{{- end }}

37
deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml

@ -21,6 +21,9 @@ metadata:
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless
{{- include "dolphinscheduler.common.labels" . | nindent 4 }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
{{- if .Values.worker.service.annotations }}
annotations: {{- toYaml .Values.worker.service.annotations | nindent 4 }}
{{- end }}
spec: spec:
clusterIP: "None" clusterIP: "None"
ports: ports:
@ -28,5 +31,39 @@ spec:
targetPort: worker-port targetPort: worker-port
protocol: TCP protocol: TCP
name: worker-port name: worker-port
- port: 1235
targetPort: actuator-port
protocol: TCP
name: actuator-port
selector: selector:
{{- include "dolphinscheduler.worker.labels" . | nindent 4 }} {{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
---
{{- if .Values.worker.service.serviceMonitor.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-worker
namespace: {{ .Release.Namespace }}
labels:
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
{{- if .Values.worker.service.serviceMonitor.labels }}
{{- toYaml .Values.worker.service.serviceMonitor.labels | nindent 4 }}
{{- end }}
{{- if .Values.worker.service.serviceMonitor.annotations }}
annotations: {{- toYaml .Values.worker.service.serviceMonitor.annotations | nindent 4 }}
{{- end }}
spec:
namespaceSelector:
matchNames:
- {{ .Values.worker.service.serviceMonitor.namespace | default .Release.Namespace }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless
{{- include "dolphinscheduler.common.labels" . | nindent 6 }}
endpoints:
- scheme: http
targetPort: actuator-port
path: {{ .Values.worker.service.serviceMonitor.path }}
interval: {{ .Values.worker.service.serviceMonitor.interval }}
{{- end }}

58
deploy/kubernetes/dolphinscheduler/values.yaml

@ -352,6 +352,21 @@ master:
MASTER_RESERVED_MEMORY: "0.3" MASTER_RESERVED_MEMORY: "0.3"
MASTER_FAILOVER_INTERVAL: "10m" MASTER_FAILOVER_INTERVAL: "10m"
MASTER_KILL_APPLICATION_WHEN_HANDLE_FAILOVER: "true" MASTER_KILL_APPLICATION_WHEN_HANDLE_FAILOVER: "true"
service:
# annotations may need to be set when want to scrapy metrics by prometheus but not install prometheus operator
annotations: {}
# serviceMonitor for prometheus operator
serviceMonitor:
# -- Enable or disable master serviceMonitor
enabled: false
# -- @param serviceMonitor.interval interval at which metrics should be scraped
interval: 15s
# -- @param serviceMonitor.path path of the metrics endpoint
path: /actuator/prometheus
# -- @param serviceMonitor.labels ServiceMonitor extra labels
labels: {}
# -- @param serviceMonitor.annotations ServiceMonitor annotations
annotations: {}
worker: worker:
## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down.
@ -453,6 +468,21 @@ worker:
# - type: Percent # - type: Percent
# value: 100 # value: 100
# periodSeconds: 15 # periodSeconds: 15
service:
# annotations may need to be set when want to scrapy metrics by prometheus but not install prometheus operator
annotations: {}
# serviceMonitor for prometheus operator
serviceMonitor:
# -- Enable or disable worker serviceMonitor
enabled: false
# -- @param serviceMonitor.interval interval at which metrics should be scraped
interval: 15s
# -- @param serviceMonitor.path path of the metrics endpoint
path: /actuator/prometheus
# -- @param serviceMonitor.labels ServiceMonitor extra labels
labels: {}
# -- @param serviceMonitor.annotations ServiceMonitor annotations
annotations: {}
alert: alert:
@ -515,6 +545,22 @@ alert:
storage: "20Gi" storage: "20Gi"
env: env:
JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m" JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m"
service:
# annotations may need to be set when want to scrapy metrics by prometheus but not install prometheus operator
annotations: {}
# serviceMonitor for prometheus operator
serviceMonitor:
# -- Enable or disable alert-server serviceMonitor
enabled: false
# -- @param serviceMonitor.interval interval at which metrics should be scraped
interval: 15s
# -- @param serviceMonitor.path path of the metrics endpoint
path: /actuator/prometheus
# -- @param serviceMonitor.labels ServiceMonitor extra labels
labels: {}
# -- @param serviceMonitor.annotations ServiceMonitor annotations
annotations: {}
api: api:
## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1.
@ -592,6 +638,18 @@ api:
## annotations may need to be set when service.type is LoadBalancer ## annotations may need to be set when service.type is LoadBalancer
## service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:EXAMPLE_CERT ## service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:EXAMPLE_CERT
annotations: {} annotations: {}
# serviceMonitor for prometheus operator
serviceMonitor:
# -- Enable or disable api-server serviceMonitor
enabled: false
# -- @param serviceMonitor.interval interval at which metrics should be scraped
interval: 15s
# -- @param serviceMonitor.path path of the metrics endpoint
path: /dolphinscheduler/actuator/prometheus
# -- @param serviceMonitor.labels ServiceMonitor extra labels
labels: {}
# -- @param serviceMonitor.annotations ServiceMonitor annotations
annotations: {}
env: env:
JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m" JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m"

Loading…
Cancel
Save