feat: add heartbeat metrics (#16267)

Signed-off-by: Gallardot <gallardot@apache.org>

dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java

@@ -39,6 +39,11 @@ public class AlertServerMetrics {
                     .description("Alert failure count")
                     .register(Metrics.globalRegistry);
 
+    private final Counter alertHeartBeatCounter =
+            Counter.builder("ds.alert.heartbeat.count")
+                    .description("alert heartbeat count")
+                    .register(Metrics.globalRegistry);
+
     public void registerPendingAlertGauge(final Supplier<Number> supplier) {
         Gauge.builder("ds.alert.pending", supplier)
                 .description("Number of pending alert")
@@ -65,4 +70,7 @@ public class AlertServerMetrics {
         alertFailCounter.increment();
     }
 
+    public void incAlertHeartbeatCount() {
+        alertHeartBeatCounter.increment();
+    }
 }

dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java

@@ -18,6 +18,7 @@
 package org.apache.dolphinscheduler.alert.registry;
 
 import org.apache.dolphinscheduler.alert.config.AlertConfig;
+import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics;
 import org.apache.dolphinscheduler.alert.service.AlertHAServer;
 import org.apache.dolphinscheduler.common.enums.ServerStatus;
 import org.apache.dolphinscheduler.common.model.AlertServerHeartBeat;
@@ -85,6 +86,7 @@ public class AlertHeartbeatTask extends BaseHeartBeatTask<AlertServerHeartBeat>
     public void writeHeartBeat(AlertServerHeartBeat heartBeat) {
         String heartBeatJson = JSONUtils.toJsonString(heartBeat);
         registryClient.persistEphemeral(heartBeatPath, heartBeatJson);
+        AlertServerMetrics.incAlertHeartbeatCount();
         log.debug("Success write master heartBeatInfo into registry, masterRegistryPath: {}, heartBeatInfo: {}",
                 heartBeatPath, heartBeatJson);
     }

dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java

@@ -43,6 +43,11 @@ public class MasterServerMetrics {
                     .description("Master server consume command count")
                     .register(Metrics.globalRegistry);
 
+    private final Counter masterHeartBeatCounter =
+            Counter.builder("ds.master.heartbeat.count")
+                    .description("master heartbeat count")
+                    .register(Metrics.globalRegistry);
+
     public void registerMasterMemoryAvailableGauge(Supplier<Number> supplier) {
         Gauge.builder("ds.master.memory.available", supplier)
                 .description("Master memory available")
@@ -75,4 +80,7 @@ public class MasterServerMetrics {
         masterConsumeCommandCounter.increment(commandCount);
     }
 
+    public void incMasterHeartbeatCount() {
+        masterHeartBeatCounter.increment();
+    }
 }

dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java

@@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.meter.metrics.SystemMetrics;
 import org.apache.dolphinscheduler.registry.api.RegistryClient;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.config.MasterServerLoadProtection;
+import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics;
 
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
@@ -80,6 +81,7 @@ public class MasterHeartBeatTask extends BaseHeartBeatTask<MasterHeartBeat> {
     public void writeHeartBeat(MasterHeartBeat masterHeartBeat) {
         String masterHeartBeatJson = JSONUtils.toJsonString(masterHeartBeat);
         registryClient.persistEphemeral(heartBeatPath, masterHeartBeatJson);
+        MasterServerMetrics.incMasterHeartbeatCount();
         log.debug("Success write master heartBeatInfo into registry, masterRegistryPath: {}, heartBeatInfo: {}",
                 heartBeatPath, masterHeartBeatJson);
     }

dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java

@@ -52,6 +52,11 @@ public class WorkerServerMetrics {
                     .description("worker resource download failure count")
                     .register(Metrics.globalRegistry);
 
+    private final Counter workerHeartBeatCounter =
+            Counter.builder("ds.worker.heartbeat.count")
+                    .description("worker heartbeat count")
+                    .register(Metrics.globalRegistry);
+
     private final Timer workerResourceDownloadDurationTimer =
             Timer.builder("ds.worker.resource.download.duration")
                     .publishPercentiles(0.5, 0.75, 0.95, 0.99)
@@ -83,6 +88,10 @@ public class WorkerServerMetrics {
         workerResourceDownloadFailCounter.increment();
     }
 
+    public void incWorkerHeartbeatCount() {
+        workerHeartBeatCounter.increment();
+    }
+
     public void recordWorkerResourceDownloadTime(final long milliseconds) {
         workerResourceDownloadDurationTimer.record(milliseconds, TimeUnit.MILLISECONDS);
     }

dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java

@@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.meter.metrics.SystemMetrics;
 import org.apache.dolphinscheduler.registry.api.RegistryClient;
 import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
 import org.apache.dolphinscheduler.server.worker.config.WorkerServerLoadProtection;
+import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics;
 import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecutorThreadPool;
 
 import lombok.NonNull;
@@ -85,6 +86,7 @@ public class WorkerHeartBeatTask extends BaseHeartBeatTask<WorkerHeartBeat> {
         String workerHeartBeatJson = JSONUtils.toJsonString(workerHeartBeat);
         String workerRegistryPath = workerConfig.getWorkerRegistryPath();
         registryClient.persistEphemeral(workerRegistryPath, workerHeartBeatJson);
+        WorkerServerMetrics.incWorkerHeartbeatCount();
         log.debug(
                 "Success write worker group heartBeatInfo into registry, workerRegistryPath: {} workerHeartBeatInfo: {}",
                 workerRegistryPath, workerHeartBeatJson);