diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java index 606834a2e9..dfe819fbfa 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java @@ -39,6 +39,11 @@ public class AlertServerMetrics { .description("Alert failure count") .register(Metrics.globalRegistry); + private final Counter alertHeartBeatCounter = + Counter.builder("ds.alert.heartbeat.count") + .description("alert heartbeat count") + .register(Metrics.globalRegistry); + public void registerPendingAlertGauge(final Supplier supplier) { Gauge.builder("ds.alert.pending", supplier) .description("Number of pending alert") @@ -65,4 +70,7 @@ public class AlertServerMetrics { alertFailCounter.increment(); } + public void incAlertHeartbeatCount() { + alertHeartBeatCounter.increment(); + } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java index a5481fdd49..b8640dd539 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.alert.registry; import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics; import org.apache.dolphinscheduler.alert.service.AlertHAServer; import org.apache.dolphinscheduler.common.enums.ServerStatus; import org.apache.dolphinscheduler.common.model.AlertServerHeartBeat; @@ -85,6 +86,7 @@ public class AlertHeartbeatTask extends BaseHeartBeatTask public void writeHeartBeat(AlertServerHeartBeat heartBeat) { String heartBeatJson = JSONUtils.toJsonString(heartBeat); registryClient.persistEphemeral(heartBeatPath, heartBeatJson); + AlertServerMetrics.incAlertHeartbeatCount(); log.debug("Success write master heartBeatInfo into registry, masterRegistryPath: {}, heartBeatInfo: {}", heartBeatPath, heartBeatJson); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java index 1fc92200df..3b99eebfc9 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java @@ -43,6 +43,11 @@ public class MasterServerMetrics { .description("Master server consume command count") .register(Metrics.globalRegistry); + private final Counter masterHeartBeatCounter = + Counter.builder("ds.master.heartbeat.count") + .description("master heartbeat count") + .register(Metrics.globalRegistry); + public void registerMasterMemoryAvailableGauge(Supplier supplier) { Gauge.builder("ds.master.memory.available", supplier) .description("Master memory available") @@ -75,4 +80,7 @@ public class MasterServerMetrics { masterConsumeCommandCounter.increment(commandCount); } + public void incMasterHeartbeatCount() { + masterHeartBeatCounter.increment(); + } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java index b7b5e7a21e..f6c1e7ee3c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.config.MasterServerLoadProtection; +import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -80,6 +81,7 @@ public class MasterHeartBeatTask extends BaseHeartBeatTask { public void writeHeartBeat(MasterHeartBeat masterHeartBeat) { String masterHeartBeatJson = JSONUtils.toJsonString(masterHeartBeat); registryClient.persistEphemeral(heartBeatPath, masterHeartBeatJson); + MasterServerMetrics.incMasterHeartbeatCount(); log.debug("Success write master heartBeatInfo into registry, masterRegistryPath: {}, heartBeatInfo: {}", heartBeatPath, masterHeartBeatJson); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java index 613e7eaf28..e17db2f346 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java @@ -52,6 +52,11 @@ public class WorkerServerMetrics { .description("worker resource download failure count") .register(Metrics.globalRegistry); + private final Counter workerHeartBeatCounter = + Counter.builder("ds.worker.heartbeat.count") + .description("worker heartbeat count") + .register(Metrics.globalRegistry); + private final Timer workerResourceDownloadDurationTimer = Timer.builder("ds.worker.resource.download.duration") .publishPercentiles(0.5, 0.75, 0.95, 0.99) @@ -83,6 +88,10 @@ public class WorkerServerMetrics { workerResourceDownloadFailCounter.increment(); } + public void incWorkerHeartbeatCount() { + workerHeartBeatCounter.increment(); + } + public void recordWorkerResourceDownloadTime(final long milliseconds) { workerResourceDownloadDurationTimer.record(milliseconds, TimeUnit.MILLISECONDS); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java index 4eefd9df10..62bee33a90 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.config.WorkerServerLoadProtection; +import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecutorThreadPool; import lombok.NonNull; @@ -85,6 +86,7 @@ public class WorkerHeartBeatTask extends BaseHeartBeatTask { String workerHeartBeatJson = JSONUtils.toJsonString(workerHeartBeat); String workerRegistryPath = workerConfig.getWorkerRegistryPath(); registryClient.persistEphemeral(workerRegistryPath, workerHeartBeatJson); + WorkerServerMetrics.incWorkerHeartbeatCount(); log.debug( "Success write worker group heartBeatInfo into registry, workerRegistryPath: {} workerHeartBeatInfo: {}", workerRegistryPath, workerHeartBeatJson);