Browse Source

feat: add heartbeat metrics (#16267)

Signed-off-by: Gallardot <gallardot@apache.org>
dev
Gallardot 4 months ago committed by GitHub
parent
commit
97d164cda7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 8
      dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java
  2. 2
      dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java
  3. 8
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java
  4. 2
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java
  5. 9
      dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java
  6. 2
      dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java

8
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertServerMetrics.java

@ -39,6 +39,11 @@ public class AlertServerMetrics {
.description("Alert failure count") .description("Alert failure count")
.register(Metrics.globalRegistry); .register(Metrics.globalRegistry);
private final Counter alertHeartBeatCounter =
Counter.builder("ds.alert.heartbeat.count")
.description("alert heartbeat count")
.register(Metrics.globalRegistry);
public void registerPendingAlertGauge(final Supplier<Number> supplier) { public void registerPendingAlertGauge(final Supplier<Number> supplier) {
Gauge.builder("ds.alert.pending", supplier) Gauge.builder("ds.alert.pending", supplier)
.description("Number of pending alert") .description("Number of pending alert")
@ -65,4 +70,7 @@ public class AlertServerMetrics {
alertFailCounter.increment(); alertFailCounter.increment();
} }
public void incAlertHeartbeatCount() {
alertHeartBeatCounter.increment();
}
} }

2
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertHeartbeatTask.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.alert.registry; package org.apache.dolphinscheduler.alert.registry;
import org.apache.dolphinscheduler.alert.config.AlertConfig; import org.apache.dolphinscheduler.alert.config.AlertConfig;
import org.apache.dolphinscheduler.alert.metrics.AlertServerMetrics;
import org.apache.dolphinscheduler.alert.service.AlertHAServer; import org.apache.dolphinscheduler.alert.service.AlertHAServer;
import org.apache.dolphinscheduler.common.enums.ServerStatus; import org.apache.dolphinscheduler.common.enums.ServerStatus;
import org.apache.dolphinscheduler.common.model.AlertServerHeartBeat; import org.apache.dolphinscheduler.common.model.AlertServerHeartBeat;
@ -85,6 +86,7 @@ public class AlertHeartbeatTask extends BaseHeartBeatTask<AlertServerHeartBeat>
public void writeHeartBeat(AlertServerHeartBeat heartBeat) { public void writeHeartBeat(AlertServerHeartBeat heartBeat) {
String heartBeatJson = JSONUtils.toJsonString(heartBeat); String heartBeatJson = JSONUtils.toJsonString(heartBeat);
registryClient.persistEphemeral(heartBeatPath, heartBeatJson); registryClient.persistEphemeral(heartBeatPath, heartBeatJson);
AlertServerMetrics.incAlertHeartbeatCount();
log.debug("Success write master heartBeatInfo into registry, masterRegistryPath: {}, heartBeatInfo: {}", log.debug("Success write master heartBeatInfo into registry, masterRegistryPath: {}, heartBeatInfo: {}",
heartBeatPath, heartBeatJson); heartBeatPath, heartBeatJson);
} }

8
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java

@ -43,6 +43,11 @@ public class MasterServerMetrics {
.description("Master server consume command count") .description("Master server consume command count")
.register(Metrics.globalRegistry); .register(Metrics.globalRegistry);
private final Counter masterHeartBeatCounter =
Counter.builder("ds.master.heartbeat.count")
.description("master heartbeat count")
.register(Metrics.globalRegistry);
public void registerMasterMemoryAvailableGauge(Supplier<Number> supplier) { public void registerMasterMemoryAvailableGauge(Supplier<Number> supplier) {
Gauge.builder("ds.master.memory.available", supplier) Gauge.builder("ds.master.memory.available", supplier)
.description("Master memory available") .description("Master memory available")
@ -75,4 +80,7 @@ public class MasterServerMetrics {
masterConsumeCommandCounter.increment(commandCount); masterConsumeCommandCounter.increment(commandCount);
} }
public void incMasterHeartbeatCount() {
masterHeartBeatCounter.increment();
}
} }

2
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java

@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.meter.metrics.SystemMetrics;
import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.RegistryClient;
import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.config.MasterServerLoadProtection; import org.apache.dolphinscheduler.server.master.config.MasterServerLoadProtection;
import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics;
import lombok.NonNull; import lombok.NonNull;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -80,6 +81,7 @@ public class MasterHeartBeatTask extends BaseHeartBeatTask<MasterHeartBeat> {
public void writeHeartBeat(MasterHeartBeat masterHeartBeat) { public void writeHeartBeat(MasterHeartBeat masterHeartBeat) {
String masterHeartBeatJson = JSONUtils.toJsonString(masterHeartBeat); String masterHeartBeatJson = JSONUtils.toJsonString(masterHeartBeat);
registryClient.persistEphemeral(heartBeatPath, masterHeartBeatJson); registryClient.persistEphemeral(heartBeatPath, masterHeartBeatJson);
MasterServerMetrics.incMasterHeartbeatCount();
log.debug("Success write master heartBeatInfo into registry, masterRegistryPath: {}, heartBeatInfo: {}", log.debug("Success write master heartBeatInfo into registry, masterRegistryPath: {}, heartBeatInfo: {}",
heartBeatPath, masterHeartBeatJson); heartBeatPath, masterHeartBeatJson);
} }

9
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java

@ -52,6 +52,11 @@ public class WorkerServerMetrics {
.description("worker resource download failure count") .description("worker resource download failure count")
.register(Metrics.globalRegistry); .register(Metrics.globalRegistry);
private final Counter workerHeartBeatCounter =
Counter.builder("ds.worker.heartbeat.count")
.description("worker heartbeat count")
.register(Metrics.globalRegistry);
private final Timer workerResourceDownloadDurationTimer = private final Timer workerResourceDownloadDurationTimer =
Timer.builder("ds.worker.resource.download.duration") Timer.builder("ds.worker.resource.download.duration")
.publishPercentiles(0.5, 0.75, 0.95, 0.99) .publishPercentiles(0.5, 0.75, 0.95, 0.99)
@ -83,6 +88,10 @@ public class WorkerServerMetrics {
workerResourceDownloadFailCounter.increment(); workerResourceDownloadFailCounter.increment();
} }
public void incWorkerHeartbeatCount() {
workerHeartBeatCounter.increment();
}
public void recordWorkerResourceDownloadTime(final long milliseconds) { public void recordWorkerResourceDownloadTime(final long milliseconds) {
workerResourceDownloadDurationTimer.record(milliseconds, TimeUnit.MILLISECONDS); workerResourceDownloadDurationTimer.record(milliseconds, TimeUnit.MILLISECONDS);
} }

2
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java

@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.meter.metrics.SystemMetrics;
import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.RegistryClient;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.worker.config.WorkerServerLoadProtection; import org.apache.dolphinscheduler.server.worker.config.WorkerServerLoadProtection;
import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics;
import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecutorThreadPool; import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecutorThreadPool;
import lombok.NonNull; import lombok.NonNull;
@ -85,6 +86,7 @@ public class WorkerHeartBeatTask extends BaseHeartBeatTask<WorkerHeartBeat> {
String workerHeartBeatJson = JSONUtils.toJsonString(workerHeartBeat); String workerHeartBeatJson = JSONUtils.toJsonString(workerHeartBeat);
String workerRegistryPath = workerConfig.getWorkerRegistryPath(); String workerRegistryPath = workerConfig.getWorkerRegistryPath();
registryClient.persistEphemeral(workerRegistryPath, workerHeartBeatJson); registryClient.persistEphemeral(workerRegistryPath, workerHeartBeatJson);
WorkerServerMetrics.incWorkerHeartbeatCount();
log.debug( log.debug(
"Success write worker group heartBeatInfo into registry, workerRegistryPath: {} workerHeartBeatInfo: {}", "Success write worker group heartBeatInfo into registry, workerRegistryPath: {} workerHeartBeatInfo: {}",
workerRegistryPath, workerHeartBeatJson); workerRegistryPath, workerHeartBeatJson);

Loading…
Cancel
Save