Add Server HealthIndicator (#15274)

dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/metrics/AlertHealthIndicator.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.alert.metrics;
+
+import org.apache.dolphinscheduler.alert.registry.AlertRegistryClient;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
+import org.springframework.stereotype.Component;
+
+@Component
+public class AlertHealthIndicator implements HealthIndicator {
+
+    @Autowired
+    private AlertRegistryClient alertRegistryClient;
+
+    @Override
+    public Health health() {
+        try {
+            if (alertRegistryClient.isAvailable()) {
+                return Health.up().build();
+            }
+            return Health.down().build();
+        } catch (Exception ex) {
+            return Health.down().withException(ex).build();
+        }
+    }
+}

dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/registry/AlertRegistryClient.java

@@ -58,4 +58,8 @@ public class AlertRegistryClient implements AutoCloseable {
         registryClient.releaseLock(RegistryNodeType.ALERT_LOCK.getRegistryPath());
         log.info("AlertRegistryClient closed...");
     }
+
+    public boolean isAvailable() {
+        return registryClient.isConnected();
+    }
 }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/metrics/ApiHealthIndicator.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.metrics;
+
+import org.apache.dolphinscheduler.registry.api.RegistryClient;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ApiHealthIndicator implements HealthIndicator {
+
+    @Autowired
+    private RegistryClient registryClient;
+
+    @Override
+    public Health health() {
+        try {
+            if (registryClient.isConnected()) {
+                return Health.up().build();
+            }
+            return Health.down().build();
+        } catch (Exception ex) {
+            return Health.down().withException(ex).build();
+        }
+    }
+
+}

dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterHealthIndicator.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.metrics;
+
+import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MasterHealthIndicator implements HealthIndicator {
+
+    @Autowired
+    private MasterRegistryClient masterRegistryClient;
+
+    @Override
+    public Health health() {
+        try {
+            if (masterRegistryClient.isAvailable()) {
+                return Health.up().build();
+            }
+            return Health.down().build();
+        } catch (Exception ex) {
+            return Health.down().withException(ex).build();
+        }
+    }
+}

dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java

@@ -195,4 +195,7 @@ public class MasterRegistryClient implements AutoCloseable {
         }
     }
 
+    public boolean isAvailable() {
+        return registryClient.isConnected();
+    }
 }

dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java

@@ -30,6 +30,8 @@ import lombok.NonNull;
  */
 public interface Registry extends Closeable {
 
+    boolean isConnected();
+
     /**
      * Connect to the registry, will wait in the given timeout
      *

dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryClient.java

@@ -41,8 +41,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import javax.annotation.PostConstruct;
-
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 
@@ -61,11 +59,14 @@ public class RegistryClient {
 
     public RegistryClient(Registry registry) {
         this.registry = registry;
+        registry.put(RegistryNodeType.MASTER.getRegistryPath(), EMPTY, false);
+        registry.put(RegistryNodeType.WORKER.getRegistryPath(), EMPTY, false);
+        registry.put(RegistryNodeType.ALERT_SERVER.getRegistryPath(), EMPTY, false);
     }
 
-    @PostConstruct
+    public boolean isConnected() {
-    public void afterConstruct() {
+        return registry.isConnected();
-        initNodes();
+
     }
 
     public void connectUntilTimeout(@NonNull Duration duration) throws RegistryException {
@@ -229,12 +230,6 @@ public class RegistryClient {
         }
     }
 
-    private void initNodes() {
-        registry.put(RegistryNodeType.MASTER.getRegistryPath(), EMPTY, false);
-        registry.put(RegistryNodeType.WORKER.getRegistryPath(), EMPTY, false);
-        registry.put(RegistryNodeType.ALERT_SERVER.getRegistryPath(), EMPTY, false);
-    }
-
     private Collection<String> getServerNodes(RegistryNodeType nodeType) {
         return getChildrenKeys(nodeType.getRegistryPath());
     }

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java

@@ -36,7 +36,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
-import javax.annotation.PostConstruct;
 import javax.net.ssl.SSLException;
 
 import lombok.NonNull;
@@ -124,16 +123,16 @@ public class EtcdRegistry implements Registry {
         log.info("Started Etcd Registry...");
         etcdConnectionStateListener = new EtcdConnectionStateListener(client);
         etcdKeepAliveLeaseManager = new EtcdKeepAliveLeaseManager(client);
-    }
 
-    /**
-     * Start the etcd Connection stateListeer
-     */
-    @PostConstruct
-    public void start() {
         log.info("Starting Etcd ConnectionListener...");
         etcdConnectionStateListener.start();
         log.info("Started Etcd ConnectionListener...");
+
+    }
+
+    @Override
+    public boolean isConnected() {
+        return client.getKVClient().get(byteSequence("/")).join() != null;
     }
 
     @Override

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcOperator.java

@@ -44,7 +44,6 @@ public class JdbcOperator {
     private JdbcRegistryDataMapper jdbcRegistryDataMapper;
     @Autowired
     private JdbcRegistryLockMapper jdbcRegistryLockMapper;
-
     private final long expireTimeWindow;
 
     public JdbcOperator(JdbcRegistryProperties registryProperties) {

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistry.java

@@ -75,6 +75,12 @@ public class JdbcRegistry implements Registry {
         log.info("Started Jdbc Registry...");
     }
 
+    @Override
+    public boolean isConnected() {
+        jdbcOperator.healthCheck();
+        return true;
+    }
+
     @Override
     public void connectUntilTimeout(@NonNull Duration timeout) throws RegistryException {
         long beginTimeMillis = System.currentTimeMillis();

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java

@@ -253,6 +253,11 @@ public final class ZookeeperRegistry implements Registry {
         return true;
     }
 
+    @Override
+    public boolean isConnected() {
+        return client.getZookeeperClient().isConnected();
+    }
+
     @Override
     public void close() {
         treeCacheMap.values().forEach(CloseableUtils::closeQuietly);

dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerHealthIndicator.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.metrics;
+
+import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
+import org.springframework.stereotype.Component;
+
+@Component
+public class WorkerHealthIndicator implements HealthIndicator {
+
+    @Autowired
+    private WorkerRegistryClient workerRegistryClient;
+
+    @Override
+    public Health health() {
+        try {
+            if (workerRegistryClient.isAvailable()) {
+                return Health.up().build();
+            }
+            return Health.down().build();
+        } catch (Exception ex) {
+            return Health.down().withException(ex).build();
+        }
+    }
+}

dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java

@@ -136,4 +136,7 @@ public class WorkerRegistryClient implements AutoCloseable {
         log.info("Worker registry client closed");
     }
 
+    public boolean isAvailable() {
+        return registryClient.isConnected();
+    }
 }