Browse Source
* Support alert server cluster * Remove WORKER_ALERT_LISTEN_HOST in statefulset-dolphinscheduler-worker.yaml3.2.0-release
Wenjun Ruan
2 years ago
committed by
GitHub
51 changed files with 589 additions and 344 deletions
@ -0,0 +1,68 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.alert.config; |
||||
|
||||
import org.apache.dolphinscheduler.common.utils.NetUtils; |
||||
|
||||
import java.time.Duration; |
||||
|
||||
import lombok.Data; |
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties; |
||||
import org.springframework.stereotype.Component; |
||||
import org.springframework.validation.Errors; |
||||
import org.springframework.validation.Validator; |
||||
|
||||
@Slf4j |
||||
@Data |
||||
@Component |
||||
@ConfigurationProperties("alert") |
||||
public final class AlertConfig implements Validator { |
||||
|
||||
private int port; |
||||
|
||||
private int waitTimeout; |
||||
|
||||
private Duration heartbeatInterval = Duration.ofSeconds(60); |
||||
|
||||
private String alertServerAddress; |
||||
|
||||
@Override |
||||
public boolean supports(Class<?> clazz) { |
||||
return AlertConfig.class.isAssignableFrom(clazz); |
||||
} |
||||
|
||||
@Override |
||||
public void validate(Object target, Errors errors) { |
||||
AlertConfig alertConfig = (AlertConfig) target; |
||||
|
||||
if (heartbeatInterval.getSeconds() <= 0) { |
||||
errors.rejectValue("heartbeat-interval", null, "should be a valid duration"); |
||||
} |
||||
|
||||
alertConfig.setAlertServerAddress(NetUtils.getAddr(port)); |
||||
printConfig(); |
||||
} |
||||
|
||||
private void printConfig() { |
||||
log.info("Alert config: port -> {}", port); |
||||
log.info("Alert config: alertServerAddress -> {}", alertServerAddress); |
||||
log.info("Alert config: heartbeatInterval -> {}", heartbeatInterval); |
||||
} |
||||
} |
||||
@ -0,0 +1,74 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.alert.registry; |
||||
|
||||
import org.apache.dolphinscheduler.alert.config.AlertConfig; |
||||
import org.apache.dolphinscheduler.common.model.AlertServerHeartBeat; |
||||
import org.apache.dolphinscheduler.common.model.BaseHeartBeatTask; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.OSUtils; |
||||
import org.apache.dolphinscheduler.registry.api.RegistryClient; |
||||
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.springframework.stereotype.Component; |
||||
|
||||
@Slf4j |
||||
@Component |
||||
public class AlertHeartbeatTask extends BaseHeartBeatTask<AlertServerHeartBeat> { |
||||
|
||||
private final AlertConfig alertConfig; |
||||
private final Integer processId; |
||||
private final RegistryClient registryClient; |
||||
private final String heartBeatPath; |
||||
private final long startupTime; |
||||
|
||||
public AlertHeartbeatTask(AlertConfig alertConfig, |
||||
RegistryClient registryClient) { |
||||
super("AlertHeartbeatTask", alertConfig.getHeartbeatInterval().toMillis()); |
||||
this.startupTime = System.currentTimeMillis(); |
||||
this.alertConfig = alertConfig; |
||||
this.registryClient = registryClient; |
||||
this.heartBeatPath = |
||||
RegistryNodeType.ALERT_SERVER.getRegistryPath() + "/" + alertConfig.getAlertServerAddress(); |
||||
this.processId = OSUtils.getProcessID(); |
||||
} |
||||
|
||||
@Override |
||||
public AlertServerHeartBeat getHeartBeat() { |
||||
return AlertServerHeartBeat.builder() |
||||
.processId(processId) |
||||
.startupTime(startupTime) |
||||
.reportTime(System.currentTimeMillis()) |
||||
.cpuUsage(OSUtils.cpuUsage()) |
||||
.memoryUsage(OSUtils.memoryUsage()) |
||||
.loadAverage(OSUtils.loadAverage()) |
||||
.availablePhysicalMemorySize(OSUtils.availablePhysicalMemorySize()) |
||||
.alertServerAddress(alertConfig.getAlertServerAddress()) |
||||
.build(); |
||||
} |
||||
|
||||
@Override |
||||
public void writeHeartBeat(AlertServerHeartBeat heartBeat) { |
||||
String heartBeatJson = JSONUtils.toJsonString(heartBeat); |
||||
registryClient.persistEphemeral(heartBeatPath, heartBeatJson); |
||||
log.debug("Success write master heartBeatInfo into registry, masterRegistryPath: {}, heartBeatInfo: {}", |
||||
heartBeatPath, heartBeatJson); |
||||
} |
||||
} |
||||
@ -0,0 +1,57 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.alert.registry; |
||||
|
||||
import org.apache.dolphinscheduler.alert.config.AlertConfig; |
||||
import org.apache.dolphinscheduler.registry.api.RegistryClient; |
||||
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Service; |
||||
|
||||
@Slf4j |
||||
@Service |
||||
public class AlertRegistryClient implements AutoCloseable { |
||||
|
||||
@Autowired |
||||
private RegistryClient registryClient; |
||||
|
||||
@Autowired |
||||
private AlertConfig alertConfig; |
||||
|
||||
private AlertHeartbeatTask alertHeartbeatTask; |
||||
|
||||
public void start() { |
||||
log.info("AlertRegistryClient starting..."); |
||||
registryClient.getLock(RegistryNodeType.ALERT_LOCK.getRegistryPath()); |
||||
alertHeartbeatTask = new AlertHeartbeatTask(alertConfig, registryClient); |
||||
alertHeartbeatTask.start(); |
||||
// start heartbeat task
|
||||
log.info("AlertRegistryClient started..."); |
||||
} |
||||
|
||||
@Override |
||||
public void close() { |
||||
log.info("AlertRegistryClient closing..."); |
||||
alertHeartbeatTask.shutdown(); |
||||
registryClient.releaseLock(RegistryNodeType.ALERT_LOCK.getRegistryPath()); |
||||
log.info("AlertRegistryClient closed..."); |
||||
} |
||||
} |
||||
@ -0,0 +1,60 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.alert.rpc; |
||||
|
||||
import org.apache.dolphinscheduler.alert.config.AlertConfig; |
||||
import org.apache.dolphinscheduler.remote.NettyRemotingServer; |
||||
import org.apache.dolphinscheduler.remote.factory.NettyRemotingServerFactory; |
||||
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; |
||||
|
||||
import java.util.List; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Service; |
||||
|
||||
@Slf4j |
||||
@Service |
||||
public class AlertRpcServer implements AutoCloseable { |
||||
|
||||
@Autowired |
||||
private List<NettyRequestProcessor> nettyRequestProcessors; |
||||
@Autowired |
||||
private AlertConfig alertConfig; |
||||
|
||||
private NettyRemotingServer nettyRemotingServer; |
||||
|
||||
public void start() { |
||||
log.info("Starting alert rpc server..."); |
||||
nettyRemotingServer = NettyRemotingServerFactory.buildNettyRemotingServer(alertConfig.getPort()); |
||||
for (NettyRequestProcessor nettyRequestProcessor : nettyRequestProcessors) { |
||||
nettyRemotingServer.registerProcessor(nettyRequestProcessor); |
||||
log.info("Success register netty processor: {}", nettyRequestProcessor.getClass().getName()); |
||||
} |
||||
nettyRemotingServer.start(); |
||||
log.info("Started alert rpc server..."); |
||||
} |
||||
|
||||
@Override |
||||
public void close() throws Exception { |
||||
log.info("Closing alert rpc server..."); |
||||
nettyRemotingServer.close(); |
||||
log.info("Closed alert rpc server..."); |
||||
} |
||||
} |
||||
@ -1,51 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.alert; |
||||
|
||||
import static org.mockito.ArgumentMatchers.any; |
||||
|
||||
import org.apache.dolphinscheduler.dao.PluginDao; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.Test; |
||||
import org.junit.jupiter.api.extension.ExtendWith; |
||||
import org.mockito.InjectMocks; |
||||
import org.mockito.Mock; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.jupiter.MockitoExtension; |
||||
|
||||
@ExtendWith(MockitoExtension.class) |
||||
public class AlertPluginManagerTest { |
||||
|
||||
@Mock |
||||
private PluginDao pluginDao; |
||||
|
||||
@InjectMocks |
||||
private AlertPluginManager alertPluginManager; |
||||
|
||||
@Test |
||||
public void testAlertPluginManager() { |
||||
Mockito.when(pluginDao.addOrUpdatePluginDefine(any())).thenReturn(0); |
||||
|
||||
alertPluginManager.installPlugin(null); |
||||
|
||||
Assertions.assertEquals(1, alertPluginManager.size()); |
||||
|
||||
Assertions.assertNotNull(alertPluginManager.getAlertChannel(0)); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue