Wenjun Ruan
10 months ago
committed by
GitHub
49 changed files with 875 additions and 526 deletions
@ -0,0 +1,73 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.config; |
||||
|
||||
import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; |
||||
|
||||
import lombok.AllArgsConstructor; |
||||
import lombok.Data; |
||||
import lombok.NoArgsConstructor; |
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
@Slf4j |
||||
@Data |
||||
@NoArgsConstructor |
||||
@AllArgsConstructor |
||||
public class MasterServerLoadProtection { |
||||
|
||||
private boolean enabled = true; |
||||
|
||||
private double maxCpuUsagePercentageThresholds = 0.7; |
||||
|
||||
private double maxJVMMemoryUsagePercentageThresholds = 0.7; |
||||
|
||||
private double maxSystemMemoryUsagePercentageThresholds = 0.7; |
||||
|
||||
private double maxDiskUsagePercentageThresholds = 0.7; |
||||
|
||||
public boolean isOverload(SystemMetrics systemMetrics) { |
||||
if (!enabled) { |
||||
return false; |
||||
} |
||||
if (systemMetrics.getTotalCpuUsedPercentage() > maxCpuUsagePercentageThresholds) { |
||||
log.info( |
||||
"Master OverLoad: the TotalCpuUsedPercentage: {} is over then the MaxCpuUsagePercentageThresholds {}", |
||||
systemMetrics.getTotalCpuUsedPercentage(), maxCpuUsagePercentageThresholds); |
||||
return true; |
||||
} |
||||
if (systemMetrics.getJvmMemoryUsedPercentage() > maxJVMMemoryUsagePercentageThresholds) { |
||||
log.info( |
||||
"Master OverLoad: the JvmMemoryUsedPercentage: {} is over then the MaxJVMMemoryUsagePercentageThresholds {}", |
||||
systemMetrics.getJvmMemoryUsedPercentage(), maxCpuUsagePercentageThresholds); |
||||
return true; |
||||
} |
||||
if (systemMetrics.getDiskUsedPercentage() > maxDiskUsagePercentageThresholds) { |
||||
log.info("Master OverLoad: the DiskUsedPercentage: {} is over then the MaxDiskUsagePercentageThresholds {}", |
||||
systemMetrics.getDiskUsedPercentage(), maxCpuUsagePercentageThresholds); |
||||
return true; |
||||
} |
||||
if (systemMetrics.getSystemMemoryUsedPercentage() > maxSystemMemoryUsagePercentageThresholds) { |
||||
log.info( |
||||
"Master OverLoad: the SystemMemoryUsedPercentage: {} is over then the MaxSystemMemoryUsagePercentageThresholds {}", |
||||
systemMetrics.getSystemMemoryUsedPercentage(), maxSystemMemoryUsagePercentageThresholds); |
||||
return true; |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,42 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.config; |
||||
|
||||
import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
class MasterServerLoadProtectionTest { |
||||
|
||||
@Test |
||||
void isOverload() { |
||||
MasterServerLoadProtection masterServerLoadProtection = new MasterServerLoadProtection(); |
||||
SystemMetrics systemMetrics = SystemMetrics.builder() |
||||
.jvmMemoryUsedPercentage(0.71) |
||||
.systemMemoryUsedPercentage(0.71) |
||||
.totalCpuUsedPercentage(0.71) |
||||
.diskUsedPercentage(0.71) |
||||
.build(); |
||||
masterServerLoadProtection.setEnabled(false); |
||||
Assertions.assertFalse(masterServerLoadProtection.isOverload(systemMetrics)); |
||||
|
||||
masterServerLoadProtection.setEnabled(true); |
||||
Assertions.assertTrue(masterServerLoadProtection.isOverload(systemMetrics)); |
||||
} |
||||
} |
||||
@ -0,0 +1,69 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.meter.metrics; |
||||
|
||||
import org.apache.dolphinscheduler.common.utils.OSUtils; |
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
import io.micrometer.core.instrument.MeterRegistry; |
||||
|
||||
@Component |
||||
public class DefaultMetricsProvider implements MetricsProvider { |
||||
|
||||
@Autowired |
||||
private MeterRegistry meterRegistry; |
||||
|
||||
private SystemMetrics systemMetrics; |
||||
|
||||
private long lastRefreshTime = 0; |
||||
|
||||
private static final long SYSTEM_METRICS_REFRESH_INTERVAL = 1_000L; |
||||
|
||||
@Override |
||||
public SystemMetrics getSystemMetrics() { |
||||
if (System.currentTimeMillis() - lastRefreshTime < SYSTEM_METRICS_REFRESH_INTERVAL) { |
||||
return systemMetrics; |
||||
} |
||||
|
||||
double systemCpuUsage = meterRegistry.get("system.cpu.usage").gauge().value(); |
||||
double processCpuUsage = meterRegistry.get("process.cpu.usage").gauge().value(); |
||||
|
||||
double jvmMemoryUsed = meterRegistry.get("jvm.memory.used").meter().measure().iterator().next().getValue(); |
||||
double jvmMemoryMax = meterRegistry.get("jvm.memory.max").meter().measure().iterator().next().getValue(); |
||||
|
||||
long totalSystemMemory = OSUtils.getTotalSystemMemory(); |
||||
long systemMemoryAvailable = OSUtils.getSystemAvailableMemoryUsed(); |
||||
|
||||
systemMetrics = SystemMetrics.builder() |
||||
.systemCpuUsagePercentage(systemCpuUsage) |
||||
.processCpuUsagePercentage(processCpuUsage) |
||||
.totalCpuUsedPercentage(systemCpuUsage + processCpuUsage) |
||||
.jvmMemoryUsed(jvmMemoryUsed) |
||||
.jvmMemoryMax(jvmMemoryMax) |
||||
.jvmMemoryUsedPercentage(jvmMemoryUsed / jvmMemoryMax) |
||||
.systemMemoryUsed(totalSystemMemory - systemMemoryAvailable) |
||||
.systemMemoryMax(totalSystemMemory) |
||||
.systemMemoryUsedPercentage((double) (totalSystemMemory - systemMemoryAvailable) / totalSystemMemory) |
||||
.build(); |
||||
lastRefreshTime = System.currentTimeMillis(); |
||||
return systemMetrics; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,24 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.meter.metrics; |
||||
|
||||
public interface MetricsProvider { |
||||
|
||||
SystemMetrics getSystemMetrics(); |
||||
|
||||
} |
||||
@ -0,0 +1,54 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.meter.metrics; |
||||
|
||||
import lombok.AllArgsConstructor; |
||||
import lombok.Builder; |
||||
import lombok.Data; |
||||
import lombok.NoArgsConstructor; |
||||
|
||||
@Data |
||||
@Builder |
||||
@NoArgsConstructor |
||||
@AllArgsConstructor |
||||
public class SystemMetrics { |
||||
|
||||
// CPU
|
||||
private double systemCpuUsagePercentage; |
||||
private double processCpuUsagePercentage; |
||||
private double totalCpuUsedPercentage; |
||||
|
||||
// JVM-Memory
|
||||
// todo: get pod memory usage
|
||||
private double jvmMemoryUsed; |
||||
private double jvmMemoryMax; |
||||
private double jvmMemoryUsedPercentage; |
||||
|
||||
// System-Memory
|
||||
// todo: get pod cpu usage
|
||||
private double systemMemoryUsed; |
||||
private double systemMemoryMax; |
||||
private double systemMemoryUsedPercentage; |
||||
|
||||
// Disk
|
||||
// todo: get pod disk usage
|
||||
private double diskUsed; |
||||
private double diskTotal; |
||||
private double diskUsedPercentage; |
||||
|
||||
} |
||||
@ -0,0 +1,73 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.config; |
||||
|
||||
import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; |
||||
|
||||
import lombok.AllArgsConstructor; |
||||
import lombok.Data; |
||||
import lombok.NoArgsConstructor; |
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
@Data |
||||
@Slf4j |
||||
@NoArgsConstructor |
||||
@AllArgsConstructor |
||||
public class WorkerServerLoadProtection { |
||||
|
||||
private boolean enabled = true; |
||||
|
||||
private double maxCpuUsagePercentageThresholds = 0.7; |
||||
|
||||
private double maxJVMMemoryUsagePercentageThresholds = 0.7; |
||||
|
||||
private double maxSystemMemoryUsagePercentageThresholds = 0.7; |
||||
|
||||
private double maxDiskUsagePercentageThresholds = 0.7; |
||||
|
||||
public boolean isOverload(SystemMetrics systemMetrics) { |
||||
if (!enabled) { |
||||
return false; |
||||
} |
||||
if (systemMetrics.getTotalCpuUsedPercentage() > maxCpuUsagePercentageThresholds) { |
||||
log.info( |
||||
"Worker OverLoad: the TotalCpuUsedPercentage: {} is over then the MaxCpuUsagePercentageThresholds {}", |
||||
systemMetrics.getTotalCpuUsedPercentage(), maxCpuUsagePercentageThresholds); |
||||
return true; |
||||
} |
||||
if (systemMetrics.getJvmMemoryUsedPercentage() > maxJVMMemoryUsagePercentageThresholds) { |
||||
log.info( |
||||
"Worker OverLoad: the JvmMemoryUsedPercentage: {} is over then the maxCpuUsagePercentageThresholds {}", |
||||
systemMetrics.getJvmMemoryUsedPercentage(), maxJVMMemoryUsagePercentageThresholds); |
||||
return true; |
||||
} |
||||
if (systemMetrics.getDiskUsedPercentage() > maxDiskUsagePercentageThresholds) { |
||||
log.info("Worker OverLoad: the DiskUsedPercentage: {} is over then the MaxCpuUsagePercentageThresholds {}", |
||||
systemMetrics.getDiskUsedPercentage(), maxDiskUsagePercentageThresholds); |
||||
return true; |
||||
} |
||||
if (systemMetrics.getSystemMemoryUsedPercentage() > maxSystemMemoryUsagePercentageThresholds) { |
||||
log.info( |
||||
"Worker OverLoad: the SystemMemoryUsedPercentage: {} is over then the MaxSystemMemoryUsagePercentageThresholds {}", |
||||
systemMetrics.getSystemMemoryUsedPercentage(), maxSystemMemoryUsagePercentageThresholds); |
||||
return true; |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,42 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.config; |
||||
|
||||
import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
class WorkerServerLoadProtectionTest { |
||||
|
||||
@Test |
||||
void isOverload() { |
||||
WorkerServerLoadProtection workerServerLoadProtection = new WorkerServerLoadProtection(); |
||||
SystemMetrics systemMetrics = SystemMetrics.builder() |
||||
.jvmMemoryUsedPercentage(0.71) |
||||
.systemMemoryUsedPercentage(0.71) |
||||
.totalCpuUsedPercentage(0.71) |
||||
.diskUsedPercentage(0.71) |
||||
.build(); |
||||
workerServerLoadProtection.setEnabled(false); |
||||
Assertions.assertFalse(workerServerLoadProtection.isOverload(systemMetrics)); |
||||
|
||||
workerServerLoadProtection.setEnabled(true); |
||||
Assertions.assertTrue(workerServerLoadProtection.isOverload(systemMetrics)); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue