From 7104063e0d055d4368ff13006099570fb4c39860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E9=98=B3?= Date: Mon, 20 Jun 2022 18:43:48 +0800 Subject: [PATCH] [Fix]Recovery Traffic Control Config (#10505) * recovery traffic config * run ci * update (cherry picked from commit 6e4b2e69927af70bccda5a2e70fe93d7b1a852e7) --- docs/docs/en/architecture/configuration.md | 8 ++- docs/docs/zh/architecture/configuration.md | 6 +- .../api/configuration/AppConfiguration.java | 2 +- .../configuration/TrafficConfiguration.java | 62 ++++--------------- .../api/interceptor/RateLimitInterceptor.java | 6 +- .../src/main/resources/application.yaml | 15 +++++ .../TrafficConfigurationTest.java | 6 +- .../interceptor/RateLimitInterceptorTest.java | 8 +-- .../src/main/resources/application.yaml | 14 +++++ 9 files changed, 63 insertions(+), 64 deletions(-) diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 85a45d0ab4..b41578438f 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -223,8 +223,14 @@ security.authentication.ldap.urls|ldap://ldap.forumsys.com:389/|LDAP urls security.authentication.ldap.base.dn|dc=example,dc=com|LDAP base dn security.authentication.ldap.username|cn=read-only-admin,dc=example,dc=com|LDAP username security.authentication.ldap.password|password|LDAP password -security.authentication.ldap.user.identity.attribute|uid|LDAP user identity attribute +security.authentication.ldap.user.identity.attribute|uid|LDAP user identity attribute security.authentication.ldap.user.email.attribute|mail|LDAP user email attribute +security.authentication.ldap.user.not-exist-action|CREATE|action when LDAP user is not exist. Default CREATE: automatically create user when user not exist, DENY: deny log-in when user not exist +traffic.control.global.switch|false|traffic control global switch +traffic.control.max-global-qps-rate|300|global max request number per second +traffic.control.tenant-switch|false|traffic control tenant switch +traffic.control.default-tenant-qps-rate|10|default tenant max request number per second +traffic.control.customize-tenant-qps-rate||customize tenant max request number per second ### master.properties [master-service log config] diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 7bcf697f34..cebb8c00ef 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -216,7 +216,11 @@ security.authentication.ldap.username|cn=read-only-admin,dc=example,dc=com|LDAP security.authentication.ldap.password|password|LDAP密码 security.authentication.ldap.user.identity.attribute|uid|LDAP用户身份标识字段名 security.authentication.ldap.user.email.attribute|mail|LDAP邮箱字段名 - +traffic.control.global.switch|false|流量控制全局开关 +traffic.control.max-global-qps-rate|300|全局最大请求数/秒 +traffic.control.tenant-switch|false|流量控制租户开关 +traffic.control.default-tenant-qps-rate|10|默认租户最大请求数/秒限制 +traffic.control.customize-tenant-qps-rate||自定义租户最大请求数/秒限制 ## 6.master.properties [Master服务配置] |参数 |默认值| 描述| diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java index 105e3738df..2a80ef86ef 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java @@ -95,7 +95,7 @@ public class AppConfiguration implements WebMvcConfigurer { public void addInterceptors(InterceptorRegistry registry) { // i18n registry.addInterceptor(localeChangeInterceptor()); - if (trafficConfiguration.isTrafficGlobalControlSwitch() || trafficConfiguration.isTrafficTenantControlSwitch()) { + if (trafficConfiguration.isGlobalSwitch() || trafficConfiguration.isTenantSwitch()) { registry.addInterceptor(createRateLimitInterceptor()); } registry.addInterceptor(loginInterceptor()) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java index 75a9c25410..3b66a2d6ca 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java @@ -17,62 +17,22 @@ package org.apache.dolphinscheduler.api.configuration; +import lombok.Data; + +import java.util.HashMap; import java.util.Map; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; +@Data @Configuration +@ConfigurationProperties(prefix = "traffic.control") public class TrafficConfiguration { - @Value("${traffic.control.global.switch:false}") - private boolean trafficGlobalControlSwitch; - @Value("${traffic.control.max.global.qps.rate:300}") - private Integer maxGlobalQpsRate; - @Value("${traffic.control.tenant.switch:false}") - private boolean trafficTenantControlSwitch; - @Value("${traffic.control.default.tenant.qps.rate:10}") - private Integer defaultTenantQpsRate; - @Value("#{'${traffic.control.customize.tenant.qps.rate:}'.empty?null:'${traffic.control.customize.tenant.qps.rate:}'}") - private Map customizeTenantQpsRate; - - public boolean isTrafficGlobalControlSwitch() { - return trafficGlobalControlSwitch; - } - - public void setTrafficGlobalControlSwitch(boolean trafficGlobalControlSwitch) { - this.trafficGlobalControlSwitch = trafficGlobalControlSwitch; - } - - public Integer getMaxGlobalQpsRate() { - return maxGlobalQpsRate; - } - - public void setMaxGlobalQpsRate(Integer maxGlobalQpsRate) { - this.maxGlobalQpsRate = maxGlobalQpsRate; - } - - public boolean isTrafficTenantControlSwitch() { - return trafficTenantControlSwitch; - } - - public void setTrafficTenantControlSwitch(boolean trafficTenantControlSwitch) { - this.trafficTenantControlSwitch = trafficTenantControlSwitch; - } - - public Integer getDefaultTenantQpsRate() { - return defaultTenantQpsRate; - } - - public void setDefaultTenantQpsRate(Integer defaultTenantQpsRate) { - this.defaultTenantQpsRate = defaultTenantQpsRate; - } - - public Map getCustomizeTenantQpsRate() { - return customizeTenantQpsRate; - } - - public void setCustomizeTenantQpsRate(Map customizeTenantQpsRate) { - this.customizeTenantQpsRate = customizeTenantQpsRate; - } + private boolean globalSwitch; + private Integer maxGlobalQpsRate = 300; + private boolean tenantSwitch; + private Integer defaultTenantQpsRate = 10; + private Map customizeTenantQpsRate = new HashMap<>(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java index 925412576c..9f6ee6fb43 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java @@ -72,7 +72,7 @@ public class RateLimitInterceptor implements HandlerInterceptor { @Override public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws ExecutionException { // tenant-level rate limit - if (trafficConfiguration.isTrafficTenantControlSwitch()) { + if (trafficConfiguration.isTenantSwitch()) { String token = request.getHeader("token"); if (!StringUtils.isEmpty(token)) { RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token); @@ -84,7 +84,7 @@ public class RateLimitInterceptor implements HandlerInterceptor { } } // global rate limit - if (trafficConfiguration.isTrafficGlobalControlSwitch()) { + if (trafficConfiguration.isGlobalSwitch()) { if (!globalRateLimiter.tryAcquire()) { response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); logger.warn("Too many request, reach global rate limit, current qps is {}", globalRateLimiter.getRate()); @@ -96,7 +96,7 @@ public class RateLimitInterceptor implements HandlerInterceptor { public RateLimitInterceptor(TrafficConfiguration trafficConfiguration) { this.trafficConfiguration = trafficConfiguration; - if (trafficConfiguration.isTrafficGlobalControlSwitch()) { + if (trafficConfiguration.isGlobalSwitch()) { this.globalRateLimiter = RateLimiter.create(trafficConfiguration.getMaxGlobalQpsRate(), 1, TimeUnit.SECONDS); } } diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 3fff4539dd..a7f8e306cb 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -153,6 +153,21 @@ security: user.identity.attribute: uid user.email.attribute: mail +# Traffic control, if you turn on this config, the maximum number of request/s will be limited. +# global max request number per second +# default tenant-level max request number +traffic: + control: + global-switch: false + max-global-qps-rate: 300 + tenant-switch: false + default-tenant-qps-rate: 10 + #customize-tenant-qps-rate: + # eg. + #tenant1: 11 + #tenant2: 20 + + # Override by profile --- diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java index bb0f6b79c5..a0c3d8a5a7 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java @@ -32,7 +32,7 @@ public class TrafficConfigurationTest extends AbstractControllerTest { @Test public void isTrafficGlobalControlSwitch() { - Assert.assertFalse(trafficConfiguration.isTrafficGlobalControlSwitch()); + Assert.assertFalse(trafficConfiguration.isGlobalSwitch()); } @Test @@ -42,7 +42,7 @@ public class TrafficConfigurationTest extends AbstractControllerTest { @Test public void isTrafficTenantControlSwitch() { - Assert.assertFalse(trafficConfiguration.isTrafficTenantControlSwitch()); + Assert.assertFalse(trafficConfiguration.isTenantSwitch()); } @Test @@ -54,4 +54,4 @@ public class TrafficConfigurationTest extends AbstractControllerTest { public void getCustomizeTenantQpsRate() { Assert.assertTrue(MapUtils.isEmpty(trafficConfiguration.getCustomizeTenantQpsRate())); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java index 95ae3e3779..c903df5009 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java @@ -48,7 +48,7 @@ public class RateLimitInterceptorTest { @Test public void testPreHandleWithTenantLevenControl() throws ExecutionException { TrafficConfiguration trafficConfiguration = new TrafficConfiguration(); - trafficConfiguration.setTrafficTenantControlSwitch(true); + trafficConfiguration.setTenantSwitch(true); Map map = new HashMap<>(); map.put("tenant1", 2); map.put("tenant2", 2); @@ -72,8 +72,8 @@ public class RateLimitInterceptorTest { @Test public void testPreHandleWithGlobalControl() throws ExecutionException { TrafficConfiguration trafficConfiguration = new TrafficConfiguration(); - trafficConfiguration.setTrafficTenantControlSwitch(true); - trafficConfiguration.setTrafficGlobalControlSwitch(true); + trafficConfiguration.setTenantSwitch(true); + trafficConfiguration.setGlobalSwitch(true); trafficConfiguration.setMaxGlobalQpsRate(3); RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(trafficConfiguration); @@ -86,4 +86,4 @@ public class RateLimitInterceptorTest { Assert.assertFalse(rateLimitInterceptor.preHandle(request, response, null)); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 12cdfe1e0d..cdc55df4c1 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -102,6 +102,20 @@ security: user.identity.attribute: uid user.email.attribute: mail +# Traffic control, if you turn on this config, the maximum number of request/s will be limited. +# global max request number per second +# default tenant-level max request number +traffic: + control: + global-switch: false + max-global-qps-rate: 300 + tenant-switch: false + default-tenant-qps-rate: 10 + #customize-tenant-qps-rate: + # eg. + #tenant1: 11 + #tenant2: 20 + master: listen-port: 5678 # master fetch command num