From 0a388afa672cf876f90433e260b41d951d9cf6d9 Mon Sep 17 00:00:00 2001 From: Kerwin <37063904+zhuangchong@users.noreply.github.com> Date: Thu, 29 Jul 2021 18:41:39 +0800 Subject: [PATCH] add http task timeout & test unit. (#5915) --- .../common/task/http/HttpParameters.java | 27 +++ .../common/task/HttpParametersTest.java | 80 +++++++ .../server/worker/task/http/HttpTask.java | 3 +- .../server/worker/task/http/HttpTaskTest.java | 206 ++++++++++++++++++ .../dag/_source/formModel/tasks/http.vue | 56 ++++- .../src/js/module/i18n/locale/en_US.js | 6 + .../src/js/module/i18n/locale/zh_CN.js | 6 + pom.xml | 3 +- 8 files changed, 382 insertions(+), 5 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java index 54284bd8b0..22e833d75d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.task.http; import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; @@ -55,7 +56,17 @@ public class HttpParameters extends AbstractParameters { */ private String condition; + /** + * Connect Timeout + * Unit: ms + */ + private int connectTimeout ; + /** + * Socket Timeout + * Unit: ms + */ + private int socketTimeout ; @Override public boolean checkParameters() { @@ -106,4 +117,20 @@ public class HttpParameters extends AbstractParameters { public void setCondition(String condition) { this.condition = condition; } + + public int getConnectTimeout() { + return connectTimeout; + } + + public void setConnectTimeout(int connectTimeout) { + this.connectTimeout = connectTimeout; + } + + public int getSocketTimeout() { + return socketTimeout; + } + + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java new file mode 100644 index 0000000000..b8f3fe8af5 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.task; + +import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; +import org.apache.dolphinscheduler.common.enums.HttpMethod; +import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.junit.Assert; +import org.junit.Test; + +/** + * http parameter + */ +public class HttpParametersTest { + + @Test + public void testGenerator(){ + String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; + HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + + Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); + Assert.assertEquals(10000,httpParameters.getSocketTimeout()); + Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl()); + Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod()); + Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition()); + Assert.assertEquals("",httpParameters.getCondition()); + + } + + @Test + public void testCheckParameters(){ + String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; + HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + + Assert.assertTrue( httpParameters.checkParameters()); + Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); + Assert.assertEquals(10000,httpParameters.getSocketTimeout()); + Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl()); + Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod()); + Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition()); + Assert.assertEquals("",httpParameters.getCondition()); + + } + + @Test + public void testCheckValues() { + String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; + HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + + Assert.assertTrue( httpParameters.checkParameters()); + Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); + Assert.assertEquals(10000,httpParameters.getSocketTimeout()); + Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl()); + Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod()); + Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition()); + Assert.assertEquals("",httpParameters.getCondition()); + Assert.assertEquals(0,httpParameters.getLocalParametersMap().size()); + Assert.assertEquals(0,httpParameters.getResourceFilesList().size()); + } + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index ef1ccdd09a..8dc56595c7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -51,7 +51,6 @@ import org.slf4j.Logger; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -303,7 +302,7 @@ public class HttpTask extends AbstractTask { * @return RequestConfig */ private RequestConfig requestConfig() { - return RequestConfig.custom().setSocketTimeout(MAX_CONNECTION_MILLISECONDS).setConnectTimeout(MAX_CONNECTION_MILLISECONDS).build(); + return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout()).setConnectTimeout(httpParameters.getConnectTimeout()).build(); } /** diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java new file mode 100644 index 0000000000..3ece0e81b1 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.task.http; + +import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; +import org.apache.dolphinscheduler.common.enums.HttpMethod; +import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; +import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; + +import java.io.IOException; +import java.util.Date; + +import static org.apache.dolphinscheduler.common.enums.CommandType.START_PROCESS; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(OSUtils.class) +@PowerMockIgnore({"javax.management.*","javax.net.ssl.*"}) +public class HttpTaskTest { + private static final Logger logger = LoggerFactory.getLogger(HttpTaskTest.class); + + private HttpTask httpTask; + + private ProcessService processService; + + private ShellCommandExecutor shellCommandExecutor; + + private ApplicationContext applicationContext; + private TaskExecutionContext taskExecutionContext; + + @Before + public void before() throws Exception { + taskExecutionContext = new TaskExecutionContext(); + + PowerMockito.mockStatic(OSUtils.class); + processService = PowerMockito.mock(ProcessService.class); + shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class); + + applicationContext = PowerMockito.mock(ApplicationContext.class); + SpringApplicationContext springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + PowerMockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + + TaskProps props = new TaskProps(); + props.setExecutePath("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstanceId(1); + props.setTenantCode("1"); + props.setEnvFile(".dolphinscheduler_env.sh"); + props.setTaskStartTime(new Date()); + props.setTaskTimeout(0); + props.setTaskParams( + "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\",\"httpMethod\":\"GET\"," + + "\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"https://github.com/\"," + + "\"connectTimeout\":\"1000\",\"socketTimeout\":\"1000\"}"); + + + taskExecutionContext = Mockito.mock(TaskExecutionContext.class); + Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(props.getTaskParams()); + Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp"); + Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn("1"); + Mockito.when(taskExecutionContext.getTenantCode()).thenReturn("root"); + Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date()); + Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000); + Mockito.when(taskExecutionContext.getLogPath()).thenReturn("/tmp/dx"); + + httpTask = new HttpTask(taskExecutionContext, logger); + httpTask.init(); + + } + + @Test + public void testGetParameters() { + Assert.assertNotNull(httpTask.getParameters()); + } + + + @Test + public void testCheckParameters() { + Assert.assertTrue(httpTask.getParameters().checkParameters()); + } + + + @Test + public void testGenerator(){ + String paramJson = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\"," + + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; + HttpParameters httpParameters = JSON.parseObject(paramJson, HttpParameters.class); + + + Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); + Assert.assertEquals(10000,httpParameters.getSocketTimeout()); + Assert.assertEquals("https://github.com/",httpParameters.getUrl()); + Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod()); + Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition()); + Assert.assertEquals("",httpParameters.getCondition()); + + } + + @Test + public void testHandle(){ + boolean flag = true ; + try { + httpTask.handle(); + } catch (Exception e) { + flag = false ; + e.printStackTrace(); + } + + Assert.assertTrue(flag); + + } + + @Test + public void testSendRequest(){ + + CloseableHttpClient client = httpTask.createHttpClient(); + + String statusCode = null; + String body = null; + + try { + + CloseableHttpResponse response = httpTask.sendRequest(client) ; + statusCode = String.valueOf(httpTask.getStatusCode(response)); + body = httpTask.getResponseBody(response); + int exitStatusCode = httpTask.validResponse(body, statusCode); + + Assert.assertNotEquals(-1,exitStatusCode); + + } catch (IOException e) { + e.printStackTrace(); + }; + } + + @Test + public void testValidResponse(){ + String body = "body"; + String statusCode = "200" ; + + int exitStatusCode = httpTask.validResponse(body,statusCode); + Assert.assertNotEquals(-1,exitStatusCode); + + } + + @Test + public void testAppendMessage(){ + httpTask.appendMessage("message"); + + Assert.assertEquals("message",httpTask.getOutput()); + } + + @Test + public void testCreateHttpClient(){ + Assert.assertNotNull(httpTask.createHttpClient()); + } + + @Test + public void testCreateRequestBuilder(){ + RequestBuilder requestBuilder = httpTask.createRequestBuilder(); + Assert.assertEquals(RequestBuilder.get().getMethod(),requestBuilder.getMethod()); + } + + private ProcessInstance getProcessInstance() { + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setCommandType(START_PROCESS); + processInstance.setScheduleTime(new Date()); + return processInstance; + } +} diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue index 3e2f6efb91..bbf91b3c97 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue @@ -85,6 +85,38 @@ + + +
{{$t('Timeout Settings')}}
+
+ +
+ + +
+
+ {{$t('Connect Timeout')}} +
+
+ + + + {{$t('ms')}} + {{$t('Socket Timeout')}} + + + + {{$t('ms')}} +
+
+
{{$t('Custom Parameters')}}
@@ -110,6 +142,9 @@ name: 'http', data () { return { + timeoutSettings: false, + connectTimeout : 60000, + socketTimeout : 60000, url: '', condition: '', localParams: [], @@ -152,6 +187,14 @@ if (!this.$refs.refHttpParams._verifValue()) { return false } + if (!_.isNumber(parseInt(this.socketTimeout))) { + this.$message.warning(`${i18n.$t('Socket Timeout be a positive integer')}`) + return false + } + if (!_.isNumber(parseInt(this.connectTimeout))) { + this.$message.warning(`${i18n.$t('Connect timeout be a positive integer')}`) + return false + } // storage this.$emit('on-params', { localParams: this.localParams, @@ -159,7 +202,9 @@ url: this.url, httpMethod: this.httpMethod, httpCheckCondition: this.httpCheckCondition, - condition: this.condition + condition: this.condition, + connectTimeout : this.connectTimeout , + socketTimeout : this.socketTimeout }) return true } @@ -172,7 +217,9 @@ url: this.url, httpMethod: this.httpMethod, httpCheckCondition: this.httpCheckCondition, - condition: this.condition + condition: this.condition, + connectTimeout : this.connectTimeout , + socketTimeout : this.socketTimeout } } }, @@ -193,6 +240,11 @@ this.httpMethod = o.params.httpMethod || 'GET' this.httpCheckCondition = o.params.httpCheckCondition || 'DEFAULT' this.condition = o.params.condition || '' + this.connectTimeout = o.params.connectTimeout + this.socketTimeout = o.params.socketTimeout + if(this.connectTimeout != 60000 || this.socketTimeout != 60000 ){ + this.timeoutSettings = true + } // backfill localParams let localParams = o.params.localParams || [] if (localParams.length) { diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 83b5757bf2..796c7f8551 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -547,6 +547,12 @@ export default { 'Http Condition': 'Http Condition', 'Please Enter Http Url': 'Please Enter Http Url(required)', 'Please Enter Http Condition': 'Please Enter Http Condition', + 'Timeout Settings': 'Timeout Settings', + 'Connect Timeout':'Connect Timeout', + 'Socket Timeout':'Socket Timeout', + 'Connect timeout be a positive integer': 'Connect timeout be a positive integer', + 'Socket Timeout be a positive integer': 'Socket Timeout be a positive integer', + 'ms':'ms', 'There is no data for this period of time': 'There is no data for this period of time', 'Worker addresses cannot be empty': 'Worker addresses cannot be empty', 'Please generate token': 'Please generate token', diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index a120953070..285d5db342 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -547,6 +547,12 @@ export default { 'Http Condition': '校验内容', 'Please Enter Http Url': '请填写请求地址(必填)', 'Please Enter Http Condition': '请填写校验内容', + 'Timeout Settings': '超时设置', + 'Connect Timeout':'连接超时', + 'Socket Timeout':'Socket超时', + 'Connect timeout be a positive integer': '连接超时必须为数字', + 'Socket Timeout be a positive integer': 'Socket超时必须为数字', + 'ms':'毫秒', 'There is no data for this period of time': '该时间段无数据', 'Worker addresses cannot be empty': 'Worker地址不能为空', 'Please generate token': '请生成Token', diff --git a/pom.xml b/pom.xml index c5ada2c9b6..65830b69e7 100644 --- a/pom.xml +++ b/pom.xml @@ -757,6 +757,7 @@ **/common/shell/ShellExecutorTest.java **/common/task/EntityTestUtils.java **/common/task/FlinkParametersTest.java + **/common/task/HttpParametersTest.java **/common/task/SqlParametersTest.java **/common/task/SqoopParameterEntityTest.java **/common/threadutils/ThreadPoolExecutorsTest.java @@ -839,7 +840,7 @@ **/server/worker/sql/SqlExecutorTest.java **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/EnvFileTest.java - **/server/worker/task/spark/SparkTaskTest.java + **/server/worker/task/http/HttpTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/worker/EnvFileTest.java **/service/process/ProcessServiceTest.java