diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java index 54284bd8b0..7e4cf7453a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java @@ -56,6 +56,17 @@ public class HttpParameters extends AbstractParameters { private String condition; + /** + * Connect Timeout + * Unit: ms + */ + private int connectTimeout ; + + /** + * Socket Timeout + * Unit: ms + */ + private int socketTimeout ; @Override public boolean checkParameters() { @@ -106,4 +117,20 @@ public class HttpParameters extends AbstractParameters { public void setCondition(String condition) { this.condition = condition; } + + public int getConnectTimeout() { + return connectTimeout; + } + + public void setConnectTimeout(int connectTimeout) { + this.connectTimeout = connectTimeout; + } + + public int getSocketTimeout() { + return socketTimeout; + } + + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java new file mode 100644 index 0000000000..abcfca83dd --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.task; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; +import org.apache.dolphinscheduler.common.enums.HttpMethod; +import org.apache.dolphinscheduler.common.process.HttpProperty; +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.junit.Assert; +import org.junit.Test; + +import com.alibaba.fastjson.JSON; + +/** + * http parameter + */ +public class HttpParametersTest { + + + @Test + public void testGenerator(){ + String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; + HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + + + Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); + Assert.assertEquals(10000,httpParameters.getSocketTimeout()); + Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl()); + Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod()); + Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition()); + Assert.assertEquals("",httpParameters.getCondition()); + + } + + + @Test + public void testCheckParameters(){ + String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; + HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + + Assert.assertTrue( httpParameters.checkParameters()); + Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); + Assert.assertEquals(10000,httpParameters.getSocketTimeout()); + Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl()); + Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod()); + Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition()); + Assert.assertEquals("",httpParameters.getCondition()); + + } + + + @Test + public void testCheckValues() { + String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; + HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + + Assert.assertTrue( httpParameters.checkParameters()); + Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); + Assert.assertEquals(10000,httpParameters.getSocketTimeout()); + Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl()); + Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod()); + Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition()); + Assert.assertEquals("",httpParameters.getCondition()); + Assert.assertEquals(0,httpParameters.getLocalParametersMap().size()); + Assert.assertEquals(0,httpParameters.getResourceFilesList().size()); + } + + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index ef1ccdd09a..130137da29 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -66,13 +66,6 @@ public class HttpTask extends AbstractTask { */ private HttpParameters httpParameters; - - - /** - * Convert mill seconds to second unit - */ - protected static final int MAX_CONNECTION_MILLISECONDS = 60000; - /** * application json */ @@ -303,7 +296,7 @@ public class HttpTask extends AbstractTask { * @return RequestConfig */ private RequestConfig requestConfig() { - return RequestConfig.custom().setSocketTimeout(MAX_CONNECTION_MILLISECONDS).setConnectTimeout(MAX_CONNECTION_MILLISECONDS).build(); + return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout()).setConnectTimeout(httpParameters.getConnectTimeout()).build(); } /** diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java new file mode 100644 index 0000000000..e50a54a372 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.worker.task.http; + +import static org.apache.dolphinscheduler.common.enums.CommandType.*; + +import java.io.IOException; +import java.util.Date; + +import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; +import org.apache.dolphinscheduler.common.enums.HttpMethod; +import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; +import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; + +import com.alibaba.fastjson.JSON; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(OSUtils.class) +@PowerMockIgnore({"javax.management.*","javax.net.ssl.*"}) +public class HttpTaskTest { + private static final Logger logger = LoggerFactory.getLogger(HttpTaskTest.class); + + + + private HttpTask httpTask; + + private ProcessService processService; + + private ShellCommandExecutor shellCommandExecutor; + + private ApplicationContext applicationContext; + private TaskExecutionContext taskExecutionContext; + + @Before + public void before() throws Exception { + taskExecutionContext = new TaskExecutionContext(); + + PowerMockito.mockStatic(OSUtils.class); + processService = PowerMockito.mock(ProcessService.class); + shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class); + + applicationContext = PowerMockito.mock(ApplicationContext.class); + SpringApplicationContext springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + PowerMockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + + TaskProps props = new TaskProps(); + props.setExecutePath("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstanceId(1); + props.setTenantCode("1"); + props.setEnvFile(".dolphinscheduler_env.sh"); + props.setTaskStartTime(new Date()); + props.setTaskTimeout(0); + props.setTaskParams( + "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\",\"httpMethod\":\"GET\"," + + "\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"https://github.com/\"," + + "\"connectTimeout\":\"1000\",\"socketTimeout\":\"1000\"}"); + + + taskExecutionContext = Mockito.mock(TaskExecutionContext.class); + Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(props.getTaskParams()); + Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp"); + Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn("1"); + Mockito.when(taskExecutionContext.getTenantCode()).thenReturn("root"); + Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date()); + Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000); + Mockito.when(taskExecutionContext.getLogPath()).thenReturn("/tmp/dx"); + + httpTask = new HttpTask(taskExecutionContext, logger); + httpTask.init(); + + } + + @Test + public void testGetParameters() { + Assert.assertNotNull(httpTask.getParameters()); + } + + + @Test + public void testCheckParameters() { + Assert.assertTrue(httpTask.getParameters().checkParameters()); + } + + + @Test + public void testGenerator(){ + String paramJson = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\"," + + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; + HttpParameters httpParameters = JSON.parseObject(paramJson, HttpParameters.class); + + + Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); + Assert.assertEquals(10000,httpParameters.getSocketTimeout()); + Assert.assertEquals("https://github.com/",httpParameters.getUrl()); + Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod()); + Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition()); + Assert.assertEquals("",httpParameters.getCondition()); + + } + + @Test + public void testHandle(){ + boolean flag = true ; + try { + httpTask.handle(); + } catch (Exception e) { + flag = false ; + e.printStackTrace(); + } + + Assert.assertTrue(flag); + + } + + @Test + public void testSendRequest(){ + + CloseableHttpClient client = httpTask.createHttpClient(); + + String statusCode = null; + String body = null; + + try { + + CloseableHttpResponse response = httpTask.sendRequest(client) ; + statusCode = String.valueOf(httpTask.getStatusCode(response)); + body = httpTask.getResponseBody(response); + int exitStatusCode = httpTask.validResponse(body, statusCode); + + Assert.assertNotEquals(-1,exitStatusCode); + + } catch (IOException e) { + e.printStackTrace(); + }; + } + + @Test + public void testValidResponse(){ + String body = "body"; + String statusCode = "200" ; + + int exitStatusCode = httpTask.validResponse(body,statusCode); + Assert.assertNotEquals(-1,exitStatusCode); + + } + + @Test + public void testAppendMessage(){ + httpTask.appendMessage("message"); + + Assert.assertEquals("message",httpTask.getOutput()); + } + + @Test + public void testCreateHttpClient(){ + Assert.assertNotNull(httpTask.createHttpClient()); + } + + @Test + public void testCreateRequestBuilder(){ + RequestBuilder requestBuilder = httpTask.createRequestBuilder(); + Assert.assertEquals(RequestBuilder.get().getMethod(),requestBuilder.getMethod()); + } + + private ProcessInstance getProcessInstance() { + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setCommandType(START_PROCESS); + processInstance.setScheduleTime(new Date()); + return processInstance; + } +} diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue index 3e2f6efb91..425cb2eb35 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue @@ -85,6 +85,39 @@ + + + +
{{$t('Timeout Settings')}}
+
+ +
+ + +
+
+ {{$t('Connect Timeout')}} +
+
+ + + + {{$t('ms')}} + {{$t('Socket Timeout')}} + + + + {{$t('ms')}} +
+
+
{{$t('Custom Parameters')}}
@@ -110,6 +143,10 @@ name: 'http', data () { return { + timeoutSettings: false, + connectTimeout : 60000 , + socketTimeout : 60000 , + url: '', condition: '', localParams: [], @@ -152,6 +189,14 @@ if (!this.$refs.refHttpParams._verifValue()) { return false } + if (!_.isNumber(parseInt(this.socketTimeout))) { + this.$message.warning(`${i18n.$t('Socket Timeout be a positive integer')}`) + return false + } + if (!_.isNumber(parseInt(this.connectTimeout))) { + this.$message.warning(`${i18n.$t('Connect timeout be a positive integer')}`) + return false + } // storage this.$emit('on-params', { localParams: this.localParams, @@ -159,7 +204,9 @@ url: this.url, httpMethod: this.httpMethod, httpCheckCondition: this.httpCheckCondition, - condition: this.condition + condition: this.condition, + connectTimeout : this.connectTimeout , + socketTimeout : this.socketTimeout }) return true } @@ -172,7 +219,9 @@ url: this.url, httpMethod: this.httpMethod, httpCheckCondition: this.httpCheckCondition, - condition: this.condition + condition: this.condition, + connectTimeout : this.connectTimeout , + socketTimeout : this.socketTimeout } } }, @@ -193,6 +242,11 @@ this.httpMethod = o.params.httpMethod || 'GET' this.httpCheckCondition = o.params.httpCheckCondition || 'DEFAULT' this.condition = o.params.condition || '' + this.connectTimeout = o.params.connectTimeout + this.socketTimeout = o.params.socketTimeout + if(this.connectTimeout != 60000 || this.socketTimeout != 60000 ){ + this.timeoutSettings = true + } // backfill localParams let localParams = o.params.localParams || [] if (localParams.length) { diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index e47b680ea7..c8d82d4246 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -600,5 +600,11 @@ export default { 'Unauthorized or deleted resources': 'Unauthorized or deleted resources', 'Please delete all non-existent resources': 'Please delete all non-existent resources', 'Enable': 'Enable', - 'Disable': 'Disable' + 'Disable': 'Disable', + 'Timeout Settings': 'Timeout Settings', + 'Connect Timeout':'Connect Timeout', + 'Socket Timeout':'Socket Timeout', + 'Connect timeout be a positive integer': 'Connect timeout be a positive integer', + 'Socket Timeout be a positive integer': 'Socket Timeout be a positive integer', + 'ms':'ms' } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index c9bb0e203a..e65449092e 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -600,5 +600,11 @@ export default { 'Unauthorized or deleted resources': '未授权或已删除资源', 'Please delete all non-existent resources': '请删除所有未授权或已删除资源', 'Enable': '启用', - 'Disable': '停用' + 'Disable': '停用', + 'Timeout Settings': '超时设置', + 'Connect Timeout':'连接超时', + 'Socket Timeout':'Socket超时', + 'Connect timeout be a positive integer': '连接超时必须为数字', + 'Socket Timeout be a positive integer': 'Socket超时必须为数字', + 'ms':'毫秒' } diff --git a/pom.xml b/pom.xml index 6aff092c22..d532bccc1f 100644 --- a/pom.xml +++ b/pom.xml @@ -749,6 +749,7 @@ **/common/shell/ShellExecutorTest.java **/common/task/EntityTestUtils.java **/common/task/FlinkParametersTest.java + **/common/task/HttpParametersTest.java **/common/task/SqoopParameterEntityTest.java **/common/threadutils/ThreadPoolExecutorsTest.java **/common/threadutils/ThreadUtilsTest.java @@ -821,6 +822,7 @@ **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java + **/server/worker/task/http/HttpTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/worker/EnvFileTest.java **/service/quartz/cron/CronUtilsTest.java