add http task timeout & test unit (#2955)

* add http task timeout & test unit

* add http task timeout & test unit

Co-authored-by: zhanglong <zhanglong@ysstech.com>
Co-authored-by: dailidong <dailidong66@gmail.com>

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java

@@ -56,6 +56,17 @@ public class HttpParameters extends AbstractParameters {
     private String condition;
 
 
+    /**
+     * Connect Timeout
+     * Unit: ms
+     */
+    private int connectTimeout ;
+
+    /**
+     * Socket Timeout
+     * Unit: ms
+     */
+    private int socketTimeout ;
 
     @Override
     public boolean checkParameters() {
@@ -106,4 +117,20 @@ public class HttpParameters extends AbstractParameters {
     public void setCondition(String condition) {
         this.condition = condition;
     }
+
+    public int getConnectTimeout() {
+        return connectTimeout;
+    }
+
+    public void setConnectTimeout(int connectTimeout) {
+        this.connectTimeout = connectTimeout;
+    }
+
+    public int getSocketTimeout() {
+        return socketTimeout;
+    }
+
+    public void setSocketTimeout(int socketTimeout) {
+        this.socketTimeout = socketTimeout;
+    }
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.common.task;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.enums.HttpCheckCondition;
+import org.apache.dolphinscheduler.common.enums.HttpMethod;
+import org.apache.dolphinscheduler.common.process.HttpProperty;
+import org.apache.dolphinscheduler.common.process.ResourceInfo;
+import org.apache.dolphinscheduler.common.task.http.HttpParameters;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.fastjson.JSON;
+
+/**
+ * http parameter
+ */
+public class HttpParametersTest  {
+
+
+    @Test
+    public void testGenerator(){
+        String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," +
+                "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
+        HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class);
+
+
+        Assert.assertEquals(10000,httpParameters.getConnectTimeout() );
+        Assert.assertEquals(10000,httpParameters.getSocketTimeout());
+        Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl());
+        Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());
+        Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());
+        Assert.assertEquals("",httpParameters.getCondition());
+
+    }
+
+
+    @Test
+    public void testCheckParameters(){
+        String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," +
+                "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
+        HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class);
+
+        Assert.assertTrue( httpParameters.checkParameters());
+        Assert.assertEquals(10000,httpParameters.getConnectTimeout() );
+        Assert.assertEquals(10000,httpParameters.getSocketTimeout());
+        Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl());
+        Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());
+        Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());
+        Assert.assertEquals("",httpParameters.getCondition());
+
+    }
+
+
+    @Test
+    public void testCheckValues() {
+        String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," +
+                "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
+        HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class);
+
+        Assert.assertTrue( httpParameters.checkParameters());
+        Assert.assertEquals(10000,httpParameters.getConnectTimeout() );
+        Assert.assertEquals(10000,httpParameters.getSocketTimeout());
+        Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl());
+        Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());
+        Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());
+        Assert.assertEquals("",httpParameters.getCondition());
+        Assert.assertEquals(0,httpParameters.getLocalParametersMap().size());
+        Assert.assertEquals(0,httpParameters.getResourceFilesList().size());
+    }
+
+
+}

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java

@@ -66,13 +66,6 @@ public class HttpTask extends AbstractTask {
      */
     private HttpParameters httpParameters;
 
-
-
-    /**
-     * Convert mill seconds to second unit
-     */
-    protected static final int MAX_CONNECTION_MILLISECONDS = 60000;
-
     /**
      * application json
      */
@@ -303,7 +296,7 @@ public class HttpTask extends AbstractTask {
      * @return RequestConfig
      */
     private RequestConfig requestConfig() {
-        return RequestConfig.custom().setSocketTimeout(MAX_CONNECTION_MILLISECONDS).setConnectTimeout(MAX_CONNECTION_MILLISECONDS).build();
+        return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout()).setConnectTimeout(httpParameters.getConnectTimeout()).build();
     }
 
     /**

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java

@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.http;
+
+import static org.apache.dolphinscheduler.common.enums.CommandType.*;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.dolphinscheduler.common.enums.HttpCheckCondition;
+import org.apache.dolphinscheduler.common.enums.HttpMethod;
+import org.apache.dolphinscheduler.common.task.http.HttpParameters;
+import org.apache.dolphinscheduler.common.utils.OSUtils;
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
+import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationContext;
+
+import com.alibaba.fastjson.JSON;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(OSUtils.class)
+@PowerMockIgnore({"javax.management.*","javax.net.ssl.*"})
+public class HttpTaskTest {
+    private static final Logger logger = LoggerFactory.getLogger(HttpTaskTest.class);
+
+
+
+    private HttpTask httpTask;
+
+    private ProcessService processService;
+
+    private ShellCommandExecutor shellCommandExecutor;
+
+    private ApplicationContext applicationContext;
+    private TaskExecutionContext taskExecutionContext;
+
+    @Before
+    public void before() throws Exception {
+        taskExecutionContext = new TaskExecutionContext();
+
+        PowerMockito.mockStatic(OSUtils.class);
+        processService = PowerMockito.mock(ProcessService.class);
+        shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class);
+
+        applicationContext = PowerMockito.mock(ApplicationContext.class);
+        SpringApplicationContext springApplicationContext = new SpringApplicationContext();
+        springApplicationContext.setApplicationContext(applicationContext);
+        PowerMockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
+
+        TaskProps props = new TaskProps();
+        props.setExecutePath("/tmp");
+        props.setTaskAppId(String.valueOf(System.currentTimeMillis()));
+        props.setTaskInstanceId(1);
+        props.setTenantCode("1");
+        props.setEnvFile(".dolphinscheduler_env.sh");
+        props.setTaskStartTime(new Date());
+        props.setTaskTimeout(0);
+        props.setTaskParams(
+                "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\",\"httpMethod\":\"GET\"," +
+                        "\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"https://github.com/\"," +
+                        "\"connectTimeout\":\"1000\",\"socketTimeout\":\"1000\"}");
+
+
+        taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
+        Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(props.getTaskParams());
+        Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp");
+        Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn("1");
+        Mockito.when(taskExecutionContext.getTenantCode()).thenReturn("root");
+        Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date());
+        Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000);
+        Mockito.when(taskExecutionContext.getLogPath()).thenReturn("/tmp/dx");
+
+        httpTask = new HttpTask(taskExecutionContext, logger);
+        httpTask.init();
+
+    }
+
+    @Test
+    public void testGetParameters() {
+        Assert.assertNotNull(httpTask.getParameters());
+    }
+
+
+    @Test
+    public void testCheckParameters() {
+        Assert.assertTrue(httpTask.getParameters().checkParameters());
+    }
+
+
+    @Test
+    public void testGenerator(){
+        String paramJson = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\"," +
+                "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
+        HttpParameters httpParameters = JSON.parseObject(paramJson, HttpParameters.class);
+
+
+        Assert.assertEquals(10000,httpParameters.getConnectTimeout() );
+        Assert.assertEquals(10000,httpParameters.getSocketTimeout());
+        Assert.assertEquals("https://github.com/",httpParameters.getUrl());
+        Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());
+        Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());
+        Assert.assertEquals("",httpParameters.getCondition());
+
+    }
+
+    @Test
+    public void testHandle(){
+        boolean flag = true ;
+        try {
+            httpTask.handle();
+        } catch (Exception e) {
+            flag = false ;
+            e.printStackTrace();
+        }
+
+        Assert.assertTrue(flag);
+
+    }
+
+    @Test
+    public void testSendRequest(){
+
+        CloseableHttpClient client = httpTask.createHttpClient();
+
+        String statusCode = null;
+        String body = null;
+
+        try {
+
+            CloseableHttpResponse response = httpTask.sendRequest(client) ;
+            statusCode = String.valueOf(httpTask.getStatusCode(response));
+            body = httpTask.getResponseBody(response);
+            int exitStatusCode = httpTask.validResponse(body, statusCode);
+
+            Assert.assertNotEquals(-1,exitStatusCode);
+
+        } catch (IOException e) {
+            e.printStackTrace();
+        };
+    }
+
+    @Test
+    public void testValidResponse(){
+        String body = "body";
+        String statusCode = "200" ;
+
+        int exitStatusCode = httpTask.validResponse(body,statusCode);
+        Assert.assertNotEquals(-1,exitStatusCode);
+
+    }
+
+    @Test
+    public void testAppendMessage(){
+        httpTask.appendMessage("message");
+
+        Assert.assertEquals("message",httpTask.getOutput());
+    }
+
+    @Test
+    public void testCreateHttpClient(){
+        Assert.assertNotNull(httpTask.createHttpClient());
+    }
+
+    @Test
+    public void testCreateRequestBuilder(){
+        RequestBuilder  requestBuilder = httpTask.createRequestBuilder();
+        Assert.assertEquals(RequestBuilder.get().getMethod(),requestBuilder.getMethod());
+    }
+
+    private ProcessInstance getProcessInstance() {
+        ProcessInstance processInstance = new ProcessInstance();
+        processInstance.setCommandType(START_PROCESS);
+        processInstance.setScheduleTime(new Date());
+        return processInstance;
+    }
+}

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue

@@ -85,6 +85,39 @@
         </x-input>
       </div>
     </m-list-box>
+
+
+    <m-list-box  >
+      <div slot="text">{{$t('Timeout Settings')}}</div>
+      <div slot="content">
+        <label class="label-box">
+          <div style="padding-top: 5px;">
+            <x-switch
+              v-model="timeoutSettings"
+              :disabled="isDetails"
+            ></x-switch>
+          </div>
+        </label>
+      </div>
+    </m-list-box>
+
+    <div class="clearfix list" v-if = "timeoutSettings" >
+      <div class="text-box">
+        <span>{{$t('Connect Timeout')}}</span>
+      </div>
+      <div class="cont-box">
+        <span  class="label-box"  style="width: 193px;display: inline-block;" >
+          <x-input v-model='connectTimeout' maxlength="7" />
+        </span>
+        <span>{{$t('ms')}}</span>
+        <span class="text-b">{{$t('Socket Timeout')}}</span>
+        <span  class="label-box" style="width: 193px;display: inline-block;" >
+          <x-input v-model='socketTimeout' maxlength="7" />
+        </span>
+        <span>{{$t('ms')}}</span>
+      </div>
+    </div>
+
     <m-list-box>
       <div slot="text">{{$t('Custom Parameters')}}</div>
       <div slot="content">
@@ -110,6 +143,10 @@
     name: 'http',
     data () {
       return {
+        timeoutSettings: false,
+        connectTimeout : 60000 ,
+        socketTimeout :  60000 ,
+
         url: '',
         condition: '',
         localParams: [],
@@ -152,6 +189,14 @@
         if (!this.$refs.refHttpParams._verifValue()) {
           return false
         }
+        if (!_.isNumber(parseInt(this.socketTimeout))) {
+          this.$message.warning(`${i18n.$t('Socket Timeout be a positive integer')}`)
+          return false
+        }
+        if (!_.isNumber(parseInt(this.connectTimeout))) {
+          this.$message.warning(`${i18n.$t('Connect timeout be a positive integer')}`)
+          return false
+        }
         // storage
         this.$emit('on-params', {
           localParams: this.localParams,
@@ -159,7 +204,9 @@
           url: this.url,
           httpMethod: this.httpMethod,
           httpCheckCondition: this.httpCheckCondition,
-          condition: this.condition
+          condition: this.condition,
+          connectTimeout : this.connectTimeout ,
+          socketTimeout : this.socketTimeout
         })
         return true
       }
@@ -172,7 +219,9 @@
           url: this.url,
           httpMethod: this.httpMethod,
           httpCheckCondition: this.httpCheckCondition,
-          condition: this.condition
+          condition: this.condition,
+          connectTimeout : this.connectTimeout ,
+          socketTimeout : this.socketTimeout
         }
       }
     },
@@ -193,6 +242,11 @@
           this.httpMethod = o.params.httpMethod || 'GET'
           this.httpCheckCondition = o.params.httpCheckCondition || 'DEFAULT'
           this.condition = o.params.condition || ''
+          this.connectTimeout = o.params.connectTimeout
+          this.socketTimeout = o.params.socketTimeout
+          if(this.connectTimeout != 60000 || this.socketTimeout != 60000 ){
+            this.timeoutSettings = true
+          }
           // backfill localParams
           let localParams = o.params.localParams || []
           if (localParams.length) {

dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@@ -600,5 +600,11 @@ export default {
   'Unauthorized or deleted resources': 'Unauthorized or deleted resources',
   'Please delete all non-existent resources': 'Please delete all non-existent resources',
   'Enable': 'Enable',
-  'Disable': 'Disable'
+  'Disable': 'Disable',
+  'Timeout Settings': 'Timeout Settings',
+  'Connect Timeout':'Connect Timeout',
+  'Socket Timeout':'Socket Timeout',
+  'Connect timeout be a positive integer': 'Connect timeout be a positive integer',
+  'Socket Timeout be a positive integer': 'Socket Timeout be a positive integer',
+  'ms':'ms'
 }

dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@@ -600,5 +600,11 @@ export default {
   'Unauthorized or deleted resources': '未授权或已删除资源',
   'Please delete all non-existent resources': '请删除所有未授权或已删除资源',
   'Enable': '启用',
-  'Disable': '停用'
+  'Disable': '停用',
+  'Timeout Settings': '超时设置',
+  'Connect Timeout':'连接超时',
+  'Socket Timeout':'Socket超时',
+  'Connect timeout be a positive integer': '连接超时必须为数字',
+  'Socket Timeout be a positive integer': 'Socket超时必须为数字',
+  'ms':'毫秒'
 }

pom.xml

@@ -749,6 +749,7 @@
                         <include>**/common/shell/ShellExecutorTest.java</include>
                         <include>**/common/task/EntityTestUtils.java</include>
                         <include>**/common/task/FlinkParametersTest.java</include>
+                        <include>**/common/task/HttpParametersTest.java</include>
                         <include>**/common/task/SqoopParameterEntityTest.java</include>
                         <include>**/common/threadutils/ThreadPoolExecutorsTest.java</include>
                         <include>**/common/threadutils/ThreadUtilsTest.java</include>
@@ -821,6 +822,7 @@
                         <include>**/server/worker/task/spark/SparkTaskTest.java</include>
                         <include>**/server/worker/task/EnvFileTest.java</include>
                         <include>**/server/worker/task/spark/SparkTaskTest.java</include>
+                        <include>**/server/worker/task/http/HttpTaskTest.java</include>
                         <include>**/server/worker/task/sqoop/SqoopTaskTest.java</include>
                         <include>**/server/worker/EnvFileTest.java</include>
                         <include>**/service/quartz/cron/CronUtilsTest.java</include>