[Improvement][HTTP] support custom rendering of http body (#15531)

Signed-off-by: Gallardot <gallardot@apache.org>
Co-authored-by: Eric Gao <ericgao.apache@gmail.com>

dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java

@@ -45,6 +45,11 @@ public class HttpParameters extends AbstractParameters {
      */
     private List<HttpProperty> httpParams;
 
+    /**
+     * httpBody
+     */
+    private String httpBody;
+
     /**
      * httpCheckCondition
      */
@@ -132,4 +137,12 @@ public class HttpParameters extends AbstractParameters {
     public void setSocketTimeout(int socketTimeout) {
         this.socketTimeout = socketTimeout;
     }
+
+    public String getHttpBody() {
+        return httpBody;
+    }
+
+    public void setHttpBody(String httpBody) {
+        this.httpBody = httpBody;
+    }
 }

dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java

@@ -17,8 +17,6 @@
 
 package org.apache.dolphinscheduler.plugin.task.http;
 
-import static org.apache.dolphinscheduler.plugin.task.http.HttpTaskConstants.APPLICATION_JSON;
-
 import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
@@ -32,7 +30,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters
 import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
 
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.io.Charsets;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.ParseException;
@@ -40,6 +37,7 @@ import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
@@ -148,7 +146,9 @@ public class HttpTask extends AbstractTask {
                 httpPropertyList.add(JSONUtils.parseObject(params, HttpProperty.class));
             }
         }
-        addRequestParams(builder, httpPropertyList);
+        String httpBody = ParameterUtils.convertParameterPlaceholders(httpParameters.getHttpBody(),
+                ParameterUtils.convert(paramsMap));
+        addRequestParams(builder, httpPropertyList, httpBody);
         String requestUrl =
                 ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParameterUtils.convert(paramsMap));
         HttpUriRequest request = builder.setUri(requestUrl).build();
@@ -250,7 +250,14 @@ public class HttpTask extends AbstractTask {
      * @param builder buidler
      * @param httpPropertyList http property list
      */
-    protected void addRequestParams(RequestBuilder builder, List<HttpProperty> httpPropertyList) {
+    protected void addRequestParams(RequestBuilder builder, List<HttpProperty> httpPropertyList, String httpBody) {
+        if (StringUtils.isNotEmpty(httpBody)) {
+            builder.setEntity(new StringEntity(
+                    httpBody,
+                    ContentType.create(ContentType.APPLICATION_JSON.getMimeType(),
+                            StandardCharsets.UTF_8)));
+        }
+
         if (CollectionUtils.isNotEmpty(httpPropertyList)) {
             ObjectNode jsonParam = JSONUtils.createObjectNode();
             for (HttpProperty property : httpPropertyList) {
@@ -262,10 +269,12 @@ public class HttpTask extends AbstractTask {
                     }
                 }
             }
-            StringEntity postingString = new StringEntity(jsonParam.toString(), Charsets.UTF_8);
-            postingString.setContentEncoding(StandardCharsets.UTF_8.name());
-            postingString.setContentType(APPLICATION_JSON);
-            builder.setEntity(postingString);
+            if (builder.getEntity() == null) {
+                builder.setEntity(new StringEntity(
+                        jsonParam.toString(),
+                        ContentType.create(ContentType.APPLICATION_JSON.getMimeType(),
+                                StandardCharsets.UTF_8)));
+            }
         }
     }
 

dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java

@@ -150,13 +150,28 @@ public class HttpTaskTest {
         prepareParamsMap.put("day", "20220812");
         // The MockWebServer will return the request body as response body directly
         // So we just need to check if the response body contains string "20220812"
-        HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST,
+        HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, null,
                 httpParams, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812",
                 HttpStatus.SC_OK, "");
         httpTask.handle(null);
         Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode());
     }
 
+    @Test
+    public void testHandleWithHttpBody() throws Exception {
+        String httpBody = "{\"day\": ${day}}";
+
+        Map<String, String> prepareParamsMap = new HashMap<>();
+        prepareParamsMap.put("day", "20220812");
+        // The MockWebServer will return the request body as response body directly
+        // So we just need to check if the response body contains string "20220812"
+        HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, httpBody,
+                null, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812",
+                HttpStatus.SC_OK, "");
+        httpTask.handle(null);
+        Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode());
+    }
+
     @Test
     public void testHandleWithHttpParameterParams() throws Exception {
         List<HttpProperty> httpParams = new ArrayList<>();
@@ -170,7 +185,7 @@ public class HttpTaskTest {
         prepareParamsMap.put("day", "20220812");
         // The MockWebServer will return the request parameter as response body directly
         // So we just need to check if the response body contains string "20220812"
-        HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY, HttpMethod.POST,
+        HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY, HttpMethod.POST, null,
                 httpParams, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812",
                 HttpStatus.SC_OK, "");
         httpTask.handle(null);
@@ -203,22 +218,24 @@ public class HttpTaskTest {
     }
 
     private HttpTask generateHttpTask(HttpMethod httpMethod, int actualResponseCode) throws IOException {
-        return generateHttpTask("/test", httpMethod, null, null,
+        return generateHttpTask("/test", httpMethod, null, null, null,
                 HttpCheckCondition.STATUS_CODE_DEFAULT, "", actualResponseCode, "");
     }
 
     private HttpTask generateHttpTask(HttpMethod httpMethod, HttpCheckCondition httpCheckConditionType,
                                       String condition, int actualResponseCode,
                                       String actualResponseBody) throws IOException {
-        return generateHttpTask("/test", httpMethod, null, null,
+        return generateHttpTask("/test", httpMethod, null, null, null,
                 httpCheckConditionType, condition, actualResponseCode, actualResponseBody);
     }
-    private HttpTask generateHttpTask(String mockPath, HttpMethod httpMethod, List<HttpProperty> httpParams,
+    private HttpTask generateHttpTask(String mockPath, HttpMethod httpMethod, String httpBody,
+                                      List<HttpProperty> httpParams,
                                       Map<String, String> prepareParamsMap, HttpCheckCondition httpCheckConditionType,
                                       String condition, int actualResponseCode,
                                       String actualResponseBody) throws IOException {
         String url = withMockWebServer(mockPath, actualResponseCode, actualResponseBody);
-        String paramData = generateHttpParameters(url, httpMethod, httpParams, httpCheckConditionType, condition);
+        String paramData =
+                generateHttpParameters(url, httpMethod, httpBody, httpParams, httpCheckConditionType, condition);
         return generateHttpTaskFromParamData(paramData, prepareParamsMap);
     }
 
@@ -240,13 +257,15 @@ public class HttpTaskTest {
         return httpTask;
     }
 
-    private String generateHttpParameters(String url, HttpMethod httpMethod, List<HttpProperty> httpParams,
+    private String generateHttpParameters(String url, HttpMethod httpMethod, String httpBody,
+                                          List<HttpProperty> httpParams,
                                           HttpCheckCondition httpCheckConditionType,
                                           String condition) throws JsonProcessingException {
         ObjectMapper mapper = new ObjectMapper();
         HttpParameters httpParameters = new HttpParameters();
         httpParameters.setUrl(url);
         httpParameters.setHttpMethod(httpMethod);
+        httpParameters.setHttpBody(httpBody);
         httpParameters.setHttpCheckCondition(httpCheckConditionType);
         httpParameters.setCondition(condition);
         httpParameters.setConnectTimeout(10000);

dolphinscheduler-ui/src/locales/en_US/project.ts

@@ -498,6 +498,9 @@ export default {
     http_url_validator: 'The request address must contain HTTP or HTTPS',
     http_method: 'Http Method',
     http_parameters: 'Http Parameters',
+    http_body: 'Http Body',
+    http_body_tips:
+      'Please fill in the http body, if filled, http parameters in the body type will be ignored',
     http_check_condition: 'Http Check Condition',
     http_condition: 'Http Condition',
     http_condition_tips: 'Please Enter Http Condition',

dolphinscheduler-ui/src/locales/zh_CN/project.ts

@@ -488,6 +488,8 @@ export default {
     http_url_validator: '请求地址需包含http或者https',
     http_method: '请求类型',
     http_parameters: '请求参数',
+    http_body: '请求Body',
+    http_body_tips: '请填写http body,如若填写将忽略请求参数中的body类型参数',
     http_check_condition: '校验条件',
     http_condition: '校验内容',
     http_condition_tips: '请填写校验内容',

dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts

@@ -127,6 +127,15 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] {
         }
       ]
     },
+    {
+      type: 'input',
+      field: 'httpBody',
+      name: t('project.node.http_body'),
+      props: {
+        type: 'textarea',
+        placeholder: t('project.node.http_body_tips')
+      }
+    },
     {
       type: 'select',
       field: 'httpCheckCondition',

dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts

@@ -86,6 +86,7 @@ export function formatParams(data: INodeData): {
   }
   if (data.taskType === 'HTTP') {
     taskParams.httpMethod = data.httpMethod
+    taskParams.httpBody = data.httpBody
     taskParams.httpCheckCondition = data.httpCheckCondition
     taskParams.httpParams = data.httpParams
     taskParams.url = data.url

dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-http.ts

@@ -44,6 +44,7 @@ export function useHttp({
     delayTime: 0,
     timeout: 30,
     httpMethod: 'GET',
+    httpBody: '',
     httpCheckCondition: 'STATUS_CODE_DEFAULT',
     httpParams: [],
     url: '',

dolphinscheduler-ui/src/views/projects/task/components/node/types.ts

@@ -287,6 +287,7 @@ interface ITaskParams {
   mainArgs?: string
   others?: string
   httpMethod?: string
+  httpBody?: string
   httpCheckCondition?: string
   httpParams?: []
   url?: string