From 3640a2834848765f796ed1b6ba5e832657069332 Mon Sep 17 00:00:00 2001 From: Gallardot Date: Wed, 31 Jan 2024 11:02:48 +0800 Subject: [PATCH] [Improvement][HTTP] support custom rendering of http body (#15531) Signed-off-by: Gallardot Co-authored-by: Eric Gao --- .../plugin/task/http/HttpParameters.java | 13 ++++++++ .../plugin/task/http/HttpTask.java | 27 ++++++++++----- .../plugin/task/http/HttpTaskTest.java | 33 +++++++++++++++---- .../src/locales/en_US/project.ts | 3 ++ .../src/locales/zh_CN/project.ts | 2 ++ .../task/components/node/fields/use-http.ts | 9 +++++ .../task/components/node/format-data.ts | 1 + .../task/components/node/tasks/use-http.ts | 1 + .../projects/task/components/node/types.ts | 1 + 9 files changed, 74 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java index 73794819dc..83dcb78c18 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java @@ -45,6 +45,11 @@ public class HttpParameters extends AbstractParameters { */ private List httpParams; + /** + * httpBody + */ + private String httpBody; + /** * httpCheckCondition */ @@ -132,4 +137,12 @@ public class HttpParameters extends AbstractParameters { public void setSocketTimeout(int socketTimeout) { this.socketTimeout = socketTimeout; } + + public String getHttpBody() { + return httpBody; + } + + public void setHttpBody(String httpBody) { + this.httpBody = httpBody; + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java index d51c99b2a7..4ece4b4ed5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.http; -import static org.apache.dolphinscheduler.plugin.task.http.HttpTaskConstants.APPLICATION_JSON; - import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; @@ -32,7 +30,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.Charsets; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.ParseException; @@ -40,6 +37,7 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -148,7 +146,9 @@ public class HttpTask extends AbstractTask { httpPropertyList.add(JSONUtils.parseObject(params, HttpProperty.class)); } } - addRequestParams(builder, httpPropertyList); + String httpBody = ParameterUtils.convertParameterPlaceholders(httpParameters.getHttpBody(), + ParameterUtils.convert(paramsMap)); + addRequestParams(builder, httpPropertyList, httpBody); String requestUrl = ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParameterUtils.convert(paramsMap)); HttpUriRequest request = builder.setUri(requestUrl).build(); @@ -250,7 +250,14 @@ public class HttpTask extends AbstractTask { * @param builder buidler * @param httpPropertyList http property list */ - protected void addRequestParams(RequestBuilder builder, List httpPropertyList) { + protected void addRequestParams(RequestBuilder builder, List httpPropertyList, String httpBody) { + if (StringUtils.isNotEmpty(httpBody)) { + builder.setEntity(new StringEntity( + httpBody, + ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), + StandardCharsets.UTF_8))); + } + if (CollectionUtils.isNotEmpty(httpPropertyList)) { ObjectNode jsonParam = JSONUtils.createObjectNode(); for (HttpProperty property : httpPropertyList) { @@ -262,10 +269,12 @@ public class HttpTask extends AbstractTask { } } } - StringEntity postingString = new StringEntity(jsonParam.toString(), Charsets.UTF_8); - postingString.setContentEncoding(StandardCharsets.UTF_8.name()); - postingString.setContentType(APPLICATION_JSON); - builder.setEntity(postingString); + if (builder.getEntity() == null) { + builder.setEntity(new StringEntity( + jsonParam.toString(), + ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), + StandardCharsets.UTF_8))); + } } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java index ec908364ad..4c8ace158c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java @@ -150,13 +150,28 @@ public class HttpTaskTest { prepareParamsMap.put("day", "20220812"); // The MockWebServer will return the request body as response body directly // So we just need to check if the response body contains string "20220812" - HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, + HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, null, httpParams, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812", HttpStatus.SC_OK, ""); httpTask.handle(null); Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode()); } + @Test + public void testHandleWithHttpBody() throws Exception { + String httpBody = "{\"day\": ${day}}"; + + Map prepareParamsMap = new HashMap<>(); + prepareParamsMap.put("day", "20220812"); + // The MockWebServer will return the request body as response body directly + // So we just need to check if the response body contains string "20220812" + HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, httpBody, + null, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812", + HttpStatus.SC_OK, ""); + httpTask.handle(null); + Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode()); + } + @Test public void testHandleWithHttpParameterParams() throws Exception { List httpParams = new ArrayList<>(); @@ -170,7 +185,7 @@ public class HttpTaskTest { prepareParamsMap.put("day", "20220812"); // The MockWebServer will return the request parameter as response body directly // So we just need to check if the response body contains string "20220812" - HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY, HttpMethod.POST, + HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY, HttpMethod.POST, null, httpParams, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812", HttpStatus.SC_OK, ""); httpTask.handle(null); @@ -203,22 +218,24 @@ public class HttpTaskTest { } private HttpTask generateHttpTask(HttpMethod httpMethod, int actualResponseCode) throws IOException { - return generateHttpTask("/test", httpMethod, null, null, + return generateHttpTask("/test", httpMethod, null, null, null, HttpCheckCondition.STATUS_CODE_DEFAULT, "", actualResponseCode, ""); } private HttpTask generateHttpTask(HttpMethod httpMethod, HttpCheckCondition httpCheckConditionType, String condition, int actualResponseCode, String actualResponseBody) throws IOException { - return generateHttpTask("/test", httpMethod, null, null, + return generateHttpTask("/test", httpMethod, null, null, null, httpCheckConditionType, condition, actualResponseCode, actualResponseBody); } - private HttpTask generateHttpTask(String mockPath, HttpMethod httpMethod, List httpParams, + private HttpTask generateHttpTask(String mockPath, HttpMethod httpMethod, String httpBody, + List httpParams, Map prepareParamsMap, HttpCheckCondition httpCheckConditionType, String condition, int actualResponseCode, String actualResponseBody) throws IOException { String url = withMockWebServer(mockPath, actualResponseCode, actualResponseBody); - String paramData = generateHttpParameters(url, httpMethod, httpParams, httpCheckConditionType, condition); + String paramData = + generateHttpParameters(url, httpMethod, httpBody, httpParams, httpCheckConditionType, condition); return generateHttpTaskFromParamData(paramData, prepareParamsMap); } @@ -240,13 +257,15 @@ public class HttpTaskTest { return httpTask; } - private String generateHttpParameters(String url, HttpMethod httpMethod, List httpParams, + private String generateHttpParameters(String url, HttpMethod httpMethod, String httpBody, + List httpParams, HttpCheckCondition httpCheckConditionType, String condition) throws JsonProcessingException { ObjectMapper mapper = new ObjectMapper(); HttpParameters httpParameters = new HttpParameters(); httpParameters.setUrl(url); httpParameters.setHttpMethod(httpMethod); + httpParameters.setHttpBody(httpBody); httpParameters.setHttpCheckCondition(httpCheckConditionType); httpParameters.setCondition(condition); httpParameters.setConnectTimeout(10000); diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 25db7c53cc..21aba58f85 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -498,6 +498,9 @@ export default { http_url_validator: 'The request address must contain HTTP or HTTPS', http_method: 'Http Method', http_parameters: 'Http Parameters', + http_body: 'Http Body', + http_body_tips: + 'Please fill in the http body, if filled, http parameters in the body type will be ignored', http_check_condition: 'Http Check Condition', http_condition: 'Http Condition', http_condition_tips: 'Please Enter Http Condition', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 039ac5435f..1c6756a22e 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -488,6 +488,8 @@ export default { http_url_validator: '请求地址需包含http或者https', http_method: '请求类型', http_parameters: '请求参数', + http_body: '请求Body', + http_body_tips: '请填写http body,如若填写将忽略请求参数中的body类型参数', http_check_condition: '校验条件', http_condition: '校验内容', http_condition_tips: '请填写校验内容', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts index 8fd7c00cb5..525b05cdb7 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts @@ -127,6 +127,15 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] { } ] }, + { + type: 'input', + field: 'httpBody', + name: t('project.node.http_body'), + props: { + type: 'textarea', + placeholder: t('project.node.http_body_tips') + } + }, { type: 'select', field: 'httpCheckCondition', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index e9c51f35c6..1c9e072fbf 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -86,6 +86,7 @@ export function formatParams(data: INodeData): { } if (data.taskType === 'HTTP') { taskParams.httpMethod = data.httpMethod + taskParams.httpBody = data.httpBody taskParams.httpCheckCondition = data.httpCheckCondition taskParams.httpParams = data.httpParams taskParams.url = data.url diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-http.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-http.ts index a1fc52757a..731929cb45 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-http.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-http.ts @@ -44,6 +44,7 @@ export function useHttp({ delayTime: 0, timeout: 30, httpMethod: 'GET', + httpBody: '', httpCheckCondition: 'STATUS_CODE_DEFAULT', httpParams: [], url: '', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index e453a3d7f9..606339b16f 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -287,6 +287,7 @@ interface ITaskParams { mainArgs?: string others?: string httpMethod?: string + httpBody?: string httpCheckCondition?: string httpParams?: [] url?: string