Browse Source

[DSIP-60][Http task plugin] Refactor http task plugin (#16413)

dev
xiangzihao 4 months ago committed by GitHub
parent
commit
a6504412cf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 24
      docs/docs/en/guide/task/http.md
  2. 27
      docs/docs/zh/guide/task/http.md
  3. BIN
      docs/img/tasks/demo/http_post.png
  4. BIN
      docs/img/tasks/demo/http_task01.png
  5. BIN
      docs/img/tasks/icons/http.png
  6. BIN
      docs/img/tasks/icons/http_get.png
  7. 13
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/utils/RequestClient.java
  8. 28
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoginController.java
  9. 40
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java
  10. 2
      dolphinscheduler-bom/pom.xml
  11. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
  12. 31
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpCheckCondition.java
  13. 32
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpMethod.java
  14. 30
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpParametersType.java
  15. 42
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/OkHttpRequestHeaderContentType.java
  16. 32
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/OkHttpRequestHeaders.java
  17. 34
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/OkHttpResponse.java
  18. 152
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OkHttpUtils.java
  19. 6
      dolphinscheduler-dist/release-docs/LICENSE
  20. 202
      dolphinscheduler-dist/release-docs/licenses/LICENSE-okhttp.txt
  21. 31
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/TemplateMethodTransformer.java
  22. 57
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/BaseHttpTemplateLoopTaskExecutor.java
  23. 65
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskDefinition.java
  24. 31
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskInstanceInfo.java
  25. 35
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskInstanceStatus.java
  26. 44
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskMethodDefinition.java
  27. 81
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskCancelTaskMethodDefinition.java
  28. 88
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskQueryStatusMethodDefinition.java
  29. 76
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskSubmitTaskMethodDefinition.java
  30. 47
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/CancelTemplateMethodTransformer.java
  31. 89
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParser.java
  32. 49
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/QueryStateTemplateMethodTransformer.java
  33. 49
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/SubmitTemplateMethodTransformer.java
  34. 66
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParserTest.java
  35. 20
      dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpConstants.java
  36. 113
      dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java
  37. 7
      dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java
  38. 84
      dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java
  39. 8
      dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpRequestMethod.java
  40. 390
      dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java
  41. 15
      dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersTest.java
  42. 114
      dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java
  43. 4
      dolphinscheduler-ui/src/locales/en_US/project.ts
  44. 3
      dolphinscheduler-ui/src/locales/zh_CN/project.ts
  45. 93
      dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts
  46. 5
      tools/dependencies/known-dependencies.txt

24
docs/docs/en/guide/task/http.md

@ -2,12 +2,12 @@
## Overview
This node is used to perform http type tasks such as the common POST and GET request types, and also supports http request validation and other functions.
This node is used to perform http type tasks and also supports http request validation and other functions.
## Create Task
- Click `Project Management -> Project Name -> Workflow Definition`, and click the `Create Workflow` button to enter the DAG editing page.
- Drag the <img src="../../../../img/tasks/icons/http.png" width="15"/> from the toolbar to the drawing board.
- Drag the <img src="../../../../img/tasks/icons/http_get.png" width="15"/> from the toolbar to the drawing board.
## Task Parameters
@ -19,7 +19,7 @@ This node is used to perform http type tasks such as the common POST and GET req
| **Parameter** | **Description** |
|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Request address | HTTP request URL. |
| Request type | Supports GET, POST, HEAD, PUT, DELETE. |
| Request type | Supports GET, POST, PUT, DELETE |
| Request parameters | Supports Parameter, Body, Headers. |
| Verification conditions | Supports default response code, custom response code, content included, content not included. |
| Verification content | When the verification condition selects a custom response code, the content contains, and the content does not contain, the verification content is required. |
@ -37,14 +37,16 @@ For example, if the current task1 is a http task, the downstream task can use `$
## Example
HTTP defines the different methods of interacting with the server, the most basic methods are GET, POST, PUT and DELETE. Here we use the http task node to demonstrate the use of POST to send a request to the system's login page to submit data.
HTTP defines the different methods of interacting with the server, the most basic methods are GET, POST, PUT, DELETE. Here we use the http task node to demonstrate the use of POST to send a request to the system's login page to submit data.
The main configuration parameters are as follows:
The main configuration parameters are as follows(All parameters can be replaced by built-in parameters):
- URL: Address to access the target resource. Here is the system's login page.
- HTTP Parameters:
- userName: Username
- userPassword: User login password
![http_task](../../../../img/tasks/demo/http_task01.png)
- Request type: GET, POST, PUT, DELETE
- Headers: Request header information, currently only supports application/json, application/x-www-form-urlencoded format. If other formats are entered, the application/json format will be used by default.
- HTTP Parameters: GET, DELETE request parameters.
- HTTP Body: POST, PUT request parameters.
- Verification conditions: Default response code 200, custom response code, content included, content not included.
- Verification content: When the verification condition is custom response code, content included, content not included, the verification content is required, and the verification content is a fuzzy match.
![http_task](../../../../img/tasks/demo/http_post.png)

27
docs/docs/zh/guide/task/http.md

@ -2,13 +2,13 @@
## 综述
该节点用于执行 http 类型的任务,例如常见的 POST、GET 等请求类型,此外还支持 http 请求校验等功能。
该节点用于执行 http 类型的任务,此外还支持 http 请求校验等功能。
## 创建任务
- 点击项目管理 -> 项目名称 -> 工作流定义,点击”创建工作流”按钮,进入 DAG 编辑页面:
- 拖动工具栏的 <img src="../../../../img/tasks/icons/http.png" width="15"/> 任务节点到画板中。
- 拖动工具栏的 <img src="../../../../img/tasks/icons/http_get.png" width="15"/> 任务节点到画板中。
## 任务参数
@ -20,7 +20,7 @@
| **任务参数** | **描述** |
|----------|-------------------------------------|
| 请求地址 | http 请求 URL |
| 请求类型 | 支持 GET、POST、HEAD、PUT、DELETE |
| 请求类型 | 支持 GET、POST、PUT、DELETE |
| 请求参数 | 支持 Parameter、Body、Headers |
| 校验条件 | 支持默认响应码、自定义响应码、内容包含、内容不包含 |
| 校验内容 | 当校验条件选择自定义响应码、内容包含、内容不包含时,需填写校验内容 |
@ -38,17 +38,16 @@
## 任务样例
HTTP 定义了与服务器交互的不同方法,最基本的方法有4种,分别是GET,POST,PUT,DELETE。这里我们使用 http 任务节点,演示使用 POST 向系统的登录页面发送请求,提交数据。
HTTP 定义了与服务器交互的不同方法,最基本的方法有4种,分别是GET,POST, PUT, DELETE。这里我们使用 http 任务节点,演示使用 POST 向系统的登录页面发送请求,提交数据。
主要配置参数如下:
主要配置参数如下(以下参数均可通过内置参数替换)
- URL:访问目标资源的地址,这里为系统的登录页面。
- HTTP Parameters
- userName:用户名;
- userPassword:用户登录密码。
![http_task](../../../../img/tasks/demo/http_task01.png)
## 注意事项
无。
- 请求类型:GET、POST、PUT、DELETE
- Headers: 请求头信息,当前仅支持 application/json、application/x-www-form-urlencoded 格式,如输入其他格式默认会使用 application/json 格式。
- HTTP Parameters(GET、DELETE请求参数)
- HTTP Body(POST、PUT请求参数)
- 校验条件:默认响应码200、自定义响应码、内容包含、内容不包含
- 校验内容:校验条件为自定义响应码、内容包含、内容不包含时,需填写校验内容,校验内容为模糊匹配
![http_task](../../../../img/tasks/demo/http_post.png)

BIN
docs/img/tasks/demo/http_post.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 150 KiB

BIN
docs/img/tasks/demo/http_task01.png

Binary file not shown.

Before

Width:  |  Height:  |  Size: 204 KiB

BIN
docs/img/tasks/icons/http.png

Binary file not shown.

Before

Width:  |  Height:  |  Size: 707 B

BIN
docs/img/tasks/icons/http_get.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 166 KiB

13
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/utils/RequestClient.java

@ -33,7 +33,7 @@ import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import java.io.File;
import java.io.FileInputStream;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
@ -91,7 +91,7 @@ public class RequestClient {
public static String getParams(Map<String, Object> params) {
StringBuilder sb = new StringBuilder(Constants.QUESTION_MARK);
if (params.size() > 0) {
if (!params.isEmpty()) {
for (Map.Entry<String, Object> item : params.entrySet()) {
Object value = item.getValue();
if (Objects.nonNull(value)) {
@ -116,7 +116,7 @@ public class RequestClient {
String requestUrl = String.format("%s%s", Constants.DOLPHINSCHEDULER_API_URL, url);
headers.put("Content-Type", Constants.REQUEST_CONTENT_TYPE);
Headers headersBuilder = Headers.of(headers);
RequestBody requestBody = FormBody.create(MediaType.parse(Constants.REQUEST_CONTENT_TYPE), getParams(params));
RequestBody requestBody = FormBody.create(getParams(params), MediaType.parse(Constants.REQUEST_CONTENT_TYPE));
log.info("POST request to {}, Headers: {}, Params: {}", requestUrl, headersBuilder, params);
Request request = new Request.Builder()
.headers(headersBuilder)
@ -147,7 +147,7 @@ public class RequestClient {
String requestUrl = String.format("%s%s", Constants.DOLPHINSCHEDULER_API_URL, url);
headers.put("Content-Type", Constants.REQUEST_CONTENT_TYPE);
Headers headersBuilder = Headers.of(headers);
RequestBody requestBody = FormBody.create(MediaType.parse(Constants.REQUEST_CONTENT_TYPE), getParams(params));
RequestBody requestBody = FormBody.create(getParams(params), MediaType.parse(Constants.REQUEST_CONTENT_TYPE));
log.info("PUT request to {}, Headers: {}, Params: {}", requestUrl, headersBuilder, params);
Request request = new Request.Builder()
.headers(headersBuilder)
@ -177,7 +177,7 @@ public class RequestClient {
builder.addTextBody("json", getParams(params), ContentType.MULTIPART_FORM_DATA);
builder.addBinaryBody(
"file",
new FileInputStream(file),
Files.newInputStream(file.toPath()),
ContentType.APPLICATION_OCTET_STREAM,
file.getName());
HttpEntity multipart = builder.build();
@ -189,8 +189,7 @@ public class RequestClient {
}
httpPost.setEntity(multipart);
CloseableHttpClient client = HttpClients.createDefault();
CloseableHttpResponse response = client.execute(httpPost);
return response;
return client.execute(httpPost);
} catch (Exception e) {
log.error("error", e);
}

28
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoginController.java

@ -32,6 +32,8 @@ import org.apache.dolphinscheduler.api.service.UsersService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaderContentType;
import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaders;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OkHttpUtils;
import org.apache.dolphinscheduler.dao.entity.Session;
@ -95,10 +97,10 @@ public class LoginController extends BaseController {
/**
* login
*
* @param userName user name
* @param userName user name
* @param userPassword user password
* @param request request
* @param response response
* @param request request
* @param response response
* @return login result
*/
@Operation(summary = "login", description = "LOGIN_NOTES")
@ -223,16 +225,28 @@ public class LoginController extends BaseController {
requestParamsMap.put("grant_type", "authorization_code");
requestParamsMap.put("redirect_uri",
String.format("%s?provider=%s", oAuth2ClientProperties.getRedirectUri(), provider));
String tokenJsonStr = OkHttpUtils.post(oAuth2ClientProperties.getTokenUri(), tokenRequestHeader,
requestParamsMap, requestBody);
OkHttpRequestHeaders okHttpRequestHeadersPost = new OkHttpRequestHeaders();
okHttpRequestHeadersPost.setHeaders(tokenRequestHeader);
okHttpRequestHeadersPost.setOkHttpRequestHeaderContentType(OkHttpRequestHeaderContentType.APPLICATION_JSON);
String tokenJsonStr = OkHttpUtils.post(oAuth2ClientProperties.getTokenUri(), okHttpRequestHeadersPost,
requestParamsMap, requestBody, Constants.HTTP_CONNECT_TIMEOUT, Constants.HTTP_CONNECT_TIMEOUT,
Constants.HTTP_CONNECT_TIMEOUT).getBody();
String accessToken = JSONUtils.getNodeString(tokenJsonStr, "access_token");
Map<String, String> userInfoRequestHeaders = new HashMap<>();
userInfoRequestHeaders.put("Accept", "application/json");
Map<String, Object> userInfoQueryMap = new HashMap<>();
userInfoQueryMap.put("access_token", accessToken);
userInfoRequestHeaders.put("Authorization", "Bearer " + accessToken);
String userInfoJsonStr =
OkHttpUtils.get(oAuth2ClientProperties.getUserInfoUri(), userInfoRequestHeaders, userInfoQueryMap);
OkHttpRequestHeaders okHttpRequestHeadersGet = new OkHttpRequestHeaders();
okHttpRequestHeadersGet.setHeaders(userInfoRequestHeaders);
String userInfoJsonStr = OkHttpUtils.get(oAuth2ClientProperties.getUserInfoUri(),
okHttpRequestHeadersGet,
userInfoQueryMap,
Constants.HTTP_CONNECT_TIMEOUT,
Constants.HTTP_CONNECT_TIMEOUT,
Constants.HTTP_CONNECT_TIMEOUT).getBody();
String username = JSONUtils.getNodeString(userInfoJsonStr, "login");
User user = usersService.getUserByUserName(username);
if (user == null) {

40
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java

@ -27,9 +27,6 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OkHttpUtils;
import org.apache.http.HttpStatus;
import java.util.Map;
@ -37,8 +34,6 @@ import javax.servlet.http.Cookie;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
@ -113,39 +108,4 @@ public class LoginControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
}
@Test
void testOauth2Redirect() throws Exception {
String tokenResult = "{\"access_token\":\"test-token\"}";
String userInfoResult = "{\"login\":\"username\"}";
MockedStatic<OkHttpUtils> okHttpUtilsMockedStatic = Mockito.mockStatic(OkHttpUtils.class);
okHttpUtilsMockedStatic
.when(() -> OkHttpUtils.post(Mockito.notNull(), Mockito.any(), Mockito.any(), Mockito.any()))
.thenReturn(tokenResult);
okHttpUtilsMockedStatic.when(() -> OkHttpUtils.get(Mockito.notNull(), Mockito.any(), Mockito.any()))
.thenReturn(userInfoResult);
MvcResult mvcResult = mockMvc.perform(get("/redirect/login/oauth2?code=test&provider=github"))
.andExpect(status().is3xxRedirection())
.andReturn();
MockHttpServletResponse response = mvcResult.getResponse();
Assertions.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY, response.getStatus());
String redirectedUrl = response.getRedirectedUrl();
Assertions.assertTrue(redirectedUrl != null && redirectedUrl.contains("sessionId"));
okHttpUtilsMockedStatic.close();
}
@Test
void testOauth2RedirectError() throws Exception {
MockedStatic<OkHttpUtils> okHttpUtilsMockedStatic = Mockito.mockStatic(OkHttpUtils.class);
okHttpUtilsMockedStatic.when(() -> OkHttpUtils.post(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
.thenThrow(new RuntimeException("oauth error"));
MvcResult mvcResult = mockMvc.perform(get("/redirect/login/oauth2?code=test&provider=github"))
.andExpect(status().is3xxRedirection())
.andReturn();
MockHttpServletResponse response = mvcResult.getResponse();
Assertions.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY, response.getStatus());
String redirectedUrl = response.getRedirectedUrl();
Assertions.assertTrue(redirectedUrl != null && redirectedUrl.contains("error"));
okHttpUtilsMockedStatic.close();
}
}

2
dolphinscheduler-bom/pom.xml

@ -89,7 +89,7 @@
<aws-sdk.version>1.12.300</aws-sdk.version>
<aliyun-sdk-oss.version>3.15.1</aliyun-sdk-oss.version>
<joda-time.version>2.10.13</joda-time.version>
<okhttp.version>4.9.3</okhttp.version>
<okhttp.version>4.12.0</okhttp.version>
<json-path.version>2.7.0</json-path.version>
<spring-cloud-dependencies.version>2021.0.3</spring-cloud-dependencies.version>
<gson.version>2.9.1</gson.version>

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java

@ -22,11 +22,10 @@ import org.apache.commons.lang3.SystemUtils;
import java.time.Duration;
import java.util.regex.Pattern;
public final class Constants {
import lombok.experimental.UtilityClass;
private Constants() {
throw new UnsupportedOperationException("Construct Constants");
}
@UtilityClass
public final class Constants {
public static final String AUTO_CLOSE_ALERT = "alert.auto-close";

31
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpCheckCondition.java

@ -1,31 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.enums;
/**
* http check condition
*/
public enum HttpCheckCondition {
/**
* 0 status_code_default:200
* 1 status_code_custom
* 2 body_contains
* 3 body_not_contains
*/
STATUS_CODE_DEFAULT, STATUS_CODE_CUSTOM, BODY_CONTAINS, BODY_NOT_CONTAINS
}

32
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpMethod.java

@ -1,32 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.enums;
/**
* http method
*/
public enum HttpMethod {
/**
* 0 get
* 1 post
* 2 head
* 3 put
* 4 delete
*/
GET, POST, HEAD, PUT, DELETE
}

30
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpParametersType.java

@ -1,30 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.enums;
/**
* http parameters type
*/
public enum HttpParametersType {
/**
* 0 parameter;
* 1 body;
* 2 headers;
*/
PARAMETER, BODY, HEADERS
}

42
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/OkHttpRequestHeaderContentType.java

@ -0,0 +1,42 @@
/*
* Licensed to Apache Software Foundation (ASF) under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Apache Software Foundation (ASF) licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.dolphinscheduler.common.model;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@Getter
@RequiredArgsConstructor
public enum OkHttpRequestHeaderContentType {
APPLICATION_JSON("application/json"),
APPLICATION_FORM_URLENCODED("application/x-www-form-urlencoded");
private final String value;
public static OkHttpRequestHeaderContentType fromValue(String value) {
for (OkHttpRequestHeaderContentType contentType : OkHttpRequestHeaderContentType.values()) {
if (contentType.getValue().equalsIgnoreCase(value)) {
return contentType;
}
}
return null;
}
}

32
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/OkHttpRequestHeaders.java

@ -0,0 +1,32 @@
/*
* Licensed to Apache Software Foundation (ASF) under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Apache Software Foundation (ASF) licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.dolphinscheduler.common.model;
import java.util.Map;
import lombok.Data;
@Data
public class OkHttpRequestHeaders {
private Map<String, String> headers;
private OkHttpRequestHeaderContentType okHttpRequestHeaderContentType;
}

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/OkHttpResponse.java

@ -0,0 +1,34 @@
/*
* Licensed to Apache Software Foundation (ASF) under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Apache Software Foundation (ASF) licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.dolphinscheduler.common.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@AllArgsConstructor
@NoArgsConstructor
@Data
public class OkHttpResponse {
private int statusCode;
private String body;
}

152
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OkHttpUtils.java

@ -17,6 +17,11 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaderContentType;
import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaders;
import org.apache.dolphinscheduler.common.model.OkHttpResponse;
import org.apache.http.HttpStatus;
import java.io.IOException;
@ -35,37 +40,115 @@ import okhttp3.Response;
public class OkHttpUtils {
private static final OkHttpClient CLIENT = new OkHttpClient.Builder()
.connectTimeout(5, TimeUnit.MINUTES) // connect timeout
.writeTimeout(5, TimeUnit.MINUTES) // write timeout
.readTimeout(5, TimeUnit.MINUTES)
.build();
public static @NonNull String get(@NonNull String url,
@Nullable Map<String, String> httpHeaders,
@Nullable Map<String, Object> requestParams) throws IOException {
private static OkHttpClient CLIENT = new OkHttpClient();
/**
* http get request
* @param connectTimeout connect timeout in milliseconds
* @param writeTimeout write timeout in milliseconds
* @param readTimeout read timeout in milliseconds
* @return OkHttpResponse
* @throws RuntimeException
*/
public static @NonNull OkHttpResponse get(@NonNull String url,
@Nullable OkHttpRequestHeaders okHttpRequestHeaders,
@Nullable Map<String, Object> requestParams,
int connectTimeout,
int writeTimeout,
int readTimeout) throws IOException {
OkHttpClient client = getHttpClient(connectTimeout, writeTimeout, readTimeout);
String finalUrl = addUrlParams(requestParams, url);
Request.Builder requestBuilder = new Request.Builder().url(finalUrl);
addHeader(httpHeaders, requestBuilder);
addHeader(okHttpRequestHeaders.getHeaders(), requestBuilder);
Request request = requestBuilder.build();
try (Response response = CLIENT.newCall(request).execute()) {
return getResponseBody(response);
try (Response response = client.newCall(request).execute()) {
return new OkHttpResponse(response.code(), getResponseBody(response));
} catch (Exception e) {
throw new RuntimeException(String.format("Get request execute failed, url: %s", url), e);
}
}
public static @NonNull String post(@NonNull String url,
@Nullable Map<String, String> httpHeaders,
@Nullable Map<String, Object> requestParamsMap,
@Nullable Map<String, Object> requestBodyMap) throws IOException {
/**
* http post request
* @param connectTimeout connect timeout in milliseconds
* @param writeTimeout write timeout in milliseconds
* @param readTimeout read timeout in milliseconds
* @return OkHttpResponse
* @throws RuntimeException
*/
public static @NonNull OkHttpResponse post(@NonNull String url,
@Nullable OkHttpRequestHeaders okHttpRequestHeaders,
@Nullable Map<String, Object> requestParamsMap,
@Nullable Map<String, Object> requestBodyMap,
int connectTimeout,
int writeTimeout,
int readTimeout) throws IOException {
OkHttpClient client = getHttpClient(connectTimeout, writeTimeout, readTimeout);
String finalUrl = addUrlParams(requestParamsMap, url);
Request.Builder requestBuilder = new Request.Builder().url(finalUrl);
addHeader(httpHeaders, requestBuilder);
addHeader(okHttpRequestHeaders.getHeaders(), requestBuilder);
if (requestBodyMap != null) {
requestBuilder = requestBuilder.post(RequestBody.create(
JSONUtils.toJsonString(requestBodyMap),
MediaType.parse(okHttpRequestHeaders.getOkHttpRequestHeaderContentType().getValue())));
}
try (Response response = client.newCall(requestBuilder.build()).execute()) {
return new OkHttpResponse(response.code(), getResponseBody(response));
} catch (Exception e) {
throw new RuntimeException(String.format("Post request execute failed, url: %s", url), e);
}
}
/**
* http put request
* @param connectTimeout connect timeout in milliseconds
* @param writeTimeout write timeout in milliseconds
* @param readTimeout read timeout in milliseconds
* @return OkHttpResponse
* @throws RuntimeException
*/
public static @NonNull OkHttpResponse put(@NonNull String url,
@Nullable OkHttpRequestHeaders okHttpRequestHeaders,
@Nullable Map<String, Object> requestBodyMap,
int connectTimeout,
int writeTimeout,
int readTimeout) throws IOException {
OkHttpClient client = getHttpClient(connectTimeout, writeTimeout, readTimeout);
Request.Builder requestBuilder = new Request.Builder().url(url);
addHeader(okHttpRequestHeaders.getHeaders(), requestBuilder);
if (requestBodyMap != null) {
requestBuilder = requestBuilder.post(RequestBody.create(MediaType.parse("application/json"),
JSONUtils.toJsonString(requestBodyMap)));
requestBuilder = requestBuilder.put(RequestBody.create(
JSONUtils.toJsonString(requestBodyMap),
MediaType.parse(okHttpRequestHeaders.getOkHttpRequestHeaderContentType().getValue())));
}
try (Response response = CLIENT.newCall(requestBuilder.build()).execute()) {
return getResponseBody(response);
try (Response response = client.newCall(requestBuilder.build()).execute()) {
return new OkHttpResponse(response.code(), getResponseBody(response));
} catch (Exception e) {
throw new RuntimeException(String.format("Put request execute failed, url: %s", url), e);
}
}
/**
* http delete request
* @param connectTimeout connect timeout in milliseconds
* @param writeTimeout write timeout in milliseconds
* @param readTimeout read timeout in milliseconds
* @return OkHttpResponse
* @throws RuntimeException
*/
public static @NonNull OkHttpResponse delete(@NonNull String url,
@Nullable OkHttpRequestHeaders okHttpRequestHeaders,
int connectTimeout,
int writeTimeout,
int readTimeout) throws IOException {
OkHttpClient client = getHttpClient(connectTimeout, writeTimeout, readTimeout);
Request.Builder requestBuilder = new Request.Builder().url(url);
addHeader(okHttpRequestHeaders.getHeaders(), requestBuilder);
requestBuilder = requestBuilder.delete();
try (Response response = client.newCall(requestBuilder.build()).execute()) {
return new OkHttpResponse(response.code(), getResponseBody(response));
} catch (Exception e) {
throw new RuntimeException(String.format("Delete request execute failed, url: %s", url), e);
}
}
@ -73,15 +156,16 @@ public class OkHttpUtils {
@Nullable String token,
@Nullable Map<String, Object> requestBodyMap) throws IOException {
StringBuffer stringBuffer = new StringBuffer();
StringBuilder stringBuffer = new StringBuilder();
if (requestBodyMap != null) {
for (String key : requestBodyMap.keySet()) {
stringBuffer.append(key + "=" + requestBodyMap.get(key) + "&");
stringBuffer.append(key).append("=").append(requestBodyMap.get(key)).append("&");
}
}
RequestBody body =
RequestBody.create(MediaType.parse("application/x-www-form-urlencoded"), stringBuffer.toString());
RequestBody.create(stringBuffer.toString(),
MediaType.parse(OkHttpRequestHeaderContentType.APPLICATION_FORM_URLENCODED.getValue()));
Request request = new Request.Builder()
.url(url)
@ -89,12 +173,14 @@ public class OkHttpUtils {
.addHeader("accpect", "application/json")
.post(body)
.build();
try (Response response = CLIENT.newCall(request).execute()) {
OkHttpClient client = getHttpClient(Constants.HTTP_CONNECT_TIMEOUT, Constants.HTTP_CONNECT_TIMEOUT,
Constants.HTTP_CONNECT_TIMEOUT);
try (Response response = client.newCall(request).execute()) {
return response.body().string();
}
}
private static String addUrlParams(@Nullable Map<String, Object> requestParams, @NonNull String url) {
if (requestParams == null) {
return url;
@ -120,10 +206,20 @@ public class OkHttpUtils {
private static String getResponseBody(@NonNull Response response) throws IOException {
if (response.code() != HttpStatus.SC_OK || response.body() == null) {
throw new RuntimeException(String.format("Request execute failed, httpCode: %s, httpBody: %s",
return String.format("Request execute failed, httpCode: %s, httpBody: %s",
response.code(),
response.body()));
response.body());
}
return response.body().string();
}
private static OkHttpClient getHttpClient(int connectTimeout,
int writeTimeout,
int readTimeout) {
return CLIENT.newBuilder()
.connectTimeout(connectTimeout, TimeUnit.MILLISECONDS)
.writeTimeout(writeTimeout, TimeUnit.MILLISECONDS)
.readTimeout(readTimeout, TimeUnit.MILLISECONDS)
.build();
}
}

6
dolphinscheduler-dist/release-docs/LICENSE vendored

@ -352,7 +352,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
netty 3.10.6.Final: https://github.com/netty/netty, Apache 2.0
netty 4.1.53.Final: https://github.com/netty/netty/blob/netty-4.1.53.Final/LICENSE.txt, Apache 2.0
nimbus-jose-jwt 9.8.1: https://mvnrepository.com/artifact/com.nimbusds/nimbus-jose-jwt/9.8.1, Apache 2.0
okhttp 3.14.9: https://mvnrepository.com/artifact/com.squareup.okhttp/okhttp/3.14.9, Apache 2.0
okhttp 4.12.0: https://mvnrepository.com/artifact/com.squareup.okhttp/okhttp/4.12.0, Apache 2.0
opencsv 2.3: https://mvnrepository.com/artifact/net.sf.opencsv/opencsv/2.3, Apache 2.0
parquet-hadoop-bundle 1.8.1: https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop-bundle/1.8.1, Apache 2.0
poi 4.1.2: https://mvnrepository.com/artifact/org.apache.poi/poi/4.1.2, Apache 2.0
@ -436,8 +436,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
zjsonpatch 0.3.0 https://mvnrepository.com/artifact/io.fabric8/zjsonpatch/0.3.0, Apache 2.0
jackson-dataformat-yaml 2.13.0 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.13.0, Apache 2.0
logging-interceptor 4.9.3 https://mvnrepository.com/artifact/com.squareup.okhttp3/logging-interceptor/4.9.3, Apache 2.0
okhttp 3.14.3 https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp/3.14.3, Apache 2.0
okio 1.17.2 https://mvnrepository.com/artifact/com.squareup.okio/okio/1.17.2, Apache 2.0
okio 3.6.0 https://mvnrepository.com/artifact/com.squareup.okio/okio/3.6.0, Apache 2.0
okio-jvm 3.6.0 https://repo1.maven.org/maven2/com/squareup/okio/okio-jvm/3.6.0, Apache 2.0
hibernate-validator 6.2.2.Final https://mvnrepository.com/artifact/org.hibernate.validator/hibernate-validator/6.2.2.Final, Apache 2.0
jakarta.validation-api 2.0.2 https://mvnrepository.com/artifact/jakarta.validation/jakarta.validation-api/2.0.2, Apache 2.0
jboss-logging:jar 3.4.3.Final https://mvnrepository.com/artifact/org.jboss.logging/jboss-logging/3.4.3.Final, Apache 2.0

202
dolphinscheduler-dist/release-docs/licenses/LICENSE-okhttp.txt vendored

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

31
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/TemplateMethodTransformer.java

@ -1,31 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.HttpLoopTaskMethodDefinition;
import lombok.NonNull;
public interface TemplateMethodTransformer<YamlMethodT extends LoopTaskYamlDefinition.LoopTaskMethodYamlDefinition, MethodT extends HttpLoopTaskMethodDefinition> {
/**
* Transform the {@link LoopTaskYamlDefinition.LoopTaskMethodYamlDefinition} to {@link HttpLoopTaskMethodDefinition}.
*/
@NonNull
MethodT transform(@NonNull YamlMethodT loopTaskAPIYamlDefinition);
}

57
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/BaseHttpTemplateLoopTaskExecutor.java

@ -1,57 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.loop.BaseLoopTaskExecutor;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskInstanceInfo;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskInstanceStatus;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.parser.HttpTaskDefinitionParser;
import javax.annotation.Nullable;
import lombok.NonNull;
public abstract class BaseHttpTemplateLoopTaskExecutor extends BaseLoopTaskExecutor {
private final HttpLoopTaskDefinition httpLoopTaskDefinition;
public BaseHttpTemplateLoopTaskExecutor(@NonNull TaskExecutionContext taskExecutionContext,
@NonNull String taskDefinitionYamlFile) {
super(taskExecutionContext);
this.httpLoopTaskDefinition = new HttpTaskDefinitionParser().parse(taskDefinitionYamlFile);
}
@Override
public @NonNull LoopTaskInstanceInfo submitLoopTask() {
return httpLoopTaskDefinition.getSubmitTaskMethod().submitLoopTask();
}
@Override
public @NonNull LoopTaskInstanceStatus queryTaskInstanceStatus(@NonNull LoopTaskInstanceInfo taskInstanceInfo) {
return httpLoopTaskDefinition.getQueryTaskStateMethod().queryTaskInstanceStatus(taskInstanceInfo);
}
@Override
public void cancelLoopTaskInstance(@Nullable LoopTaskInstanceInfo taskInstanceInfo) {
if (taskInstanceInfo == null) {
return;
}
httpLoopTaskDefinition.getCancelTaskMethod().cancelTaskInstance(taskInstanceInfo);
}
}

65
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskDefinition.java

@ -1,65 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method.HttpLoopTaskCancelTaskMethodDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method.HttpLoopTaskQueryStatusMethodDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method.HttpLoopTaskSubmitTaskMethodDefinition;
import lombok.NonNull;
public class HttpLoopTaskDefinition
implements
LoopTaskDefinition<HttpLoopTaskSubmitTaskMethodDefinition, HttpLoopTaskQueryStatusMethodDefinition, HttpLoopTaskCancelTaskMethodDefinition> {
private final String taskName;
private final HttpLoopTaskSubmitTaskMethodDefinition submitTaskMethod;
private final HttpLoopTaskQueryStatusMethodDefinition queryTaskStateMethod;
private final HttpLoopTaskCancelTaskMethodDefinition cancelTaskMethod;
public HttpLoopTaskDefinition(@NonNull String taskName,
@NonNull HttpLoopTaskSubmitTaskMethodDefinition submitTaskMethod,
@NonNull HttpLoopTaskQueryStatusMethodDefinition queryTaskStateMethod,
@NonNull HttpLoopTaskCancelTaskMethodDefinition cancelTaskMethod) {
this.taskName = taskName;
this.submitTaskMethod = submitTaskMethod;
this.queryTaskStateMethod = queryTaskStateMethod;
this.cancelTaskMethod = cancelTaskMethod;
}
@Override
public @NonNull String getTaskName() {
return taskName;
}
@Override
public @NonNull HttpLoopTaskSubmitTaskMethodDefinition getSubmitTaskMethod() {
return submitTaskMethod;
}
@Override
public @NonNull HttpLoopTaskQueryStatusMethodDefinition getQueryTaskStateMethod() {
return queryTaskStateMethod;
}
@Override
public @NonNull HttpLoopTaskCancelTaskMethodDefinition getCancelTaskMethod() {
return cancelTaskMethod;
}
}

31
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskInstanceInfo.java

@ -1,31 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskInstanceInfo;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class HttpLoopTaskInstanceInfo implements LoopTaskInstanceInfo {
private final String taskInstanceId;
}

35
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskInstanceStatus.java

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskInstanceStatus;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class HttpLoopTaskInstanceStatus implements LoopTaskInstanceStatus {
private final boolean finished;
@Override
public boolean isSuccess() {
return true;
}
}

44
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskMethodDefinition.java

@ -1,44 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskMethodDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskMethodType;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NonNull;
@Data
@AllArgsConstructor
public abstract class HttpLoopTaskMethodDefinition implements LoopTaskMethodDefinition {
protected final String url;
protected final String httpMethodType;
protected final String dataType;
protected final Map<String, String> httpHeaders;
protected final Map<String, Object> requestParams;
protected final Map<String, Object> requestBody;
@Override
public @NonNull LoopTaskMethodType getLoopTaskMethodType() {
return LoopTaskMethodType.HTTP;
}
}

81
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskCancelTaskMethodDefinition.java

@ -1,81 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method;
import org.apache.dolphinscheduler.common.utils.OkHttpUtils;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskCancelMethodDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskInstanceInfo;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.HttpLoopTaskMethodDefinition;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
import javax.annotation.Nullable;
public class HttpLoopTaskCancelTaskMethodDefinition extends HttpLoopTaskMethodDefinition
implements
LoopTaskCancelMethodDefinition {
private final String taskInstanceIdHolder = "${taskInstanceId}";
public HttpLoopTaskCancelTaskMethodDefinition(String url,
String httpMethodType,
String dataType,
Map<String, String> httpHeaders,
Map<String, Object> requestParams,
Map<String, Object> requestBody) {
super(url, httpMethodType, dataType, httpHeaders, requestParams, requestBody);
}
@Override
public void cancelTaskInstance(@Nullable LoopTaskInstanceInfo loopTaskInstanceInfo) {
if (loopTaskInstanceInfo == null) {
return;
}
if (requestParams != null) {
for (Map.Entry<String, Object> entry : requestParams.entrySet()) {
if (StringUtils.equals(entry.getValue().toString(), taskInstanceIdHolder)) {
entry.setValue(loopTaskInstanceInfo.getTaskInstanceId());
}
}
}
if (requestBody != null) {
for (Map.Entry<String, Object> entry : requestBody.entrySet()) {
if (StringUtils.equalsIgnoreCase(entry.getValue().toString(), taskInstanceIdHolder)) {
entry.setValue(loopTaskInstanceInfo.getTaskInstanceId());
}
}
}
try {
if (StringUtils.equalsIgnoreCase("get", httpMethodType)) {
OkHttpUtils.get(url, httpHeaders, requestParams);
} else if (StringUtils.equalsIgnoreCase("post", httpMethodType)) {
OkHttpUtils.post(url, httpHeaders, requestParams, requestBody);
} else {
throw new IllegalArgumentException(String.format("http method type: %s is not supported",
httpMethodType));
}
} catch (IllegalArgumentException ex) {
throw ex;
} catch (Exception ex) {
throw new RuntimeException("Query loop task instance status failed", ex);
}
}
}

88
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskQueryStatusMethodDefinition.java

@ -1,88 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method;
import org.apache.dolphinscheduler.common.utils.OkHttpUtils;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskInstanceInfo;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskInstanceStatus;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskQueryStatusMethodDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.HttpLoopTaskInstanceStatus;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.HttpLoopTaskMethodDefinition;
import org.apache.dolphinscheduler.plugin.task.api.utils.JsonPathUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
import lombok.NonNull;
public class HttpLoopTaskQueryStatusMethodDefinition extends HttpLoopTaskMethodDefinition
implements
LoopTaskQueryStatusMethodDefinition {
private final String taskInstanceFinishedJPath;
// inject the taskInstanceId into template
private final String taskInstanceIdHolder = "${taskInstanceId}";
public HttpLoopTaskQueryStatusMethodDefinition(String url,
String httpMethodType,
String dataType,
Map<String, String> httpHeaders,
Map<String, Object> requestParams,
Map<String, Object> requestBody,
String taskInstanceFinishedJPath) {
super(url, httpMethodType, dataType, httpHeaders, requestParams, requestBody);
this.taskInstanceFinishedJPath = taskInstanceFinishedJPath;
}
@Override
public @NonNull LoopTaskInstanceStatus queryTaskInstanceStatus(@NonNull LoopTaskInstanceInfo loopTaskInstanceInfo) {
// set the loopTaskInstanceId to request params;
if (requestParams != null) {
for (Map.Entry<String, Object> entry : requestParams.entrySet()) {
if (StringUtils.equals(entry.getValue().toString(), taskInstanceIdHolder)) {
entry.setValue(loopTaskInstanceInfo.getTaskInstanceId());
}
}
}
if (requestBody != null) {
for (Map.Entry<String, Object> entry : requestBody.entrySet()) {
if (StringUtils.equalsIgnoreCase(entry.getValue().toString(), taskInstanceIdHolder)) {
entry.setValue(loopTaskInstanceInfo.getTaskInstanceId());
}
}
}
String responseBody;
try {
if (StringUtils.equalsIgnoreCase("get", httpMethodType)) {
responseBody = OkHttpUtils.get(url, httpHeaders, requestParams);
} else if (StringUtils.equalsIgnoreCase("post", httpMethodType)) {
responseBody = OkHttpUtils.post(url, httpHeaders, requestParams, requestBody);
} else {
throw new IllegalArgumentException(String.format("http method type: %s is not supported",
httpMethodType));
}
} catch (IllegalArgumentException ex) {
throw ex;
} catch (Exception ex) {
throw new RuntimeException("Query loop task instance status failed", ex);
}
return new HttpLoopTaskInstanceStatus(JsonPathUtils.exist(responseBody, taskInstanceFinishedJPath));
}
}

76
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskSubmitTaskMethodDefinition.java

@ -1,76 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method;
import org.apache.dolphinscheduler.common.utils.OkHttpUtils;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskInstanceInfo;
import org.apache.dolphinscheduler.plugin.task.api.loop.LoopTaskSubmitTaskMethodDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.HttpLoopTaskInstanceInfo;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.HttpLoopTaskMethodDefinition;
import org.apache.dolphinscheduler.plugin.task.api.utils.JsonPathUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
import java.util.Optional;
import lombok.NonNull;
public class HttpLoopTaskSubmitTaskMethodDefinition extends HttpLoopTaskMethodDefinition
implements
LoopTaskSubmitTaskMethodDefinition {
private final String taskInstanceIdJPath;
public HttpLoopTaskSubmitTaskMethodDefinition(String url,
String httpMethodType,
String dataType,
Map<String, String> httpHeaders,
Map<String, Object> requestParams,
Map<String, Object> requestBody,
@NonNull String taskInstanceIdJPath) {
super(url, httpMethodType, dataType, httpHeaders, requestParams, requestBody);
this.taskInstanceIdJPath = taskInstanceIdJPath;
}
@Override
public @NonNull LoopTaskInstanceInfo submitLoopTask() {
// todo: call http api to submit task
String responseBody;
try {
if (StringUtils.equalsIgnoreCase(httpMethodType, "GET")) {
responseBody = OkHttpUtils.get(url, httpHeaders, requestParams);
} else if (StringUtils.equalsIgnoreCase(httpMethodType, "POST")) {
responseBody = OkHttpUtils.post(url, httpHeaders, requestParams, requestBody);
} else {
throw new IllegalArgumentException(String.format("The request method type: %s is not supported.",
httpMethodType));
}
} catch (IllegalArgumentException ex) {
throw ex;
} catch (Exception ex) {
throw new RuntimeException("Submit loop task error", ex);
}
Optional<String> taskInstanceIdOptional = JsonPathUtils.read(responseBody, taskInstanceIdJPath);
String taskInstanceId = taskInstanceIdOptional.orElseThrow(() -> new RuntimeException(String.format(
"Resolve the taskInstanceId error, responseBody: %s, taskInstanceIdJPath: %s",
responseBody,
taskInstanceIdJPath)));
return new HttpLoopTaskInstanceInfo(taskInstanceId);
}
}

47
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/CancelTemplateMethodTransformer.java

@ -1,47 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http.parser;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.LoopTaskYamlDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.TemplateMethodTransformer;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method.HttpLoopTaskCancelTaskMethodDefinition;
import java.util.Map;
import lombok.NonNull;
public class CancelTemplateMethodTransformer
implements
TemplateMethodTransformer<LoopTaskYamlDefinition.LoopTaskCancelYamlDefinition, HttpLoopTaskCancelTaskMethodDefinition> {
@Override
public @NonNull HttpLoopTaskCancelTaskMethodDefinition transform(@NonNull LoopTaskYamlDefinition.LoopTaskCancelYamlDefinition loopTaskAPIYamlDefinition) {
String url = loopTaskAPIYamlDefinition.getUrl();
String method = loopTaskAPIYamlDefinition.getMethod();
String dataType = loopTaskAPIYamlDefinition.getDataType();
Map<String, String> httpHeaders = loopTaskAPIYamlDefinition.getHttpHeaders();
Map<String, Object> requestParams = loopTaskAPIYamlDefinition.getRequestParams();
Map<String, Object> requestBody = loopTaskAPIYamlDefinition.getRequestBody();
return new HttpLoopTaskCancelTaskMethodDefinition(url,
method,
dataType,
httpHeaders,
requestParams,
requestBody);
}
}

89
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParser.java

@ -1,89 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http.parser;
import org.apache.dolphinscheduler.common.utils.ClassFilterConstructor;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.LoopTaskYamlDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.TaskDefinitionParser;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.HttpLoopTaskDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method.HttpLoopTaskCancelTaskMethodDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method.HttpLoopTaskQueryStatusMethodDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method.HttpLoopTaskSubmitTaskMethodDefinition;
import org.apache.commons.lang3.StringUtils;
import java.io.FileReader;
import java.io.IOException;
import java.util.Map;
import lombok.NonNull;
import org.yaml.snakeyaml.Yaml;
import com.google.common.base.Preconditions;
public class HttpTaskDefinitionParser implements TaskDefinitionParser<HttpLoopTaskDefinition> {
@Override
public @NonNull HttpLoopTaskDefinition parse(@NonNull String yamlConfigFile) {
LoopTaskYamlDefinition loopTaskYamlDefinition;
try {
loopTaskYamlDefinition = parseYamlConfigFile(yamlConfigFile);
} catch (IOException ex) {
throw new IllegalArgumentException(String.format("Parse yaml file: %s error", yamlConfigFile), ex);
}
validateYamlDefinition(loopTaskYamlDefinition);
LoopTaskYamlDefinition.LoopTaskServiceYamlDefinition service = loopTaskYamlDefinition.getService();
LoopTaskYamlDefinition.LoopTaskAPIYamlDefinition api = service.getApi();
HttpLoopTaskSubmitTaskMethodDefinition submitTaskMethod =
new SubmitTemplateMethodTransformer().transform(api.getSubmit());
HttpLoopTaskQueryStatusMethodDefinition queryTaskStateMethod =
new QueryStateTemplateMethodTransformer().transform(api.getQueryState());
HttpLoopTaskCancelTaskMethodDefinition cancelTaskMethod =
new CancelTemplateMethodTransformer().transform(api.getCancel());
return new HttpLoopTaskDefinition(service.getName(), submitTaskMethod, queryTaskStateMethod, cancelTaskMethod);
}
protected @NonNull LoopTaskYamlDefinition parseYamlConfigFile(@NonNull String yamlConfigFile) throws IOException {
try (FileReader fileReader = new FileReader(yamlConfigFile)) {
return new Yaml(new ClassFilterConstructor(new Class[]{
LoopTaskYamlDefinition.class,
LoopTaskYamlDefinition.LoopTaskServiceYamlDefinition.class,
LoopTaskYamlDefinition.LoopTaskAPIYamlDefinition.class,
LoopTaskYamlDefinition.LoopTaskSubmitMethodYamlDefinition.class,
LoopTaskYamlDefinition.LoopTaskQueryStateYamlDefinition.class,
LoopTaskYamlDefinition.LoopTaskCancelYamlDefinition.class,
LoopTaskYamlDefinition.LoopTaskMethodYamlDefinition.class,
LoopTaskYamlDefinition.LoopTaskQueryStateYamlDefinition.class,
Map.class,
String.class
}))
.loadAs(fileReader, LoopTaskYamlDefinition.class);
}
}
protected void validateYamlDefinition(@NonNull LoopTaskYamlDefinition loopTaskYamlDefinition) {
LoopTaskYamlDefinition.LoopTaskServiceYamlDefinition service = loopTaskYamlDefinition.getService();
Preconditions.checkNotNull(service, "service is null");
Preconditions.checkNotNull(service.getName(), "service name is null");
if (!StringUtils.equalsIgnoreCase(service.getType(), "http")) {
throw new IllegalArgumentException(String.format("service type: %s is invalidated", service.getType()));
}
}
}

49
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/QueryStateTemplateMethodTransformer.java

@ -1,49 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http.parser;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.LoopTaskYamlDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.TemplateMethodTransformer;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method.HttpLoopTaskQueryStatusMethodDefinition;
import java.util.Map;
import lombok.NonNull;
public class QueryStateTemplateMethodTransformer
implements
TemplateMethodTransformer<LoopTaskYamlDefinition.LoopTaskQueryStateYamlDefinition, HttpLoopTaskQueryStatusMethodDefinition> {
@Override
public @NonNull HttpLoopTaskQueryStatusMethodDefinition transform(@NonNull LoopTaskYamlDefinition.LoopTaskQueryStateYamlDefinition loopTaskAPIYamlDefinition) {
String url = loopTaskAPIYamlDefinition.getUrl();
String method = loopTaskAPIYamlDefinition.getMethod();
String dataType = loopTaskAPIYamlDefinition.getDataType();
Map<String, String> httpHeaders = loopTaskAPIYamlDefinition.getHttpHeaders();
Map<String, Object> requestParams = loopTaskAPIYamlDefinition.getRequestParams();
Map<String, Object> requestBody = loopTaskAPIYamlDefinition.getRequestBody();
String taskInstanceFinishedJPath = loopTaskAPIYamlDefinition.getTaskInstanceFinishedJPath();
return new HttpLoopTaskQueryStatusMethodDefinition(url,
method,
dataType,
httpHeaders,
requestParams,
requestBody,
taskInstanceFinishedJPath);
}
}

49
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/SubmitTemplateMethodTransformer.java

@ -1,49 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http.parser;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.LoopTaskYamlDefinition;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.TemplateMethodTransformer;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method.HttpLoopTaskSubmitTaskMethodDefinition;
import java.util.Map;
import lombok.NonNull;
public class SubmitTemplateMethodTransformer
implements
TemplateMethodTransformer<LoopTaskYamlDefinition.LoopTaskSubmitMethodYamlDefinition, HttpLoopTaskSubmitTaskMethodDefinition> {
@Override
public @NonNull HttpLoopTaskSubmitTaskMethodDefinition transform(@NonNull LoopTaskYamlDefinition.LoopTaskSubmitMethodYamlDefinition loopTaskAPIYamlDefinition) {
String url = loopTaskAPIYamlDefinition.getUrl();
String method = loopTaskAPIYamlDefinition.getMethod();
String dataType = loopTaskAPIYamlDefinition.getDataType();
Map<String, String> httpHeaders = loopTaskAPIYamlDefinition.getHttpHeaders();
Map<String, Object> requestParams = loopTaskAPIYamlDefinition.getRequestParams();
Map<String, Object> requestBody = loopTaskAPIYamlDefinition.getRequestBody();
String taskInstanceIdJPath = loopTaskAPIYamlDefinition.getTaskInstanceIdJPath();
return new HttpLoopTaskSubmitTaskMethodDefinition(url,
method,
dataType,
httpHeaders,
requestParams,
requestBody,
taskInstanceIdJPath);
}
}

66
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParserTest.java

@ -1,66 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.loop.template.http.parser;
import org.apache.dolphinscheduler.plugin.task.api.loop.template.LoopTaskYamlDefinition;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class HttpTaskDefinitionParserTest {
private static final String yamlFile = HttpTaskDefinitionParserTest.class.getResource("/mock_loop_task.yaml")
.getFile();
@Test
public void parseYamlConfigFile() throws IOException {
LoopTaskYamlDefinition loopTaskYamlDefinition = new HttpTaskDefinitionParser().parseYamlConfigFile(yamlFile);
// check not null
Assertions.assertNotNull(loopTaskYamlDefinition);
Assertions.assertNotNull(loopTaskYamlDefinition.getService());
Assertions.assertNotNull(loopTaskYamlDefinition.getService().getName());
Assertions.assertNotNull(loopTaskYamlDefinition.getService().getType());
Assertions.assertNotNull(loopTaskYamlDefinition.getService().getApi());
Assertions.assertNotNull(loopTaskYamlDefinition.getService().getApi().getSubmit());
Assertions.assertNotNull(loopTaskYamlDefinition.getService().getApi().getQueryState());
Assertions.assertNotNull(loopTaskYamlDefinition.getService().getApi().getCancel());
// check data consistency
LoopTaskYamlDefinition.LoopTaskServiceYamlDefinition service = loopTaskYamlDefinition.getService();
Assertions.assertEquals("MockService", service.getName());
Assertions.assertEquals("Http", service.getType());
Map<String, String> expectedHeaders = new HashMap<>();
expectedHeaders.put("Content-Type", "text/html");
expectedHeaders.put("Content-Length", "1234");
Assertions.assertEquals("/api/v1/submit", service.getApi().getSubmit().getUrl());
Assertions.assertEquals(expectedHeaders, service.getApi().getSubmit().getHttpHeaders());
}
@Test
public void validateYamlDefinition() throws IOException {
HttpTaskDefinitionParser httpTaskDefinitionParser = new HttpTaskDefinitionParser();
LoopTaskYamlDefinition loopTaskYamlDefinition = httpTaskDefinitionParser.parseYamlConfigFile(yamlFile);
httpTaskDefinitionParser.validateYamlDefinition(loopTaskYamlDefinition);
// if no exception assert true
Assertions.assertTrue(true);
}
}

20
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpMethod.java → dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpConstants.java

@ -17,16 +17,12 @@
package org.apache.dolphinscheduler.plugin.task.http;
/**
* http method
*/
public enum HttpMethod {
/**
* 0 get
* 1 post
* 2 head
* 3 put
* 4 delete
*/
GET, POST, HEAD, PUT, DELETE
import lombok.experimental.UtilityClass;
@UtilityClass
public class HttpConstants {
public static final String CONTENT_TYPE = "Content-Type";
public static final int RESPONSE_CODE_SUCCESS = 200;
}

113
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java

@ -17,47 +17,37 @@
package org.apache.dolphinscheduler.plugin.task.http;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
import lombok.EqualsAndHashCode;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* http parameter
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class HttpParameters extends AbstractParameters {
/**
* url
*/
private String url;
/**
* httpMethod
*/
private HttpMethod httpMethod;
@JsonProperty("httpMethod")
private HttpRequestMethod httpRequestMethod;
/**
* http params
*/
private List<HttpProperty> httpParams;
@JsonProperty("httpParams")
private List<HttpProperty> httpRequestParams;
/**
* httpBody
*/
private String httpBody;
@JsonProperty("httpBody")
private String httpRequestBody;
/**
* httpCheckCondition
*/
private HttpCheckCondition httpCheckCondition = HttpCheckCondition.STATUS_CODE_DEFAULT;
/**
* condition
*/
private String condition;
/**
@ -66,83 +56,10 @@ public class HttpParameters extends AbstractParameters {
*/
private int connectTimeout;
/**
* Socket Timeout
* Unit: ms
*/
private int socketTimeout;
@Override
public boolean checkParameters() {
return StringUtils.isNotEmpty(url);
}
@Override
public List<ResourceInfo> getResourceFilesList() {
return new ArrayList<>();
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public HttpMethod getHttpMethod() {
return httpMethod;
}
public void setHttpMethod(HttpMethod httpMethod) {
this.httpMethod = httpMethod;
}
public List<HttpProperty> getHttpParams() {
return httpParams;
}
public void setHttpParams(List<HttpProperty> httpParams) {
this.httpParams = httpParams;
}
public HttpCheckCondition getHttpCheckCondition() {
return httpCheckCondition;
}
public void setHttpCheckCondition(HttpCheckCondition httpCheckCondition) {
this.httpCheckCondition = httpCheckCondition;
}
public String getCondition() {
return condition;
return StringUtils.isNotEmpty(url) && httpRequestMethod != null
&& connectTimeout > 0;
}
public void setCondition(String condition) {
this.condition = condition;
}
public int getConnectTimeout() {
return connectTimeout;
}
public void setConnectTimeout(int connectTimeout) {
this.connectTimeout = connectTimeout;
}
public int getSocketTimeout() {
return socketTimeout;
}
public void setSocketTimeout(int socketTimeout) {
this.socketTimeout = socketTimeout;
}
public String getHttpBody() {
return httpBody;
}
public void setHttpBody(String httpBody) {
this.httpBody = httpBody;
}
}

7
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java

@ -21,10 +21,5 @@ package org.apache.dolphinscheduler.plugin.task.http;
* http parameters type
*/
public enum HttpParametersType {
/**
* 0 parameter;
* 1 body;
* 2 headers;
*/
PARAMETER, BODY, HEADERS
PARAMETER, HEADERS
}

84
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java

@ -19,80 +19,23 @@ package org.apache.dolphinscheduler.plugin.task.http;
import java.util.Objects;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class HttpProperty {
/**
* key
*/
private String prop;
/**
* httpParametersType
*/
private HttpParametersType httpParametersType;
/**
* value
*/
private String value;
public HttpProperty() {
}
public HttpProperty(String prop, HttpParametersType httpParametersType, String value) {
this.prop = prop;
this.httpParametersType = httpParametersType;
this.value = value;
}
/**
* getter method
*
* @return the prop
* @see HttpProperty#prop
*/
public String getProp() {
return prop;
}
/**
* setter method
*
* @param prop the prop to set
* @see HttpProperty#prop
*/
public void setProp(String prop) {
this.prop = prop;
}
/**
* getter method
*
* @return the value
* @see HttpProperty#value
*/
public String getValue() {
return value;
}
/**
* setter method
*
* @param value the value to set
* @see HttpProperty#value
*/
public void setValue(String value) {
this.value = value;
}
public HttpParametersType getHttpParametersType() {
return httpParametersType;
}
public void setHttpParametersType(HttpParametersType httpParametersType) {
this.httpParametersType = httpParametersType;
}
@Override
public boolean equals(Object o) {
if (this == o) {
@ -111,13 +54,4 @@ public class HttpProperty {
return Objects.hash(prop, value);
}
@Override
public String toString() {
return "HttpProperty{"
+ "prop='" + prop + '\''
+ ", httpParametersType=" + httpParametersType
+ ", value='" + value + '\''
+ '}';
}
}

8
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskConstants.java → dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpRequestMethod.java

@ -17,7 +17,9 @@
package org.apache.dolphinscheduler.plugin.task.http;
public class HttpTaskConstants {
public static final String APPLICATION_JSON = "application/json";
/**
* http method
*/
public enum HttpRequestMethod {
GET, POST, PUT, DELETE
}

390
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java

@ -17,8 +17,12 @@
package org.apache.dolphinscheduler.plugin.task.http;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaderContentType;
import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaders;
import org.apache.dolphinscheduler.common.model.OkHttpResponse;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OkHttpUtils;
import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
@ -29,45 +33,19 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.ParseException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import com.fasterxml.jackson.databind.node.ObjectNode;
@Slf4j
public class HttpTask extends AbstractTask {
/**
* output
*/
protected String output;
/**
* http parameters
*/
private HttpParameters httpParameters;
/**
* taskExecutionContext
*/
private TaskExecutionContext taskExecutionContext;
/**
@ -83,7 +61,7 @@ public class HttpTask extends AbstractTask {
@Override
public void init() {
this.httpParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), HttpParameters.class);
log.info("Initialize http task params {}", JSONUtils.toPrettyJsonString(httpParameters));
log.info("Initialize http task params: {}", JSONUtils.toPrettyJsonString(httpParameters));
if (httpParameters == null || !httpParameters.checkParameters()) {
throw new RuntimeException("http task params is not valid");
@ -92,250 +70,192 @@ public class HttpTask extends AbstractTask {
@Override
public void handle(TaskCallBack taskCallBack) throws TaskException {
long startTime = System.currentTimeMillis();
String formatTimeStamp = DateUtils.formatTimeStamp(startTime);
String statusCode = null;
String body = null;
try (
CloseableHttpClient client = createHttpClient();
CloseableHttpResponse response = sendRequest(client)) {
statusCode = String.valueOf(getStatusCode(response));
body = getResponseBody(response);
exitStatusCode = validResponse(body, statusCode);
addDefaultOutput(body);
long costTime = System.currentTimeMillis() - startTime;
log.info(
"startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {} milliseconds, statusCode : {}, body : {}, log : {}",
formatTimeStamp, httpParameters.getUrl(),
httpParameters.getHttpMethod(), costTime, statusCode, body, output);
} catch (Exception e) {
appendMessage(e.toString());
exitStatusCode = -1;
log.error("httpUrl[" + httpParameters.getUrl() + "] connection failed:" + output, e);
throw new TaskException("Execute http task failed", e);
}
OkHttpResponse httpResponse = sendRequest();
validateResponse(httpResponse.getBody(), httpResponse.getStatusCode());
}
@Override
public void cancel() throws TaskException {
}
/**
* send request
*
* @param client client
* @return CloseableHttpResponse
* @throws IOException io exception
*/
protected CloseableHttpResponse sendRequest(CloseableHttpClient client) throws IOException {
RequestBuilder builder = createRequestBuilder();
// replace placeholder,and combine local and global parameters
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
List<HttpProperty> httpPropertyList = new ArrayList<>();
if (CollectionUtils.isNotEmpty(httpParameters.getHttpParams())) {
for (HttpProperty httpProperty : httpParameters.getHttpParams()) {
String jsonObject = JSONUtils.toJsonString(httpProperty);
String params =
ParameterUtils.convertParameterPlaceholders(jsonObject, ParameterUtils.convert(paramsMap));
log.info("http request params:{}", params);
httpPropertyList.add(JSONUtils.parseObject(params, HttpProperty.class));
}
}
String httpBody = ParameterUtils.convertParameterPlaceholders(httpParameters.getHttpBody(),
ParameterUtils.convert(paramsMap));
addRequestParams(builder, httpPropertyList, httpBody);
String requestUrl =
ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParameterUtils.convert(paramsMap));
HttpUriRequest request = builder.setUri(requestUrl).build();
setHeaders(request, httpPropertyList);
return client.execute(request);
}
/**
* get response body
*
* @param httpResponse http response
* @return response body
* @throws ParseException parse exception
* @throws IOException io exception
*/
protected String getResponseBody(CloseableHttpResponse httpResponse) throws ParseException, IOException {
if (httpResponse == null) {
return null;
}
HttpEntity entity = httpResponse.getEntity();
if (entity == null) {
return null;
}
return EntityUtils.toString(entity, StandardCharsets.UTF_8.name());
}
/**
* get status code
*
* @param httpResponse http response
* @return status code
*/
protected int getStatusCode(CloseableHttpResponse httpResponse) {
return httpResponse.getStatusLine().getStatusCode();
}
/**
* valid response
*
* @param body body
* @param statusCode status code
* @return exit status code
*/
protected int validResponse(String body, String statusCode) {
int exitStatusCode = 0;
private void validateResponse(String body, int statusCode) {
switch (httpParameters.getHttpCheckCondition()) {
case BODY_CONTAINS:
if (StringUtils.isEmpty(body) || !body.contains(httpParameters.getCondition())) {
appendMessage(httpParameters.getUrl() + " doesn contain "
+ httpParameters.getCondition());
exitStatusCode = -1;
log.error("http request failed, url: {}, statusCode: {}, checkCondition: {}, body: {}",
httpParameters.getUrl(), statusCode, HttpCheckCondition.BODY_CONTAINS.name(), body);
exitStatusCode = Constants.EXIT_CODE_FAILURE;
return;
}
break;
case BODY_NOT_CONTAINS:
if (StringUtils.isEmpty(body) || body.contains(httpParameters.getCondition())) {
appendMessage(httpParameters.getUrl() + " contains "
+ httpParameters.getCondition());
exitStatusCode = -1;
log.error("http request failed, url: {}, statusCode: {}, checkCondition: {}, body: {}",
httpParameters.getUrl(), statusCode, HttpCheckCondition.BODY_NOT_CONTAINS.name(), body);
exitStatusCode = Constants.EXIT_CODE_FAILURE;
return;
}
break;
case STATUS_CODE_CUSTOM:
if (!statusCode.equals(httpParameters.getCondition())) {
appendMessage(httpParameters.getUrl() + " statuscode: " + statusCode + ", Must be: "
+ httpParameters.getCondition());
exitStatusCode = -1;
if (statusCode != Integer.parseInt(httpParameters.getCondition())) {
log.error("http request failed, url: {}, statusCode: {}, checkCondition: {}, body: {}",
httpParameters.getUrl(), statusCode, HttpCheckCondition.STATUS_CODE_CUSTOM.name(), body);
exitStatusCode = Constants.EXIT_CODE_FAILURE;
return;
}
break;
default:
if (!"200".equals(statusCode)) {
appendMessage(httpParameters.getUrl() + " statuscode: " + statusCode + ", Must be: 200");
exitStatusCode = -1;
case STATUS_CODE_DEFAULT:
if (HttpConstants.RESPONSE_CODE_SUCCESS != statusCode) {
log.error("http request failed, url: {}, statusCode: {}, checkCondition: {}, body: {}",
httpParameters.getUrl(), statusCode, HttpCheckCondition.STATUS_CODE_DEFAULT.name(), body);
exitStatusCode = Constants.EXIT_CODE_FAILURE;
return;
}
break;
default:
throw new TaskException(String.format("http check condition %s not supported",
httpParameters.getHttpCheckCondition()));
}
return exitStatusCode;
}
public String getOutput() {
return output;
// default success log
log.info("http request success, url: {}, statusCode: {}, body: {}", httpParameters.getUrl(), statusCode, body);
exitStatusCode = Constants.EXIT_CODE_SUCCESS;
}
/**
* append message
*
* @param message message
*/
protected void appendMessage(String message) {
if (output == null) {
output = "";
}
if (message != null && !message.trim().isEmpty()) {
output += message;
private OkHttpResponse sendRequest() {
switch (httpParameters.getHttpRequestMethod()) {
case GET:
return sendGetRequest();
case POST:
return sendPostRequest();
case PUT:
return sendPutRequest();
case DELETE:
return sendDeleteRequest();
default:
throw new TaskException(String.format("http request method %s not supported",
httpParameters.getHttpRequestMethod()));
}
}
/**
* add request params
*
* @param builder buidler
* @param httpPropertyList http property list
*/
protected void addRequestParams(RequestBuilder builder, List<HttpProperty> httpPropertyList, String httpBody) {
if (StringUtils.isNotEmpty(httpBody)) {
builder.setEntity(new StringEntity(
httpBody,
ContentType.create(ContentType.APPLICATION_JSON.getMimeType(),
StandardCharsets.UTF_8)));
}
@SneakyThrows
private OkHttpResponse sendGetRequest() {
OkHttpRequestHeaders okHttpRequestHeaders = new OkHttpRequestHeaders();
okHttpRequestHeaders.setHeaders(getHeaders());
okHttpRequestHeaders.setOkHttpRequestHeaderContentType(getContentType());
Map<String, Object> requestParams = getRequestParams();
OkHttpResponse okHttpResponse = OkHttpUtils.get(httpParameters.getUrl(), okHttpRequestHeaders,
requestParams, httpParameters.getConnectTimeout(),
httpParameters.getConnectTimeout(), httpParameters.getConnectTimeout());
addDefaultOutput(JSONUtils.toJsonString(okHttpResponse));
return okHttpResponse;
}
if (CollectionUtils.isNotEmpty(httpPropertyList)) {
ObjectNode jsonParam = JSONUtils.createObjectNode();
for (HttpProperty property : httpPropertyList) {
if (property.getHttpParametersType() != null) {
if (property.getHttpParametersType().equals(HttpParametersType.PARAMETER)) {
builder.addParameter(property.getProp(), property.getValue());
} else if (property.getHttpParametersType().equals(HttpParametersType.BODY)) {
jsonParam.put(property.getProp(), property.getValue());
}
}
}
if (builder.getEntity() == null) {
builder.setEntity(new StringEntity(
jsonParam.toString(),
ContentType.create(ContentType.APPLICATION_JSON.getMimeType(),
StandardCharsets.UTF_8)));
}
}
@SneakyThrows
private OkHttpResponse sendPostRequest() {
OkHttpRequestHeaders okHttpRequestHeaders = new OkHttpRequestHeaders();
okHttpRequestHeaders.setHeaders(getHeaders());
okHttpRequestHeaders.setOkHttpRequestHeaderContentType(getContentType());
Map<String, Object> requestBody = getRequestBody();
OkHttpResponse okHttpResponse = OkHttpUtils.post(httpParameters.getUrl(), okHttpRequestHeaders, null,
requestBody, httpParameters.getConnectTimeout(),
httpParameters.getConnectTimeout(), httpParameters.getConnectTimeout());
addDefaultOutput(JSONUtils.toJsonString(okHttpResponse));
return okHttpResponse;
}
/**
* set headers
*
* @param request request
* @param httpPropertyList http property list
*/
protected void setHeaders(HttpUriRequest request, List<HttpProperty> httpPropertyList) {
if (CollectionUtils.isNotEmpty(httpPropertyList)) {
for (HttpProperty property : httpPropertyList) {
if (HttpParametersType.HEADERS.equals(property.getHttpParametersType())) {
request.addHeader(property.getProp(), property.getValue());
}
}
@SneakyThrows
private OkHttpResponse sendPutRequest() {
OkHttpRequestHeaders okHttpRequestHeaders = new OkHttpRequestHeaders();
okHttpRequestHeaders.setHeaders(getHeaders());
okHttpRequestHeaders.setOkHttpRequestHeaderContentType(getContentType());
Map<String, Object> requestBody = getRequestBody();
OkHttpResponse okHttpResponse = OkHttpUtils.put(httpParameters.getUrl(), okHttpRequestHeaders,
requestBody, httpParameters.getConnectTimeout(),
httpParameters.getConnectTimeout(), httpParameters.getConnectTimeout());
addDefaultOutput(JSONUtils.toJsonString(okHttpResponse));
return okHttpResponse;
}
@SneakyThrows
private OkHttpResponse sendDeleteRequest() {
OkHttpRequestHeaders okHttpRequestHeaders = new OkHttpRequestHeaders();
okHttpRequestHeaders.setHeaders(getHeaders());
okHttpRequestHeaders.setOkHttpRequestHeaderContentType(getContentType());
OkHttpResponse okHttpResponse = OkHttpUtils.delete(httpParameters.getUrl(), okHttpRequestHeaders,
httpParameters.getConnectTimeout(), httpParameters.getConnectTimeout(),
httpParameters.getConnectTimeout());
addDefaultOutput(JSONUtils.toJsonString(okHttpResponse));
return okHttpResponse;
}
private Map<String, String> getHeaders() {
if (httpParameters.getHttpRequestParams() == null) {
return null;
}
return httpParameters.getHttpRequestParams().stream()
.filter(httpProperty -> httpProperty.getHttpParametersType() != null)
.filter(httpProperty -> httpProperty.getHttpParametersType().equals(HttpParametersType.HEADERS)
&& !httpProperty.getProp().equalsIgnoreCase(HttpConstants.CONTENT_TYPE))
.peek((httpProperty) -> {
httpProperty.setProp(ParameterUtils.convertParameterPlaceholders(httpProperty.getProp(),
ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap())));
httpProperty.setValue(ParameterUtils.convertParameterPlaceholders(httpProperty.getValue(),
ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap())));
})
.collect(Collectors.toMap(HttpProperty::getProp, HttpProperty::getValue));
}
/**
* create http client
*
* @return CloseableHttpClient
*/
protected CloseableHttpClient createHttpClient() {
final RequestConfig requestConfig = requestConfig();
HttpClientBuilder httpClientBuilder;
httpClientBuilder = HttpClients.custom().setDefaultRequestConfig(requestConfig);
return httpClientBuilder.build();
private OkHttpRequestHeaderContentType getContentType() {
if (httpParameters.getHttpRequestParams() == null) {
return OkHttpRequestHeaderContentType.APPLICATION_JSON;
}
return OkHttpRequestHeaderContentType.fromValue(
httpParameters.getHttpRequestParams().stream()
.filter(httpProperty -> httpProperty.getHttpParametersType().equals(HttpParametersType.HEADERS)
&& httpProperty.getProp().equalsIgnoreCase(HttpConstants.CONTENT_TYPE))
.filter(httpProperty -> OkHttpRequestHeaderContentType
.fromValue(httpProperty.getValue()) != null)
.findFirst()
.orElse(HttpProperty.builder().value(OkHttpRequestHeaderContentType.APPLICATION_JSON.getValue())
.build())
.getValue());
}
/**
* request config
*
* @return RequestConfig
*/
private RequestConfig requestConfig() {
return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout())
.setConnectTimeout(httpParameters.getConnectTimeout()).build();
private Map<String, Object> getRequestParams() {
if (httpParameters.getHttpRequestParams() == null) {
return null;
}
return httpParameters.getHttpRequestParams().stream()
.filter(httpProperty -> httpProperty.getHttpParametersType().equals(HttpParametersType.PARAMETER))
.peek((httpProperty) -> {
httpProperty.setProp(ParameterUtils.convertParameterPlaceholders(httpProperty.getProp(),
ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap())));
httpProperty.setValue(ParameterUtils.convertParameterPlaceholders(httpProperty.getValue(),
ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap())));
})
.collect(Collectors.toMap(HttpProperty::getProp, HttpProperty::getValue));
}
/**
* create request builder
*
* @return RequestBuilder
*/
protected RequestBuilder createRequestBuilder() {
if (httpParameters.getHttpMethod().equals(HttpMethod.GET)) {
return RequestBuilder.get();
} else if (httpParameters.getHttpMethod().equals(HttpMethod.POST)) {
return RequestBuilder.post();
} else if (httpParameters.getHttpMethod().equals(HttpMethod.HEAD)) {
return RequestBuilder.head();
} else if (httpParameters.getHttpMethod().equals(HttpMethod.PUT)) {
return RequestBuilder.put();
} else if (httpParameters.getHttpMethod().equals(HttpMethod.DELETE)) {
return RequestBuilder.delete();
} else {
private Map<String, Object> getRequestBody() {
String convertedParams = ParameterUtils.convertParameterPlaceholders(httpParameters.getHttpRequestBody(),
ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap()));
Map<String, String> requestBody = JSONUtils.toMap(convertedParams);
if (requestBody == null) {
return null;
}
return requestBody.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
@Override

15
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersTest.java

@ -30,13 +30,12 @@ public class HttpParametersTest {
@Test
public void testGenerator() {
String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\","
+ "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
+ "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\"}";
HttpParameters httpParameters = JSONUtils.parseObject(paramData, HttpParameters.class);
Assertions.assertEquals(10000, httpParameters.getConnectTimeout());
Assertions.assertEquals(10000, httpParameters.getSocketTimeout());
Assertions.assertEquals("https://www.baidu.com/", httpParameters.getUrl());
Assertions.assertEquals(HttpMethod.GET, httpParameters.getHttpMethod());
Assertions.assertEquals(HttpRequestMethod.GET, httpParameters.getHttpRequestMethod());
Assertions.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT, httpParameters.getHttpCheckCondition());
Assertions.assertEquals("", httpParameters.getCondition());
@ -45,14 +44,13 @@ public class HttpParametersTest {
@Test
public void testCheckParameters() {
String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\","
+ "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
+ "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\"}";
HttpParameters httpParameters = JSONUtils.parseObject(paramData, HttpParameters.class);
Assertions.assertTrue(httpParameters.checkParameters());
Assertions.assertEquals(10000, httpParameters.getConnectTimeout());
Assertions.assertEquals(10000, httpParameters.getSocketTimeout());
Assertions.assertEquals("https://www.baidu.com/", httpParameters.getUrl());
Assertions.assertEquals(HttpMethod.GET, httpParameters.getHttpMethod());
Assertions.assertEquals(HttpRequestMethod.GET, httpParameters.getHttpRequestMethod());
Assertions.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT, httpParameters.getHttpCheckCondition());
Assertions.assertEquals("", httpParameters.getCondition());
@ -61,14 +59,13 @@ public class HttpParametersTest {
@Test
public void testCheckValues() {
String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\","
+ "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
+ "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\"}";
HttpParameters httpParameters = JSONUtils.parseObject(paramData, HttpParameters.class);
Assertions.assertTrue(httpParameters.checkParameters());
Assertions.assertEquals(10000, httpParameters.getConnectTimeout());
Assertions.assertEquals(10000, httpParameters.getSocketTimeout());
Assertions.assertEquals("https://www.baidu.com/", httpParameters.getUrl());
Assertions.assertEquals(HttpMethod.GET, httpParameters.getHttpMethod());
Assertions.assertEquals(HttpRequestMethod.GET, httpParameters.getHttpRequestMethod());
Assertions.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT, httpParameters.getHttpCheckCondition());
Assertions.assertEquals("", httpParameters.getCondition());
Assertions.assertEquals(0, httpParameters.getLocalParametersMap().size());

114
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java

@ -34,9 +34,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import okhttp3.HttpUrl;
import okhttp3.mockwebserver.Dispatcher;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
@ -62,9 +60,7 @@ public class HttpTaskTest {
private static final String APPLICATION_JSON_VALUE = "application/json";
private static final String MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY = "/requestBody/to/responseBody";
private static final String MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY = "/requestParams/to/responseBody";
private static final String DEFAULT_MOCK_PATH = "/test";
private final List<MockWebServer> mockWebServers = new ArrayList<>();
@ -76,26 +72,23 @@ public class HttpTaskTest {
@Test
public void testHandleCheckCodeDefaultSuccess() throws Exception {
HttpTask getHttpTask = generateHttpTask(HttpMethod.GET, HttpStatus.SC_OK);
HttpTask postHttpTask = generateHttpTask(HttpMethod.POST, HttpStatus.SC_OK);
HttpTask headHttpTask = generateHttpTask(HttpMethod.HEAD, HttpStatus.SC_OK);
HttpTask putHttpTask = generateHttpTask(HttpMethod.PUT, HttpStatus.SC_OK);
HttpTask deleteHttpTask = generateHttpTask(HttpMethod.DELETE, HttpStatus.SC_OK);
HttpTask getHttpTask = generateHttpTask(HttpRequestMethod.GET, HttpStatus.SC_OK);
HttpTask postHttpTask = generateHttpTask(HttpRequestMethod.POST, HttpStatus.SC_OK);
HttpTask putHttpTask = generateHttpTask(HttpRequestMethod.PUT, HttpStatus.SC_OK);
HttpTask deleteHttpTask = generateHttpTask(HttpRequestMethod.DELETE, HttpStatus.SC_OK);
getHttpTask.handle(null);
postHttpTask.handle(null);
headHttpTask.handle(null);
putHttpTask.handle(null);
deleteHttpTask.handle(null);
Assertions.assertEquals(EXIT_CODE_SUCCESS, getHttpTask.getExitStatusCode());
Assertions.assertEquals(EXIT_CODE_SUCCESS, postHttpTask.getExitStatusCode());
Assertions.assertEquals(EXIT_CODE_SUCCESS, headHttpTask.getExitStatusCode());
Assertions.assertEquals(EXIT_CODE_SUCCESS, putHttpTask.getExitStatusCode());
Assertions.assertEquals(EXIT_CODE_SUCCESS, deleteHttpTask.getExitStatusCode());
}
@Test
public void testHandleCheckCodeDefaultError() throws Exception {
HttpTask getHttpTask = generateHttpTask(HttpMethod.GET, HttpStatus.SC_BAD_REQUEST);
HttpTask getHttpTask = generateHttpTask(HttpRequestMethod.GET, HttpStatus.SC_BAD_REQUEST);
getHttpTask.handle(null);
Assertions.assertEquals(EXIT_CODE_FAILURE, getHttpTask.getExitStatusCode());
}
@ -103,9 +96,9 @@ public class HttpTaskTest {
@Test
public void testHandleCheckCodeCustom() throws Exception {
String condition = HttpStatus.SC_CREATED + "";
HttpTask httpTask = generateHttpTask(HttpMethod.GET, HttpCheckCondition.STATUS_CODE_CUSTOM,
condition, HttpStatus.SC_CREATED, "");
HttpTask httpErrorTask = generateHttpTask(HttpMethod.GET, HttpCheckCondition.STATUS_CODE_CUSTOM,
HttpTask httpTask = generateHttpTask(HttpRequestMethod.GET, HttpCheckCondition.STATUS_CODE_CUSTOM,
condition, HttpStatus.SC_CREATED, "{\"status\": 201}");
HttpTask httpErrorTask = generateHttpTask(HttpRequestMethod.GET, HttpCheckCondition.STATUS_CODE_CUSTOM,
condition, HttpStatus.SC_OK, "");
httpTask.handle(null);
httpErrorTask.handle(null);
@ -115,9 +108,9 @@ public class HttpTaskTest {
@Test
public void testHandleCheckBodyContains() throws Exception {
HttpTask httpTask = generateHttpTask(HttpMethod.GET, HttpCheckCondition.BODY_CONTAINS,
HttpTask httpTask = generateHttpTask(HttpRequestMethod.GET, HttpCheckCondition.BODY_CONTAINS,
"success", HttpStatus.SC_OK, "{\"status\": \"success\"}");
HttpTask httpErrorTask = generateHttpTask(HttpMethod.GET, HttpCheckCondition.BODY_CONTAINS,
HttpTask httpErrorTask = generateHttpTask(HttpRequestMethod.GET, HttpCheckCondition.BODY_CONTAINS,
"success", HttpStatus.SC_OK, "{\"status\": \"failed\"}");
httpTask.handle(null);
httpErrorTask.handle(null);
@ -127,9 +120,9 @@ public class HttpTaskTest {
@Test
public void testHandleCheckBodyNotContains() throws Exception {
HttpTask httpTask = generateHttpTask(HttpMethod.GET, HttpCheckCondition.BODY_NOT_CONTAINS,
HttpTask httpTask = generateHttpTask(HttpRequestMethod.GET, HttpCheckCondition.BODY_NOT_CONTAINS,
"failed", HttpStatus.SC_OK, "{\"status\": \"success\"}");
HttpTask httpErrorTask = generateHttpTask(HttpMethod.GET, HttpCheckCondition.BODY_NOT_CONTAINS,
HttpTask httpErrorTask = generateHttpTask(HttpRequestMethod.GET, HttpCheckCondition.BODY_NOT_CONTAINS,
"failed", HttpStatus.SC_OK, "{\"status\": \"failed\"}");
httpTask.handle(null);
httpErrorTask.handle(null);
@ -137,37 +130,17 @@ public class HttpTaskTest {
Assertions.assertEquals(EXIT_CODE_FAILURE, httpErrorTask.getExitStatusCode());
}
@Test
public void testHandleWithHttpBodyParams() throws Exception {
List<HttpProperty> httpParams = new ArrayList<>();
HttpProperty property = new HttpProperty();
property.setProp("day");
property.setValue("${day}");
property.setHttpParametersType(HttpParametersType.BODY);
httpParams.add(property);
Map<String, String> prepareParamsMap = new HashMap<>();
prepareParamsMap.put("day", "20220812");
// The MockWebServer will return the request body as response body directly
// So we just need to check if the response body contains string "20220812"
HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, null,
httpParams, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812",
HttpStatus.SC_OK, "");
httpTask.handle(null);
Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode());
}
@Test
public void testHandleWithHttpBody() throws Exception {
String httpBody = "{\"day\": ${day}}";
String httpResponse = "{\"day\": \"20220812\"}";
Map<String, String> prepareParamsMap = new HashMap<>();
prepareParamsMap.put("day", "20220812");
// The MockWebServer will return the request body as response body directly
// So we just need to check if the response body contains string "20220812"
HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, httpBody,
HttpTask httpTask = generateHttpTask(DEFAULT_MOCK_PATH, HttpRequestMethod.POST, httpBody,
null, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812",
HttpStatus.SC_OK, "");
HttpStatus.SC_OK, httpResponse);
httpTask.handle(null);
Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode());
}
@ -183,18 +156,18 @@ public class HttpTaskTest {
Map<String, String> prepareParamsMap = new HashMap<>();
prepareParamsMap.put("day", "20220812");
// The MockWebServer will return the request parameter as response body directly
// So we just need to check if the response body contains string "20220812"
HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY, HttpMethod.POST, null,
String httpResponse = "{\"day\": \"20220812\"}";
HttpTask httpTask = generateHttpTask(DEFAULT_MOCK_PATH, HttpRequestMethod.POST, null,
httpParams, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812",
HttpStatus.SC_OK, "");
HttpStatus.SC_OK, httpResponse);
httpTask.handle(null);
Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode());
}
@Test
public void testAddDefaultOutput() throws Exception {
HttpTask httpTask = generateHttpTask(HttpMethod.GET, HttpStatus.SC_OK);
HttpTask httpTask = generateHttpTask(HttpRequestMethod.GET, HttpStatus.SC_OK);
AbstractParameters httpParameters = httpTask.getParameters();
String response = "{\"status\": \"success\"}";
httpTask.addDefaultOutput(response);
@ -217,25 +190,26 @@ public class HttpTaskTest {
return server.url(path).toString();
}
private HttpTask generateHttpTask(HttpMethod httpMethod, int actualResponseCode) throws IOException {
return generateHttpTask("/test", httpMethod, null, null, null,
private HttpTask generateHttpTask(HttpRequestMethod httpRequestMethod, int actualResponseCode) throws IOException {
return generateHttpTask(DEFAULT_MOCK_PATH, httpRequestMethod, "", new ArrayList<>(), null,
HttpCheckCondition.STATUS_CODE_DEFAULT, "", actualResponseCode, "");
}
private HttpTask generateHttpTask(HttpMethod httpMethod, HttpCheckCondition httpCheckConditionType,
private HttpTask generateHttpTask(HttpRequestMethod httpRequestMethod, HttpCheckCondition httpCheckConditionType,
String condition, int actualResponseCode,
String actualResponseBody) throws IOException {
return generateHttpTask("/test", httpMethod, null, null, null,
return generateHttpTask(DEFAULT_MOCK_PATH, httpRequestMethod, "", new ArrayList<>(), null,
httpCheckConditionType, condition, actualResponseCode, actualResponseBody);
}
private HttpTask generateHttpTask(String mockPath, HttpMethod httpMethod, String httpBody,
private HttpTask generateHttpTask(String mockPath, HttpRequestMethod httpRequestMethod, String httpBody,
List<HttpProperty> httpParams,
Map<String, String> prepareParamsMap, HttpCheckCondition httpCheckConditionType,
String condition, int actualResponseCode,
String actualResponseBody) throws IOException {
String url = withMockWebServer(mockPath, actualResponseCode, actualResponseBody);
String paramData =
generateHttpParameters(url, httpMethod, httpBody, httpParams, httpCheckConditionType, condition);
generateHttpParameters(url, httpRequestMethod, httpBody, httpParams, httpCheckConditionType, condition);
return generateHttpTaskFromParamData(paramData, prepareParamsMap);
}
@ -257,20 +231,19 @@ public class HttpTaskTest {
return httpTask;
}
private String generateHttpParameters(String url, HttpMethod httpMethod, String httpBody,
private String generateHttpParameters(String url, HttpRequestMethod httpRequestMethod, String httpBody,
List<HttpProperty> httpParams,
HttpCheckCondition httpCheckConditionType,
String condition) throws JsonProcessingException {
ObjectMapper mapper = new ObjectMapper();
HttpParameters httpParameters = new HttpParameters();
httpParameters.setUrl(url);
httpParameters.setHttpMethod(httpMethod);
httpParameters.setHttpBody(httpBody);
httpParameters.setHttpRequestMethod(httpRequestMethod);
httpParameters.setHttpRequestBody(httpBody);
httpParameters.setHttpCheckCondition(httpCheckConditionType);
httpParameters.setCondition(condition);
httpParameters.setConnectTimeout(10000);
httpParameters.setSocketTimeout(10000);
httpParameters.setHttpParams(httpParams);
httpParameters.setHttpRequestParams(httpParams);
return mapper.writeValueAsString(httpParameters);
}
@ -283,26 +256,7 @@ public class HttpTaskTest {
.setResponseCode(actualResponseCode)
.setHeader(CONTENT_TYPE, APPLICATION_JSON_VALUE);
if (request.getPath().startsWith(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY)) {
// return request body as mock response body
mockResponse.setBody(request.getBody().readUtf8());
} else if (request.getPath().startsWith(MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY)) {
// return request params as mock response body
ObjectMapper mapper = new ObjectMapper();
HttpUrl httpUrl = request.getRequestUrl();
Set<String> parameterNames = httpUrl.queryParameterNames();
Map<String, String> resBodyMap = new HashMap<>();
parameterNames.forEach(name -> resBodyMap.put(name, httpUrl.queryParameter(name)));
try {
mockResponse.setBody(mapper.writeValueAsString(resBodyMap));
} catch (JsonProcessingException e) {
throw new IllegalArgumentException(e);
}
} else if (request.getPath().startsWith(path)) {
mockResponse.setBody(actualResponseBody);
} else {
mockResponse.setResponseCode(HttpStatus.SC_NOT_FOUND);
}
mockResponse.setBody(actualResponseBody);
return mockResponse;
}
};

4
dolphinscheduler-ui/src/locales/en_US/project.ts

@ -517,15 +517,13 @@ export default {
http_method: 'Http Method',
http_parameters: 'Http Parameters',
http_body: 'Http Body',
http_body_tips:
'Please fill in the http body, if filled, http parameters in the body type will be ignored',
http_body_tips: 'http body(required)',
http_check_condition: 'Http Check Condition',
http_condition: 'Http Condition',
http_condition_tips: 'Please Enter Http Condition',
timeout_settings: 'Timeout Settings',
connect_timeout: 'Connect Timeout',
ms: 'ms',
socket_timeout: 'Socket Timeout',
status_code_default: 'Default response code 200',
status_code_custom: 'Custom response code',
body_contains: 'Content includes',

3
dolphinscheduler-ui/src/locales/zh_CN/project.ts

@ -504,14 +504,13 @@ export default {
http_method: '请求类型',
http_parameters: '请求参数',
http_body: '请求Body',
http_body_tips: '请填写http body,如若填写将忽略请求参数中的body类型参数',
http_body_tips: '请填写http body',
http_check_condition: '校验条件',
http_condition: '校验内容',
http_condition_tips: '请填写校验内容',
timeout_settings: '超时设置',
connect_timeout: '连接超时',
ms: '毫秒',
socket_timeout: 'Socket超时',
status_code_default: '默认响应码200',
status_code_custom: '自定义响应码',
body_contains: '内容包含',

93
dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts

@ -17,9 +17,12 @@
import { useI18n } from 'vue-i18n'
import { useCustomParams } from '.'
import type { IJsonItem } from '../types'
import { ref, watch } from 'vue'
export function useHttp(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
const httpBodySpan = ref(0)
const httpParametersType = ref(GET_HTTP_PARAMETERS_TYPE)
const HTTP_CHECK_CONDITIONS = [
{
@ -40,6 +43,40 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] {
}
]
const resetHttpParametersType = () => {
switch (model.httpMethod) {
case 'GET':
case 'DELETE':
httpParametersType.value = GET_HTTP_PARAMETERS_TYPE
break
case 'POST':
case 'PUT':
httpParametersType.value = POST_HTTP_PARAMETERS_TYPE
break
}
}
const resetSpan = () => {
switch (model.httpMethod) {
case 'GET':
case 'DELETE':
httpBodySpan.value = 0
break
case 'POST':
case 'PUT':
httpBodySpan.value = 24
break
}
}
watch(
() => [model.httpMethod],
() => {
resetSpan()
resetHttpParametersType()
}
)
return [
{
type: 'input',
@ -104,8 +141,8 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] {
type: 'select',
field: 'httpParametersType',
span: 6,
options: POSITIONS,
value: 'PARAMETER'
options: httpParametersType,
value: 'HEADERS'
},
{
type: 'input',
@ -128,12 +165,18 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] {
]
},
{
type: 'input',
type: 'editor',
field: 'httpBody',
name: t('project.node.http_body'),
props: {
type: 'textarea',
placeholder: t('project.node.http_body_tips')
span: httpBodySpan,
validate: {
trigger: ['blur', 'input'],
required: true,
validator(validate, value) {
if (httpBodySpan.value && !value) {
return new Error(t('project.node.http_body_tips'))
}
}
}
},
{
@ -174,29 +217,6 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] {
}
}
},
{
type: 'input-number',
field: 'socketTimeout',
name: t('project.node.socket_timeout'),
span: 12,
props: {
max: Math.pow(7, 10) - 1
},
slots: {
suffix: () => t('project.node.ms')
},
validate: {
trigger: ['input', 'blur'],
validator(validate: any, value: string) {
if (!Number.isInteger(parseInt(value))) {
return new Error(
t('project.node.socket_timeout') +
t('project.node.positive_integer_tips')
)
}
}
}
},
...useCustomParams({
model,
field: 'localParams',
@ -214,10 +234,6 @@ const HTTP_METHODS = [
value: 'POST',
label: 'POST'
},
{
value: 'HEAD',
label: 'HEAD'
},
{
value: 'PUT',
label: 'PUT'
@ -228,15 +244,18 @@ const HTTP_METHODS = [
}
]
const POSITIONS = [
const GET_HTTP_PARAMETERS_TYPE = [
{
value: 'PARAMETER',
label: 'Parameter'
},
{
value: 'BODY',
label: 'Body'
},
value: 'HEADERS',
label: 'Headers'
}
]
const POST_HTTP_PARAMETERS_TYPE = [
{
value: 'HEADERS',
label: 'Headers'

5
tools/dependencies/known-dependencies.txt

@ -251,8 +251,9 @@ netty-transport-native-unix-common-4.1.53.Final.jar
nimbus-jose-jwt-9.8.1.jar
nimbus-jose-jwt-9.10.jar
okhttp-2.7.5.jar
okhttp-4.9.3.jar
okio-2.8.0.jar
okhttp-4.12.0.jar
okio-3.6.0.jar
okio-jvm-3.6.0.jar
opencsv-2.3.jar
oshi-core-6.1.1.jar
paranamer-2.3.jar

Loading…
Cancel
Save