Browse Source

[API-Test] [Checkstyle] Reformat api-test code and enable checkstyle (#10805)

* reformat api-test code

* reformat api-test code

* reformat api-test code

* reformat api-test code

* reformat api-test code

* reformat api-test code

* fix checkstyle
3.1.0-release
xiangzihao 2 years ago committed by GitHub
parent
commit
515b5e3ef6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      .github/workflows/api-test.yml
  2. 4
      dolphinscheduler-api-test/README.md
  3. 3
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/cases/TenantAPITest.java
  4. 4
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseData.java
  5. 4
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseTotalList.java
  6. 1
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/LoginPage.java
  7. 1
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/security/TenantPage.java
  8. 50
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/JSONUtils.java
  9. 38
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/RequestClient.java
  10. 28
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/basic/docker-compose.yaml
  11. 2
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
  12. 2
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
  13. 2
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
  14. 25
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties
  15. 2
      dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml
  16. 3
      dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/Constants.java
  17. 25
      dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinSchedulerExtension.java
  18. 33
      dolphinscheduler-api-test/pom.xml

1
.github/workflows/api-test.yml

@ -112,6 +112,7 @@ jobs:
run: | run: |
./mvnw -B -f dolphinscheduler-api-test/pom.xml -am \ ./mvnw -B -f dolphinscheduler-api-test/pom.xml -am \
-DfailIfNoTests=false \ -DfailIfNoTests=false \
-Dcheckstyle.skip=false \
-Dtest=${{ matrix.case.class }} test -Dtest=${{ matrix.case.class }} test
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
if: always() if: always()

4
dolphinscheduler-api-test/README.md

@ -9,8 +9,8 @@ Every page of DolphinScheduler's api is abstracted into a class for better maint
### Example ### Example
The login page's api is abstracted The login page's api is abstracted
as [`LoginPage`](dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/LoginPage.java), with the as [`LoginPage`](dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/LoginPage.java)
following fields, , with the following fields,
```java ```java
public HttpResponse login(String username, String password) { public HttpResponse login(String username, String password) {

3
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/cases/TenantAPITest.java

@ -19,7 +19,6 @@
package org.apache.dolphinscheduler.api.test.cases; package org.apache.dolphinscheduler.api.test.cases;
import lombok.extern.slf4j.Slf4j;
import org.apache.dolphinscheduler.api.test.core.DolphinScheduler; import org.apache.dolphinscheduler.api.test.core.DolphinScheduler;
import org.apache.dolphinscheduler.api.test.entity.HttpResponse; import org.apache.dolphinscheduler.api.test.entity.HttpResponse;
import org.apache.dolphinscheduler.api.test.entity.LoginResponseData; import org.apache.dolphinscheduler.api.test.entity.LoginResponseData;
@ -28,12 +27,14 @@ import org.apache.dolphinscheduler.api.test.entity.TenantListPagingResponseTotal
import org.apache.dolphinscheduler.api.test.pages.LoginPage; import org.apache.dolphinscheduler.api.test.pages.LoginPage;
import org.apache.dolphinscheduler.api.test.pages.security.TenantPage; import org.apache.dolphinscheduler.api.test.pages.security.TenantPage;
import org.apache.dolphinscheduler.api.test.utils.JSONUtils; import org.apache.dolphinscheduler.api.test.utils.JSONUtils;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import lombok.extern.slf4j.Slf4j;
@DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml") @DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml")
@Slf4j @Slf4j

4
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseData.java

@ -19,12 +19,12 @@
package org.apache.dolphinscheduler.api.test.entity; package org.apache.dolphinscheduler.api.test.entity;
import java.util.List;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.util.List;
@AllArgsConstructor @AllArgsConstructor
@NoArgsConstructor @NoArgsConstructor
@Data @Data

4
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseTotalList.java

@ -19,12 +19,12 @@
package org.apache.dolphinscheduler.api.test.entity; package org.apache.dolphinscheduler.api.test.entity;
import java.util.Date;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.util.Date;
@AllArgsConstructor @AllArgsConstructor
@NoArgsConstructor @NoArgsConstructor
@Data @Data

1
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/LoginPage.java

@ -19,7 +19,6 @@
package org.apache.dolphinscheduler.api.test.pages; package org.apache.dolphinscheduler.api.test.pages;
import org.apache.dolphinscheduler.api.test.entity.HttpResponse; import org.apache.dolphinscheduler.api.test.entity.HttpResponse;
import org.apache.dolphinscheduler.api.test.utils.RequestClient; import org.apache.dolphinscheduler.api.test.utils.RequestClient;

1
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/security/TenantPage.java

@ -19,7 +19,6 @@
package org.apache.dolphinscheduler.api.test.pages.security; package org.apache.dolphinscheduler.api.test.pages.security;
import org.apache.dolphinscheduler.api.test.core.Constants; import org.apache.dolphinscheduler.api.test.core.Constants;
import org.apache.dolphinscheduler.api.test.entity.HttpResponse; import org.apache.dolphinscheduler.api.test.entity.HttpResponse;
import org.apache.dolphinscheduler.api.test.utils.RequestClient; import org.apache.dolphinscheduler.api.test.utils.RequestClient;

50
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/JSONUtils.java

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.api.test.utils; package org.apache.dolphinscheduler.api.test.utils;
import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.charset.StandardCharsets.UTF_8;
import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT;
@ -23,6 +24,8 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKN
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
import org.apache.dolphinscheduler.api.test.core.Constants;
import java.io.IOException; import java.io.IOException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
@ -32,9 +35,9 @@ import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.TimeZone; import java.util.TimeZone;
import org.apache.dolphinscheduler.api.test.core.Constants;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.testcontainers.shaded.org.apache.commons.lang.StringUtils;
import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonParser;
@ -52,7 +55,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode; import com.fasterxml.jackson.databind.node.TextNode;
import com.fasterxml.jackson.databind.type.CollectionType; import com.fasterxml.jackson.databind.type.CollectionType;
import org.testcontainers.shaded.org.apache.commons.lang.StringUtils;
/** /**
* json utils * json utils
@ -62,19 +64,15 @@ public class JSONUtils {
private static final Logger logger = LoggerFactory.getLogger(JSONUtils.class); private static final Logger logger = LoggerFactory.getLogger(JSONUtils.class);
static { static {
logger.info("init timezone: {}",TimeZone.getDefault()); logger.info("init timezone: {}", TimeZone.getDefault());
} }
/** /**
* can use static singleton, inject: just make sure to reuse! * can use static singleton, inject: just make sure to reuse!
*/ */
private static final ObjectMapper objectMapper = new ObjectMapper() private static final ObjectMapper objectMapper =
.configure(FAIL_ON_UNKNOWN_PROPERTIES, false) new ObjectMapper().configure(FAIL_ON_UNKNOWN_PROPERTIES, false).configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true).configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
.configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) .configure(REQUIRE_SETTERS_FOR_GETTERS, true).setTimeZone(TimeZone.getDefault()).setDateFormat(new SimpleDateFormat(Constants.YYYY_MM_DD_HH_MM_SS));
.configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
.configure(REQUIRE_SETTERS_FOR_GETTERS, true)
.setTimeZone(TimeZone.getDefault())
.setDateFormat(new SimpleDateFormat(Constants.YYYY_MM_DD_HH_MM_SS));
private JSONUtils() { private JSONUtils() {
throw new UnsupportedOperationException("Construct JSONUtils"); throw new UnsupportedOperationException("Construct JSONUtils");
@ -95,7 +93,7 @@ public class JSONUtils {
/** /**
* json representation of object * json representation of object
* *
* @param object object * @param object object
* @param feature feature * @param feature feature
* @return object to json string * @return object to json string
*/ */
@ -118,9 +116,9 @@ public class JSONUtils {
* the fields of the specified object are generics, just the object itself should not be a * the fields of the specified object are generics, just the object itself should not be a
* generic type. * generic type.
* *
* @param json the string from which the object is to be deserialized * @param json the string from which the object is to be deserialized
* @param clazz the class of T * @param clazz the class of T
* @param <T> T * @param <T> T
* @return an object of type T from the string * @return an object of type T from the string
* classOfT * classOfT
*/ */
@ -138,11 +136,11 @@ public class JSONUtils {
} }
/** /**
* deserialize * deserialize
* *
* @param src byte array * @param src byte array
* @param clazz class * @param clazz class
* @param <T> deserialize type * @param <T> deserialize type
* @return deserialize type * @return deserialize type
*/ */
public static <T> T parseObject(byte[] src, Class<T> clazz) { public static <T> T parseObject(byte[] src, Class<T> clazz) {
@ -156,9 +154,9 @@ public class JSONUtils {
/** /**
* json to list * json to list
* *
* @param json json string * @param json json string
* @param clazz class * @param clazz class
* @param <T> T * @param <T> T
* @return list * @return list
*/ */
public static <T> List<T> toList(String json, Class<T> clazz) { public static <T> List<T> toList(String json, Class<T> clazz) {
@ -203,7 +201,7 @@ public class JSONUtils {
* node or its child nodes, and returning value it has. * node or its child nodes, and returning value it has.
* If no matching field is found in this node or its descendants, returns null. * If no matching field is found in this node or its descendants, returns null.
* *
* @param jsonNode json node * @param jsonNode json node
* @param fieldName Name of field to look for * @param fieldName Name of field to look for
* @return Value of first matching node found, if any; null if none * @return Value of first matching node found, if any; null if none
*/ */
@ -225,17 +223,18 @@ public class JSONUtils {
* @return json to map * @return json to map
*/ */
public static Map<String, String> toMap(String json) { public static Map<String, String> toMap(String json) {
return parseObject(json, new TypeReference<Map<String, String>>() {}); return parseObject(json, new TypeReference<Map<String, String>>() {
});
} }
/** /**
* json to map * json to map
* *
* @param json json * @param json json
* @param classK classK * @param classK classK
* @param classV classV * @param classV classV
* @param <K> K * @param <K> K
* @param <V> V * @param <V> V
* @return to map * @return to map
*/ */
public static <K, V> Map<K, V> toMap(String json, Class<K> classK, Class<V> classV) { public static <K, V> Map<K, V> toMap(String json, Class<K> classK, Class<V> classV) {
@ -255,7 +254,8 @@ public class JSONUtils {
/** /**
* from the key-value generated json to get the str value no matter the real type of value * from the key-value generated json to get the str value no matter the real type of value
* @param json the json str *
* @param json the json str
* @param nodeName key * @param nodeName key
* @return the str value of key * @return the str value of key
*/ */
@ -315,7 +315,7 @@ public class JSONUtils {
* @param <T> object type * @param <T> object type
* @return byte array * @return byte array
*/ */
public static <T> byte[] toJsonByteArray(T obj) { public static <T> byte[] toJsonByteArray(T obj) {
if (obj == null) { if (obj == null) {
return null; return null;
} }

38
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/RequestClient.java

@ -19,11 +19,14 @@
package org.apache.dolphinscheduler.api.test.utils; package org.apache.dolphinscheduler.api.test.utils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.dolphinscheduler.api.test.core.Constants; import org.apache.dolphinscheduler.api.test.core.Constants;
import org.apache.dolphinscheduler.api.test.entity.HttpResponse; import org.apache.dolphinscheduler.api.test.entity.HttpResponse;
import org.apache.dolphinscheduler.api.test.entity.HttpResponseBody; import org.apache.dolphinscheduler.api.test.entity.HttpResponseBody;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import org.testcontainers.shaded.okhttp3.FormBody; import org.testcontainers.shaded.okhttp3.FormBody;
import org.testcontainers.shaded.okhttp3.Headers; import org.testcontainers.shaded.okhttp3.Headers;
import org.testcontainers.shaded.okhttp3.MediaType; import org.testcontainers.shaded.okhttp3.MediaType;
@ -32,9 +35,8 @@ import org.testcontainers.shaded.okhttp3.Request;
import org.testcontainers.shaded.okhttp3.RequestBody; import org.testcontainers.shaded.okhttp3.RequestBody;
import org.testcontainers.shaded.okhttp3.Response; import org.testcontainers.shaded.okhttp3.Response;
import java.util.HashMap; import lombok.SneakyThrows;
import java.util.Map; import lombok.extern.slf4j.Slf4j;
import java.util.Objects;
@Slf4j @Slf4j
public class RequestClient { public class RequestClient {
@ -56,10 +58,10 @@ public class RequestClient {
LOGGER.info("GET request to {}, Headers: {}", requestUrl, headersBuilder); LOGGER.info("GET request to {}, Headers: {}", requestUrl, headersBuilder);
Request request = new Request.Builder() Request request = new Request.Builder()
.url(requestUrl) .url(requestUrl)
.headers(headersBuilder) .headers(headersBuilder)
.get() .get()
.build(); .build();
Response response = this.httpClient.newCall(request).execute(); Response response = this.httpClient.newCall(request).execute();
@ -111,14 +113,13 @@ public class RequestClient {
LOGGER.info("POST request to {}, Headers: {}, Params: {}", requestUrl, headersBuilder, params); LOGGER.info("POST request to {}, Headers: {}, Params: {}", requestUrl, headersBuilder, params);
Request request = new Request.Builder() Request request = new Request.Builder()
.headers(headersBuilder) .headers(headersBuilder)
.url(requestUrl) .url(requestUrl)
.post(requestBody) .post(requestBody)
.build(); .build();
Response response = this.httpClient.newCall(request).execute(); Response response = this.httpClient.newCall(request).execute();
int responseCode = response.code(); int responseCode = response.code();
HttpResponseBody responseData = null; HttpResponseBody responseData = null;
if (response.body() != null) { if (response.body() != null) {
@ -147,14 +148,13 @@ public class RequestClient {
LOGGER.info("DELETE request to {}, Headers: {}, Params: {}", requestUrl, headersBuilder, params); LOGGER.info("DELETE request to {}, Headers: {}, Params: {}", requestUrl, headersBuilder, params);
Request request = new Request.Builder() Request request = new Request.Builder()
.headers(headersBuilder) .headers(headersBuilder)
.url(requestUrl) .url(requestUrl)
.delete() .delete()
.build(); .build();
Response response = this.httpClient.newCall(request).execute(); Response response = this.httpClient.newCall(request).execute();
int responseCode = response.code(); int responseCode = response.code();
HttpResponseBody responseData = null; HttpResponseBody responseData = null;
if (response.body() != null) { if (response.body() != null) {

28
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/basic/docker-compose.yaml

@ -18,20 +18,20 @@
version: "3.8" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci image: apache/dolphinscheduler-standalone-server:ci
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
ports: ports:
- "12345:12345" - "12345:12345"
networks: networks:
- api-test - api-test
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 60s timeout: 60s
retries: 120 retries: 120
networks: networks:
api-test: api-test:

2
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml

@ -52,7 +52,7 @@ services:
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] test: [ "CMD", "wget", "--spider", "-q", "localhost:8123/ping" ]
interval: 5s interval: 5s
timeout: 60s timeout: 60s
retries: 120 retries: 120

2
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/docker-compose.yaml

@ -107,7 +107,7 @@ services:
expose: expose:
- "5432" - "5432"
healthcheck: healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"] test: [ "CMD-SHELL", "pg_isready -U postgres" ]
interval: 5s interval: 5s
timeout: 60s timeout: 60s
retries: 120 retries: 120

2
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml

@ -45,7 +45,7 @@ services:
expose: expose:
- 5432 - 5432
healthcheck: healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"] test: [ "CMD-SHELL", "pg_isready -U postgres" ]
interval: 5s interval: 5s
timeout: 60s timeout: 60s
retries: 120 retries: 120

25
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties

@ -14,65 +14,45 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
# user data local directory path, please make sure the directory exists and have read write permissions # user data local directory path, please make sure the directory exists and have read write permissions
data.basedir.path=/tmp/dolphinscheduler data.basedir.path=/tmp/dolphinscheduler
# resource storage type: HDFS, S3, NONE # resource storage type: HDFS, S3, NONE
resource.storage.type=S3 resource.storage.type=S3
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended # please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.upload.path=/dolphinscheduler resource.upload.path=/dolphinscheduler
# whether to startup kerberos # whether to startup kerberos
hadoop.security.authentication.startup.state=false hadoop.security.authentication.startup.state=false
# java.security.krb5.conf path # java.security.krb5.conf path
java.security.krb5.conf.path=/opt/krb5.conf java.security.krb5.conf.path=/opt/krb5.conf
# login user from keytab username # login user from keytab username
login.user.keytab.username=hdfs-mycluster@ESZ.COM login.user.keytab.username=hdfs-mycluster@ESZ.COM
# login user from keytab path # login user from keytab path
login.user.keytab.path=/opt/hdfs.headless.keytab login.user.keytab.path=/opt/hdfs.headless.keytab
# kerberos expire time, the unit is hour # kerberos expire time, the unit is hour
kerberos.expire.time=2 kerberos.expire.time=2
# resource view suffixs # resource view suffixs
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
hdfs.root.user=hdfs hdfs.root.user=hdfs
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
fs.defaultFS=s3a://dolphinscheduler fs.defaultFS=s3a://dolphinscheduler
# resourcemanager port, the default value is 8088 if not specified # resourcemanager port, the default value is 8088 if not specified
resource.manager.httpaddress.port=8088 resource.manager.httpaddress.port=8088
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) # job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
# datasource encryption enable # datasource encryption enable
datasource.encryption.enable=false datasource.encryption.enable=false
# datasource encryption salt # datasource encryption salt
datasource.encryption.salt=!@#$%^&* datasource.encryption.salt=!@#$%^&*
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions # use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
sudo.enable=true sudo.enable=true
# network interface preferred like eth0, default: empty # network interface preferred like eth0, default: empty
#dolphin.scheduler.network.interface.preferred= #dolphin.scheduler.network.interface.preferred=
# network IP gets priority, default: inner outer # network IP gets priority, default: inner outer
#dolphin.scheduler.network.priority.strategy=default #dolphin.scheduler.network.priority.strategy=default
# system env path # system env path
@ -85,6 +65,5 @@ aws.access.key.id=accessKey123
aws.secret.access.key=secretKey123 aws.secret.access.key=secretKey123
aws.region=us-east-1 aws.region=us-east-1
aws.endpoint=http://s3:9000 aws.endpoint=http://s3:9000
# Task resource limit state # Task resource limit state
task.resource.limit.state=false task.resource.limit.state=false

2
dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml

@ -53,7 +53,7 @@ services:
MINIO_ROOT_USER: accessKey123 MINIO_ROOT_USER: accessKey123
MINIO_ROOT_PASSWORD: secretKey123 MINIO_ROOT_PASSWORD: secretKey123
healthcheck: healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
interval: 5s interval: 5s
timeout: 120s timeout: 120s
retries: 120 retries: 120

3
dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/Constants.java

@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.api.test.core;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
import java.nio.file.Path;
import java.nio.file.Paths;
@UtilityClass @UtilityClass
public final class Constants { public final class Constants {

25
dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinSchedulerExtension.java

@ -27,7 +27,6 @@ import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import lombok.SneakyThrows;
import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.AfterAllCallback;
import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ExtensionContext;
@ -38,15 +37,15 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCallback { final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCallback {
private final boolean LOCAL_MODE = Objects.equals(System.getProperty("local"), "true"); private final boolean localMode = Objects.equals(System.getProperty("local"), "true");
private final String SERVICE_NAME = "dolphinscheduler_1"; private final String serviceName = "dolphinscheduler_1";
private DockerComposeContainer<?> compose; private DockerComposeContainer<?> compose;
@Override @Override
public void beforeAll(ExtensionContext context) { public void beforeAll(ExtensionContext context) {
if (!LOCAL_MODE) { if (!localMode) {
compose = createDockerCompose(context); compose = createDockerCompose(context);
compose.start(); compose.start();
} }
@ -63,17 +62,17 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
final Class<?> clazz = context.getRequiredTestClass(); final Class<?> clazz = context.getRequiredTestClass();
final DolphinScheduler annotation = clazz.getAnnotation(DolphinScheduler.class); final DolphinScheduler annotation = clazz.getAnnotation(DolphinScheduler.class);
final List<File> files = Stream.of(annotation.composeFiles()) final List<File> files = Stream.of(annotation.composeFiles())
.map(it -> DolphinScheduler.class.getClassLoader().getResource(it)) .map(it -> DolphinScheduler.class.getClassLoader().getResource(it))
.filter(Objects::nonNull) .filter(Objects::nonNull)
.map(URL::getPath) .map(URL::getPath)
.map(File::new) .map(File::new)
.collect(Collectors.toList()); .collect(Collectors.toList());
compose = new DockerComposeContainer<>(files) compose = new DockerComposeContainer<>(files)
.withPull(true) .withPull(true)
.withTailChildContainers(true) .withTailChildContainers(true)
.withLogConsumer(SERVICE_NAME, outputFrame -> LOGGER.info(outputFrame.getUtf8String())) .withLogConsumer(serviceName, outputFrame -> LOGGER.info(outputFrame.getUtf8String()))
.waitingFor(SERVICE_NAME, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(Constants.DOCKER_COMPOSE_DEFAULT_TIMEOUT))); .waitingFor(serviceName, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(Constants.DOCKER_COMPOSE_DEFAULT_TIMEOUT)));
return compose; return compose;
} }

33
dolphinscheduler-api-test/pom.xml

@ -45,6 +45,7 @@
<log4j-slf4j-impl.version>2.17.2</log4j-slf4j-impl.version> <log4j-slf4j-impl.version>2.17.2</log4j-slf4j-impl.version>
<guava.version>31.0.1-jre</guava.version> <guava.version>31.0.1-jre</guava.version>
<jackson.version>2.13.2</jackson.version> <jackson.version>2.13.2</jackson.version>
<checkstyle.version>3.1.2</checkstyle.version>
</properties> </properties>
<dependencies> <dependencies>
@ -137,6 +138,38 @@
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version> <version>2.22.2</version>
</plugin> </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>${checkstyle.version}</version>
<dependencies>
<dependency>
<groupId>com.puppycrawl.tools</groupId>
<artifactId>checkstyle</artifactId>
<version>8.45</version>
</dependency>
</dependencies>
<configuration>
<consoleOutput>true</consoleOutput>
<encoding>UTF-8</encoding>
<configLocation>../style/checkstyle.xml</configLocation>
<failOnViolation>true</failOnViolation>
<violationSeverity>warning</violationSeverity>
<includeTestSourceDirectory>true</includeTestSourceDirectory>
<sourceDirectories>
<sourceDirectory>${project.build.sourceDirectory}</sourceDirectory>
</sourceDirectories>
<excludes>**\/generated-sources\/</excludes>
</configuration>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins> </plugins>
</build> </build>
</project> </project>

Loading…
Cancel
Save