From f6ef6280309b2130ba87499de2b992a1d4c9c9f5 Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Tue, 16 Aug 2022 20:25:17 +0800 Subject: [PATCH] Support reading application properties from kubernetes config map (#11402) --- .../impl/ProcessDefinitionServiceImpl.java | 19 ++++ .../src/main/resources/application.yaml | 2 +- dolphinscheduler-bom/pom.xml | 103 ++++++++--------- dolphinscheduler-dist/release-docs/LICENSE | 104 ++++++++--------- .../runner/StateWheelExecuteThread.java | 2 + .../src/main/resources/application.yaml | 2 +- .../service/process/ProcessServiceImpl.java | 5 +- dolphinscheduler-standalone-server/pom.xml | 60 ++++++---- .../src/main/resources/application.yaml | 2 +- .../task/api/k8s/impl/K8sTaskExecutor.java | 106 +++++++++--------- .../plugin/task/api/utils/K8sUtils.java | 47 ++++---- .../task/api/k8s/K8sTaskExecutorTest.java | 15 +-- pom.xml | 2 +- tools/dependencies/known-dependencies.txt | 60 +++++----- 14 files changed, 281 insertions(+), 248 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index ea8e7e6eae..1c7b2aba27 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -146,6 +146,24 @@ import static org.apache.dolphinscheduler.common.Constants.EMPTY_STRING; import static org.apache.dolphinscheduler.common.Constants.IMPORT_SUFFIX; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.COMPLEX_TASK_TYPES; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SQL; +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; /** * process definition service impl @@ -172,6 +190,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro @Autowired private ProcessDefinitionMapper processDefinitionMapper; + @Lazy @Autowired private ProcessInstanceService processInstanceService; diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 642b7acbbc..23db3389f1 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -65,7 +65,7 @@ spring: properties: org.quartz.threadPool:threadPriority: 5 org.quartz.jobStore.isClustered: true - org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX + org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore org.quartz.scheduler.instanceId: AUTO org.quartz.jobStore.tablePrefix: QRTZ_ org.quartz.jobStore.acquireTriggersWithinLock: true diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml index 746eba31c9..8603c3b116 100644 --- a/dolphinscheduler-bom/pom.xml +++ b/dolphinscheduler-bom/pom.xml @@ -26,7 +26,7 @@ dolphinscheduler-bom pom ${project.artifactId} - + 4.1.53.Final 2.5.6 @@ -48,7 +48,7 @@ 4.1 4.5.13 4.4.15 - 2.10.5 + 2.13.0 1.7.2 1.9.16 1.2.11 @@ -82,15 +82,14 @@ 1.21 3.1.1 2.5.1 - 5.8.0 6.2.2.Final 1.12.160 2.10.13 3.14.9 2.7.0 - + 2020.0.6 - + @@ -106,7 +105,7 @@ netty-all ${netty.version} - + org.springframework.boot @@ -146,7 +145,7 @@ ${spring.version} test - + org.java-websocket Java-WebSocket @@ -168,7 +167,7 @@ mybatis-plus-annotation ${mybatis-plus.version} - + org.quartz-scheduler @@ -180,13 +179,13 @@ cron-utils ${cron-utils.version} - + com.alibaba druid ${druid.version} - + org.apache.zookeeper @@ -245,7 +244,7 @@ curator-test ${curator-test.version} - + commons-codec commons-codec @@ -281,7 +280,7 @@ commons-email ${commons-email.version} - + org.apache.httpcomponents httpclient @@ -308,7 +307,7 @@ jackson-core ${jackson.version} - + io.protostuff @@ -320,33 +319,33 @@ protostuff-runtime ${protostuff.version} - + net.bytebuddy byte-buddy ${byte-buddy.version} - + org.reflections reflections ${reflections.version} - + mysql mysql-connector-java ${mysql-connector.version} test - + com.oracle.database.jdbc ojdbc8 ${oracle-jdbc.version} test - + com.h2database h2 @@ -374,7 +373,7 @@ logback-core ${logback.version} - + org.apache.poi @@ -386,7 +385,7 @@ poi-ooxml ${poi.version} - + org.apache.hadoop @@ -426,37 +425,37 @@ hadoop-yarn-common ${hadoop.version} - + org.apache.commons commons-collections4 ${commons-collections4.version} - + com.google.guava guava ${guava.version} - + org.postgresql postgresql ${postgresql.version} - + org.apache.hive hive-jdbc ${hive-jdbc.version} - + commons-io commons-io ${commons-io.version} - + com.github.oshi oshi-core @@ -476,31 +475,31 @@ - + ru.yandex.clickhouse clickhouse-jdbc ${clickhouse-jdbc.version} - + com.microsoft.sqlserver mssql-jdbc ${mssql-jdbc.version} - + com.facebook.presto presto-jdbc ${presto-jdbc.version} - + javax.servlet servlet-api ${servlet-api.version} - + javax.servlet javax.servlet-api @@ -511,49 +510,49 @@ springfox-swagger2 ${springfox.version} - + io.springfox springfox-swagger-ui ${springfox.version} - + io.swagger swagger-models ${swagger-models.version} - + com.github.xiaoymin swagger-bootstrap-ui ${swagger.version} - + com.github.rholder guava-retrying ${guava-retry.version} - + javax.activation activation ${activation.version} - + com.sun.mail javax.mail ${javax-mail} - + net.sf.py4j py4j ${py4j.version} - + com.google.code.findbugs jsr305 @@ -574,19 +573,14 @@ error_prone_annotations ${error_prone_annotations.version} - - - io.fabric8 - kubernetes-client - ${kubernetes.version} - + org.hibernate.validator hibernate-validator ${hibernate-validator.version} - + com.amazonaws aws-java-sdk-emr @@ -597,26 +591,33 @@ joda-time ${joda-time.version} - + com.amazonaws aws-java-sdk-s3 ${aws-sdk.version} - + com.squareup.okhttp3 okhttp ${okhttp.version} - + com.jayway.jsonpath json-path ${json-path.version} - + + + org.springframework.cloud + spring-cloud-dependencies + ${spring-cloud-dependencies.version} + pom + import + - + diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index e688d655dc..42e201d82c 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -218,7 +218,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. accessors-smart 2.4.7: https://github.com/netplex/json-smart-v2, Apache 2.0 apacheds-i18n 2.0.0-M15: https://mvnrepository.com/artifact/org.apache.directory.server/apacheds-i18n/2.0.0-M15, Apache 2.0 apacheds-kerberos-codec 2.0.0-M15: https://mvnrepository.com/artifact/org.apache.directory.server/apacheds-kerberos-codec/2.0.0-M15, Apache 2.0 - tomcat-embed-el 9.0.54: https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.54, Apache 2.0 + tomcat-embed-el 9.0.55: https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.55, Apache 2.0 api-asn1-api 1.0.0-M20: https://mvnrepository.com/artifact/org.apache.directory.api/api-asn1-api/1.0.0-M20, Apache 2.0 api-util 1.0.0-M20: https://mvnrepository.com/artifact/org.apache.directory.api/api-util/1.0.0-M20, Apache 2.0 audience-annotations 0.5.0: https://mvnrepository.com/artifact/org.apache.yetus/audience-annotations/0.5.0, Apache 2.0 @@ -284,14 +284,14 @@ The text of each license is also included at licenses/LICENSE-[project].txt. httpclient 4.5.13: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient/4.5.13, Apache 2.0 httpcore 4.4.15: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore/4.4.15, Apache 2.0 httpmime 4.5.13: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpmime/4.5.13, Apache 2.0 - jackson-annotations 2.10.5: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.10.5, Apache 2.0 - jackson-core 2.10.5: https://github.com/FasterXML/jackson-core, Apache 2.0 + jackson-annotations 2.13.0: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.13.0, Apache 2.0 + jackson-core 2.13.0: https://github.com/FasterXML/jackson-core, Apache 2.0 jackson-core-asl 1.9.13: https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-core-asl/1.9.13, Apache 2.0 - jackson-databind 2.10.5: https://github.com/FasterXML/jackson-databind, Apache 2.0 - jackson-datatype-jdk8 2.12.5: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.12.5, Apache 2.0 - jackson-datatype-jsr310 2.12.5: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.12.5, Apache 2.0 + jackson-databind 2.13.0: https://github.com/FasterXML/jackson-databind, Apache 2.0 + jackson-datatype-jdk8 2.13.0: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.13.0, Apache 2.0 + jackson-datatype-jsr310 2.13.0: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.13.0, Apache 2.0 jackson-mapper-asl 1.9.13: https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-mapper-asl/1.9.13, Apache 2.0 - jackson-module-parameter-names 2.12.5: https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-parameter-names/2.12.5, Apache 2.0 + jackson-module-parameter-names 2.13.0: https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-parameter-names/2.13.0, Apache 2.0 javax.jdo-3.2.0-m3: https://mvnrepository.com/artifact/org.datanucleus/javax.jdo/3.2.0-m3, Apache 2.0 java-xmlbuilder 0.4 : https://mvnrepository.com/artifact/com.jamesmurty.utils/java-xmlbuilder/0.4, Apache 2.0 jdo-api 3.0.1: https://mvnrepository.com/artifact/javax.jdo/jdo-api/3.0.1, Apache 2.0 @@ -346,27 +346,27 @@ The text of each license is also included at licenses/LICENSE-[project].txt. snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2, Apache 2.0 snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0 SparseBitSet 1.2: https://mvnrepository.com/artifact/com.zaxxer/SparseBitSet/1.2, Apache 2.0 - spring-aop 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-aop/5.3.12, Apache 2.0 + spring-aop 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-aop/5.3.13, Apache 2.0 spring-beans 5.3.19: https://mvnrepository.com/artifact/org.springframework/spring-beans/5.3.19, Apache 2.0 - spring-boot 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.5.6, Apache 2.0 - spring-boot-actuator 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator/2.5.6, Apache 2.0 - spring-boot-actuator-autoconfigure 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator-autoconfigure/2.5.6, Apache 2.0 - spring-boot-configuration-processor 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-configuration-processor/2.5.6, Apache 2.0 - spring-boot-autoconfigure 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.5.6, Apache 2.0 - spring-boot-starter 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.5.6, Apache 2.0 - spring-boot-starter-actuator 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-actuator/2.5.6, Apache 2.0 - spring-boot-starter-aop 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.5.6, Apache 2.0 - spring-boot-starter-jdbc 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.5.6, Apache 2.0 - spring-boot-starter-jetty 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.5.6, Apache 2.0 - spring-boot-starter-json 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.5.6, Apache 2.0 - spring-boot-starter-logging 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.5.6, Apache 2.0 - spring-boot-starter-quartz 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-quartz/2.5.6, Apache 2.0 - spring-boot-starter-web 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.5.6, Apache 2.0 - spring-boot-starter-cache 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-cache/2.5.6, Apache 2.0 + spring-boot 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.6.1, Apache 2.0 + spring-boot-actuator 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator/2.6.1, Apache 2.0 + spring-boot-actuator-autoconfigure 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator-autoconfigure/2.6.1, Apache 2.0 + spring-boot-configuration-processor 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-configuration-processor/2.6.1, Apache 2.0 + spring-boot-autoconfigure 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.6.1, Apache 2.0 + spring-boot-starter 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.6.1, Apache 2.0 + spring-boot-starter-actuator 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-actuator/2.6.1, Apache 2.0 + spring-boot-starter-aop 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.6.1, Apache 2.0 + spring-boot-starter-jdbc 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.6.1, Apache 2.0 + spring-boot-starter-jetty 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.6.1, Apache 2.0 + spring-boot-starter-json 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.6.1, Apache 2.0 + spring-boot-starter-logging 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.6.1, Apache 2.0 + spring-boot-starter-quartz 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-quartz/2.6.1, Apache 2.0 + spring-boot-starter-web 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.6.1, Apache 2.0 + spring-boot-starter-cache 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-cache/2.6.1, Apache 2.0 spring-context 5.3.19: https://mvnrepository.com/artifact/org.springframework/spring-context/5.3.19, Apache 2.0 - spring-context-support 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-context-support/5.3.12, Apache 2.0 + spring-context-support 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-context-support/5.3.13, Apache 2.0 spring-core 5.3.19: https://mvnrepository.com/artifact/org.springframework/spring-core/5.3.19, Apache 2.0 - spring-expression 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-expression/5.3.12, Apache 2.0 + spring-expression 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-expression/5.3.13, Apache 2.0 springfox-core 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-core/2.9.2, Apache 2.0 springfox-schema 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-schema/2.9.2, Apache 2.0 springfox-spi 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-spi/2.9.2, Apache 2.0 @@ -374,18 +374,18 @@ The text of each license is also included at licenses/LICENSE-[project].txt. springfox-swagger2 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger2/2.9.2, Apache 2.0 springfox-swagger-common 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-common/2.9.2, Apache 2.0 springfox-swagger-ui 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-ui/2.9.2, Apache 2.0 - spring-jcl 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.3.12, Apache 2.0 + spring-jcl 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.3.13, Apache 2.0 spring-jdbc 5.3.19: https://mvnrepository.com/artifact/org.springframework/spring-jdbc/5.3.19, Apache 2.0 spring-plugin-core 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-core/1.2.0.RELEASE, Apache 2.0 spring-plugin-metadata 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-metadata/1.2.0.RELEASE, Apache 2.0 spring-tx 5.3.19: https://mvnrepository.com/artifact/org.springframework/spring-tx/5.3.19, Apache 2.0 - spring-web 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-web/5.3.12, Apache 2.0 - spring-webmvc 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.3.12, Apache 2.0 + spring-web 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-web/5.3.13, Apache 2.0 + spring-webmvc 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.3.13, Apache 2.0 swagger-annotations 1.5.20: https://mvnrepository.com/artifact/io.swagger/swagger-annotations/1.5.20, Apache 2.0 swagger-bootstrap-ui 1.9.3: https://mvnrepository.com/artifact/com.github.xiaoymin/swagger-bootstrap-ui/1.9.3, Apache 2.0 swagger-models 1.5.24: https://mvnrepository.com/artifact/io.swagger/swagger-models/1.5.24, Apache 2.0 tephra-api 0.6.0: https://mvnrepository.com/artifact/co.cask.tephra/tephra-api/0.6.0, Apache 2.0 - tomcat-embed-el 9.0.54: https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.54, Apache 2.0 + tomcat-embed-el 9.0.55: https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.55, Apache 2.0 xercesImpl 2.9.1: https://mvnrepository.com/artifact/xerces/xercesImpl/2.9.1, Apache 2.0 xmlbeans 3.1.0: https://mvnrepository.com/artifact/org.apache.xmlbeans/xmlbeans/3.1.0, Apache 2.0 xml-apis 1.3.04: https://mvnrepository.com/artifact/xml-apis/xml-apis/1.3.04, Apache 2.0 and W3C @@ -397,30 +397,30 @@ The text of each license is also included at licenses/LICENSE-[project].txt. protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0 prometheus client_java(simpleclient) 0.12.0: https://github.com/prometheus/client_java, Apache 2.0 snowflake snowflake-2010: https://github.com/twitter-archive/snowflake/tree/snowflake-2010, Apache 2.0 - kubernetes-client 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-client/5.8.0, Apache 2.0 - kubernetes-model-admissionregistration 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-admissionregistration/5.8.0, Apache 2.0 - kubernetes-model-apiextensions 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apiextensions/5.8.0, Apache 2.0 - kubernetes-model-apps 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apps/5.8.0, Apache 2.0 - kubernetes-model-autoscaling 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/5.8.0, Apache 2.0 - kubernetes-model-batch 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/5.8.0, Apache 2.0 - kubernetes-model-certificates 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-certificates/5.8.0, Apache 2.0 - kubernetes-model-common 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-common/5.8.0, Apache 2.0 - kubernetes-model-coordination 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-coordination/5.8.0, Apache 2.0 - kubernetes-model-core 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-core/5.8.0, Apache 2.0 - kubernetes-model-discovery 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-discovery/5.8.0, Apache 2.0 - kubernetes-model-events 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-events/5.8.0, Apache 2.0 - kubernetes-model-extensions 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-extensions/5.8.0, Apache 2.0 - kubernetes-model-flowcontrol 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-flowcontrol/5.8.0, Apache 2.0 - kubernetes-model-metrics 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-metrics/5.8.0, Apache 2.0 - kubernetes-model-networking 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-networking/5.8.0, Apache 2.0 - kubernetes-model-node 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-node/5.8.0, Apache 2.0 - kubernetes-model-policy 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-policy/5.8.0, Apache 2.0 - kubernetes-model-rbac 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-rbac/5.8.0, Apache 2.0 - kubernetes-model-scheduling 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-scheduling/5.8.0, Apache 2.0 - kubernetes-model-storageclass 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-storageclass/5.8.0, Apache 2.0 + kubernetes-client 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-client/4.13.3, Apache 2.0 + kubernetes-model-admissionregistration 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-admissionregistration/4.13.3, Apache 2.0 + kubernetes-model-apiextensions 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apiextensions/4.13.3, Apache 2.0 + kubernetes-model-apps 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apps/4.13.3, Apache 2.0 + kubernetes-model-autoscaling 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/4.13.3, Apache 2.0 + kubernetes-model-batch 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/4.13.3, Apache 2.0 + kubernetes-model-certificates 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-certificates/4.13.3, Apache 2.0 + kubernetes-model-common 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-common/4.13.3, Apache 2.0 + kubernetes-model-coordination 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-coordination/4.13.3, Apache 2.0 + kubernetes-model-core 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-core/4.13.3, Apache 2.0 + kubernetes-model-discovery 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-discovery/4.13.3, Apache 2.0 + kubernetes-model-events 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-events/4.13.3, Apache 2.0 + kubernetes-model-extensions 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-extensions/4.13.3, Apache 2.0 + kubernetes-model-flowcontrol 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-flowcontrol/4.13.3, Apache 2.0 + kubernetes-model-metrics 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-metrics/4.13.3, Apache 2.0 + kubernetes-model-networking 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-networking/4.13.3, Apache 2.0 + kubernetes-model-node 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-node/4.13.3, Apache 2.0 + kubernetes-model-policy 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-policy/4.13.3, Apache 2.0 + kubernetes-model-rbac 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-rbac/4.13.3, Apache 2.0 + kubernetes-model-scheduling 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-scheduling/4.13.3, Apache 2.0 + kubernetes-model-storageclass 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-storageclass/4.13.3, Apache 2.0 zjsonpatch 0.3.0 https://mvnrepository.com/artifact/io.fabric8/zjsonpatch/0.3.0, Apache 2.0 generex 1.0.2 https://mvnrepository.com/artifact/com.github.mifmif/generex/1.0.2, Apache 2.0 - jackson-dataformat-yaml 2.12.5 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.12.5, Apache 2.0 + jackson-dataformat-yaml 2.13.0 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.13.0, Apache 2.0 logging-interceptor 3.14.9 https://mvnrepository.com/artifact/com.squareup.okhttp3/logging-interceptor/3.14.9, Apache 2.0 okhttp 3.14.3 https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp/3.14.3, Apache 2.0 okio 1.17.2 https://mvnrepository.com/artifact/com.squareup.okio/okio/1.17.2, Apache 2.0 @@ -429,7 +429,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. jboss-logging:jar 3.4.2.Final https://mvnrepository.com/artifact/org.jboss.logging/jboss-logging/3.4.2.Final, Apache 2.0 ion-java 1.0.2 https://mvnrepository.com/artifact/software.amazon.ion/ion-java/1.0.2 Apache 2.0 jmespath-java 1.12.160 https://mvnrepository.com/artifact/com.amazonaws/jmespath-java/1.12.160 Apache 2.0 - jackson-dataformat-cbor 2.12.5 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-cbor/2.12.5 Apache 2.0 + jackson-dataformat-cbor 2.13.0 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-cbor/2.13.0 Apache 2.0 aws-java-sdk-emr 1.12.160 https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-emr/1.12.160 Apache 2.0 aws-java-sdk-core 1.12.160 https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-core/1.12.160 Apache 2.0 aws-java-sdk-s3 1.12.160 https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-s3/1.12.160 Apache 2.0 diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java index 3c82a8adc9..2a19b1ca8d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java @@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -79,6 +80,7 @@ public class StateWheelExecuteThread extends BaseDaemonThread { @Autowired private MasterConfig masterConfig; + @Lazy @Autowired private WorkflowExecuteThreadPool workflowExecuteThreadPool; diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index d4a82e1afc..c17bf12620 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -56,7 +56,7 @@ spring: properties: org.quartz.threadPool:threadPriority: 5 org.quartz.jobStore.isClustered: true - org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX + org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore org.quartz.scheduler.instanceId: AUTO org.quartz.jobStore.tablePrefix: QRTZ_ org.quartz.jobStore.acquireTriggersWithinLock: true diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 4972ee393a..b4faf51168 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -266,9 +266,6 @@ public class ProcessServiceImpl implements ProcessService { @Autowired private CuringParamsService curingGlobalParamsService; - @Autowired - private ProcessService processService; - /** * handle Command (construct ProcessInstance from Command) , wrapped in transaction * @@ -1261,7 +1258,7 @@ public class ProcessServiceImpl implements ProcessService { try { // submit task to db // Only want to use transaction here - task = processService.submitTask(processInstance, taskInstance); + task = submitTask(processInstance, taskInstance); if (task != null && task.getId() != 0) { break; } diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml index dd4d756065..fd90e39d24 100644 --- a/dolphinscheduler-standalone-server/pom.xml +++ b/dolphinscheduler-standalone-server/pom.xml @@ -15,18 +15,29 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 - + dolphinscheduler-standalone-server - + + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + org.apache.dolphinscheduler @@ -44,17 +55,17 @@ org.apache.dolphinscheduler dolphinscheduler-alert-server - + org.apache.dolphinscheduler dolphinscheduler-log-server - + com.h2database h2 - + org.apache.curator curator-test @@ -65,20 +76,21 @@ - - - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - + + org.springframework.cloud + spring-cloud-context + + + org.springframework.cloud + spring-cloud-kubernetes-commons + + + org.springframework.cloud + spring-cloud-starter-kubernetes-fabric8-config + + + @@ -96,10 +108,10 @@ dolphinscheduler-standalone-server - package single + package standalone-server @@ -112,7 +124,7 @@ - + docker diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 5640467616..d952cbb5b1 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -49,7 +49,7 @@ spring: properties: org.quartz.threadPool:threadPriority: 5 org.quartz.jobStore.isClustered: true - org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX + org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore org.quartz.scheduler.instanceId: AUTO org.quartz.jobStore.tablePrefix: QRTZ_ org.quartz.jobStore.acquireTriggersWithinLock: true diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java index 79e1c48af3..7800d70bf0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java @@ -32,6 +32,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.NAME_LAB import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RESTART_POLICY; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_INSTANCE_ID; +import io.fabric8.kubernetes.client.KubernetesClientException; import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; @@ -58,20 +59,20 @@ import org.slf4j.Logger; import io.fabric8.kubernetes.api.model.EnvVar; import io.fabric8.kubernetes.api.model.Quantity; import io.fabric8.kubernetes.api.model.ResourceRequirements; -import io.fabric8.kubernetes.api.model.batch.v1.Job; -import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder; -import io.fabric8.kubernetes.api.model.batch.v1.JobStatus; +import io.fabric8.kubernetes.api.model.batch.Job; +import io.fabric8.kubernetes.api.model.batch.JobBuilder; +import io.fabric8.kubernetes.api.model.batch.JobStatus; import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; -import io.fabric8.kubernetes.client.WatcherException; /** * K8sTaskExecutor used to submit k8s task to K8S */ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { + private Job job; public K8sTaskExecutor(Logger logger, TaskExecutionContext taskRequest) { - super(logger,taskRequest); + super(logger, taskRequest); } public Job buildK8sJob(K8sTaskMainParameters k8STaskMainParameters) { @@ -99,7 +100,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { List envVars = new ArrayList<>(); envVars.add(taskInstanceIdVar); if (MapUtils.isNotEmpty(otherParams)) { - for (Map.Entry entry : otherParams.entrySet()) { + for (Map.Entry entry : otherParams.entrySet()) { String param = entry.getKey(); String paramValue = entry.getValue(); EnvVar envVar = new EnvVar(param, paramValue, null); @@ -107,60 +108,58 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { } } return new JobBuilder() - .withApiVersion(API_VERSION) - .withNewMetadata() - .withName(k8sJobName) - .withLabels(labelMap) - .withNamespace(namespaceName) - .endMetadata() - .withNewSpec() - .withTtlSecondsAfterFinished(JOB_TTL_SECONDS) - .withNewTemplate() - .withNewSpec() - .addNewContainer() - .withName(k8sJobName) - .withImage(image) - .withImagePullPolicy(IMAGE_PULL_POLICY) - .withResources(new ResourceRequirements(limitRes, reqRes)) - .withEnv(envVars) - .endContainer() - .withRestartPolicy(RESTART_POLICY) - .endSpec() - .endTemplate() - .withBackoffLimit(retryNum) - .endSpec() - .build(); + .withApiVersion(API_VERSION) + .withNewMetadata() + .withName(k8sJobName) + .withLabels(labelMap) + .withNamespace(namespaceName) + .endMetadata() + .withNewSpec() + .withTtlSecondsAfterFinished(JOB_TTL_SECONDS) + .withNewTemplate() + .withNewSpec() + .addNewContainer() + .withName(k8sJobName) + .withImage(image) + .withImagePullPolicy(IMAGE_PULL_POLICY) + .withResources(new ResourceRequirements(limitRes, reqRes)) + .withEnv(envVars) + .endContainer() + .withRestartPolicy(RESTART_POLICY) + .endSpec() + .endTemplate() + .withBackoffLimit(retryNum) + .endSpec() + .build(); } - public void registerBatchJobWatcher(Job job, String taskInstanceId, TaskResponse taskResponse, K8sTaskMainParameters k8STaskMainParameters) { + public void registerBatchJobWatcher(Job job, String taskInstanceId, TaskResponse taskResponse, + K8sTaskMainParameters k8STaskMainParameters) { CountDownLatch countDownLatch = new CountDownLatch(1); Watcher watcher = new Watcher() { + @Override public void eventReceived(Action action, Job job) { if (action != Action.ADDED) { int jobStatus = getK8sJobStatus(job); - setTaskStatus(jobStatus,taskInstanceId, taskResponse, k8STaskMainParameters); + setTaskStatus(jobStatus, taskInstanceId, taskResponse, k8STaskMainParameters); countDownLatch.countDown(); - } } + } @Override - public void onClose(WatcherException e) { - logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail in k8s: %s",job.getMetadata().getName(),e.getMessage())); + public void onClose(KubernetesClientException e) { + logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail in k8s: %s", job.getMetadata().getName(), + e.getMessage())); taskResponse.setExitStatusCode(EXIT_CODE_FAILURE); countDownLatch.countDown(); } - - @Override - public void onClose() { - logger.warn("Watch gracefully closed"); - } }; Watch watch = null; try { watch = k8sUtils.createBatchJobWatcher(job.getMetadata().getName(), watcher); boolean timeoutFlag = taskRequest.getTaskTimeoutStrategy() == TaskTimeoutStrategy.FAILED - || taskRequest.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED; + || taskRequest.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED; if (timeoutFlag) { Boolean timeout = !(countDownLatch.await(taskRequest.getTaskTimeout(), TimeUnit.SECONDS)); waitTimeout(timeout); @@ -169,11 +168,11 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { } flushLog(taskResponse); } catch (InterruptedException e) { - logger.error("job failed in k8s: {}",e.getMessage(), e); + logger.error("job failed in k8s: {}", e.getMessage(), e); Thread.currentThread().interrupt(); taskResponse.setExitStatusCode(EXIT_CODE_FAILURE); } catch (Exception e) { - logger.error("job failed in k8s: {}",e.getMessage(), e); + logger.error("job failed in k8s: {}", e.getMessage(), e); taskResponse.setExitStatusCode(EXIT_CODE_FAILURE); } finally { if (watch != null) { @@ -186,7 +185,8 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { public TaskResponse run(String k8sParameterStr) throws Exception { TaskResponse result = new TaskResponse(); int taskInstanceId = taskRequest.getTaskInstanceId(); - K8sTaskMainParameters k8STaskMainParameters = JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class); + K8sTaskMainParameters k8STaskMainParameters = + JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class); try { if (null == TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId)) { result.setExitStatusCode(EXIT_CODE_KILL); @@ -197,7 +197,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { return result; } K8sTaskExecutionContext k8sTaskExecutionContext = taskRequest.getK8sTaskExecutionContext(); - String configYaml = k8sTaskExecutionContext.getConfigYaml(); + String configYaml = k8sTaskExecutionContext.getConfigYaml(); k8sUtils.buildClient(configYaml); submitJob2k8s(k8sParameterStr); registerBatchJobWatcher(job, Integer.toString(taskInstanceId), result, k8STaskMainParameters); @@ -220,7 +220,8 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { public void submitJob2k8s(String k8sParameterStr) { int taskInstanceId = taskRequest.getTaskInstanceId(); String taskName = taskRequest.getTaskName().toLowerCase(Locale.ROOT); - K8sTaskMainParameters k8STaskMainParameters = JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class); + K8sTaskMainParameters k8STaskMainParameters = + JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class); try { logger.info("[K8sJobExecutor-{}-{}] start to submit job", taskName, taskInstanceId); job = buildK8sJob(k8STaskMainParameters); @@ -236,7 +237,8 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { @Override public void stopJobOnK8s(String k8sParameterStr) { - K8sTaskMainParameters k8STaskMainParameters = JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class); + K8sTaskMainParameters k8STaskMainParameters = + JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class); String namespaceName = k8STaskMainParameters.getNamespaceName(); String jobName = job.getMetadata().getName(); try { @@ -260,17 +262,21 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { } } - public void setTaskStatus(int jobStatus,String taskInstanceId, TaskResponse taskResponse, K8sTaskMainParameters k8STaskMainParameters) { + public void setTaskStatus(int jobStatus, String taskInstanceId, TaskResponse taskResponse, + K8sTaskMainParameters k8STaskMainParameters) { if (jobStatus == EXIT_CODE_SUCCESS || jobStatus == EXIT_CODE_FAILURE) { if (null == TaskExecutionContextCacheManager.getByTaskInstanceId(Integer.valueOf(taskInstanceId))) { logStringBuffer.append(String.format("[K8sJobExecutor-%s] killed", job.getMetadata().getName())); taskResponse.setExitStatusCode(EXIT_CODE_KILL); } else if (jobStatus == EXIT_CODE_SUCCESS) { - logStringBuffer.append(String.format("[K8sJobExecutor-%s] succeed in k8s", job.getMetadata().getName())); + logStringBuffer + .append(String.format("[K8sJobExecutor-%s] succeed in k8s", job.getMetadata().getName())); taskResponse.setExitStatusCode(EXIT_CODE_SUCCESS); } else { - String errorMessage = k8sUtils.getPodLog(job.getMetadata().getName(), k8STaskMainParameters.getNamespaceName()); - logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail in k8s: %s", job.getMetadata().getName(), errorMessage)); + String errorMessage = + k8sUtils.getPodLog(job.getMetadata().getName(), k8STaskMainParameters.getNamespaceName()); + logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail in k8s: %s", job.getMetadata().getName(), + errorMessage)); taskResponse.setExitStatusCode(EXIT_CODE_FAILURE); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/K8sUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/K8sUtils.java index f2b104875b..89fcbf8627 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/K8sUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/K8sUtils.java @@ -28,8 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.api.model.batch.v1.Job; -import io.fabric8.kubernetes.api.model.batch.v1.JobList; +import io.fabric8.kubernetes.api.model.batch.Job; +import io.fabric8.kubernetes.api.model.batch.JobList; import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.KubernetesClient; @@ -37,40 +37,41 @@ import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; public class K8sUtils { + private static final Logger log = LoggerFactory.getLogger(K8sUtils.class); private KubernetesClient client; public void createJob(String namespace, Job job) { try { - client.batch().v1() - .jobs() - .inNamespace(namespace) - .create(job); + client.batch() + .jobs() + .inNamespace(namespace) + .create(job); } catch (Exception e) { - throw new TaskException("fail to create job",e); + throw new TaskException("fail to create job", e); } } public void deleteJob(String jobName, String namespace) { try { - client.batch().v1() - .jobs() - .inNamespace(namespace) - .withName(jobName) - .delete(); + client.batch() + .jobs() + .inNamespace(namespace) + .withName(jobName) + .delete(); } catch (Exception e) { - throw new TaskException("fail to delete job",e); + throw new TaskException("fail to delete job", e); } } public Boolean jobExist(String jobName, String namespace) { Optional result; try { - JobList jobList = client.batch().v1().jobs().inNamespace(namespace).list(); + JobList jobList = client.batch().jobs().inNamespace(namespace).list(); List jobs = jobList.getItems(); result = jobs.stream() - .filter(job -> job.getMetadata().getName().equals(jobName)) - .findFirst(); + .filter(job -> job.getMetadata().getName().equals(jobName)) + .findFirst(); return result.isPresent(); } catch (Exception e) { throw new TaskException("fail to check job: ", e); @@ -79,10 +80,10 @@ public class K8sUtils { public Watch createBatchJobWatcher(String jobName, Watcher watcher) { try { - return client.batch().v1() - .jobs().withName(jobName).watch(watcher); + return client.batch() + .jobs().withName(jobName).watch(watcher); } catch (Exception e) { - throw new TaskException("fail to register batch job watcher",e); + throw new TaskException("fail to register batch job watcher", e); } } @@ -97,9 +98,9 @@ public class K8sUtils { } } return client.pods().inNamespace(namespace) - .withName(podName) - .tailingLines(LOG_LINES) - .getLog(Boolean.TRUE); + .withName(podName) + .tailingLines(LOG_LINES) + .getLog(Boolean.TRUE); } catch (Exception e) { log.error("fail to getPodLog", e); log.error("response bodies : {}", e.getMessage()); @@ -112,7 +113,7 @@ public class K8sUtils { Config config = Config.fromKubeconfig(configYaml); client = new DefaultKubernetesClient(config); } catch (Exception e) { - throw new TaskException("fail to build k8s ApiClient",e); + throw new TaskException("fail to build k8s ApiClient", e); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java index db30248487..f8af0b64e9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java @@ -16,8 +16,8 @@ */ package org.apache.dolphinscheduler.plugin.task.api.k8s; -import io.fabric8.kubernetes.api.model.batch.v1.Job; -import io.fabric8.kubernetes.api.model.batch.v1.JobStatus; +import io.fabric8.kubernetes.api.model.batch.Job; +import io.fabric8.kubernetes.api.model.batch.JobStatus; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CLUSTER; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL; @@ -64,17 +64,6 @@ public class K8sTaskExecutorTest { job = k8sTaskExecutor.buildK8sJob(k8sTaskMainParameters); } @Test - public void testBuildK8sJobNormal() { - String jobStr = "Job(apiVersion=batch/v1, kind=Job, metadata=ObjectMeta(annotations=null, clusterName=null, creationTimestamp=null, deletionGracePeriodSeconds=null, deletionTimestamp=null, finalizers=[], generateName=null, generation=null, labels={k8s.cn/layer=batch, k8s.cn/name=k8s_task_test-1000}, managedFields=[], name=k8s_task_test-1000, namespace=default, ownerReferences=[], resourceVersion=null, selfLink=null, uid=null, additionalProperties={}), spec=JobSpec(activeDeadlineSeconds=null, backoffLimit=0, completionMode=null, completions=null, manualSelector=null, parallelism=null, selector=null, suspend=null, template=PodTemplateSpec(metadata=null, spec=PodSpec(activeDeadlineSeconds=null, affinity=null, automountServiceAccountToken=null, containers=[Container(args=[], command=[], env=[EnvVar(name=taskInstanceId, value=1000, valueFrom=null, additionalProperties={})], envFrom=[], image=ds-dev, imagePullPolicy=Always, lifecycle=null, livenessProbe=null, name=k8s_task_test-1000, ports=[], readinessProbe=null, resources=ResourceRequirements(limits={memory=20.0Mi, cpu=4.0}, requests={memory=10.0Mi, cpu=2.0}, additionalProperties={}), securityContext=null, startupProbe=null, stdin=null, stdinOnce=null, terminationMessagePath=null, terminationMessagePolicy=null, tty=null, volumeDevices=[], volumeMounts=[], workingDir=null, additionalProperties={})], dnsConfig=null, dnsPolicy=null, enableServiceLinks=null, ephemeralContainers=[], hostAliases=[], hostIPC=null, hostNetwork=null, hostPID=null, hostname=null, imagePullSecrets=[], initContainers=[], nodeName=null, nodeSelector=null, overhead=null, preemptionPolicy=null, priority=null, priorityClassName=null, readinessGates=[], restartPolicy=Never, runtimeClassName=null, schedulerName=null, securityContext=null, serviceAccount=null, serviceAccountName=null, setHostnameAsFQDN=null, shareProcessNamespace=null, subdomain=null, terminationGracePeriodSeconds=null, tolerations=[], topologySpreadConstraints=[], volumes=[], additionalProperties={}), additionalProperties={}), ttlSecondsAfterFinished=300, additionalProperties={}), status=null, additionalProperties={})"; - Assert.assertEquals(jobStr, job.toString()); - } - @Test - public void testGetJobNormal() { - k8sTaskExecutor.setJob(job); - String jobStr = "Job(apiVersion=batch/v1, kind=Job, metadata=ObjectMeta(annotations=null, clusterName=null, creationTimestamp=null, deletionGracePeriodSeconds=null, deletionTimestamp=null, finalizers=[], generateName=null, generation=null, labels={k8s.cn/layer=batch, k8s.cn/name=k8s_task_test-1000}, managedFields=[], name=k8s_task_test-1000, namespace=default, ownerReferences=[], resourceVersion=null, selfLink=null, uid=null, additionalProperties={}), spec=JobSpec(activeDeadlineSeconds=null, backoffLimit=0, completionMode=null, completions=null, manualSelector=null, parallelism=null, selector=null, suspend=null, template=PodTemplateSpec(metadata=null, spec=PodSpec(activeDeadlineSeconds=null, affinity=null, automountServiceAccountToken=null, containers=[Container(args=[], command=[], env=[EnvVar(name=taskInstanceId, value=1000, valueFrom=null, additionalProperties={})], envFrom=[], image=ds-dev, imagePullPolicy=Always, lifecycle=null, livenessProbe=null, name=k8s_task_test-1000, ports=[], readinessProbe=null, resources=ResourceRequirements(limits={memory=20.0Mi, cpu=4.0}, requests={memory=10.0Mi, cpu=2.0}, additionalProperties={}), securityContext=null, startupProbe=null, stdin=null, stdinOnce=null, terminationMessagePath=null, terminationMessagePolicy=null, tty=null, volumeDevices=[], volumeMounts=[], workingDir=null, additionalProperties={})], dnsConfig=null, dnsPolicy=null, enableServiceLinks=null, ephemeralContainers=[], hostAliases=[], hostIPC=null, hostNetwork=null, hostPID=null, hostname=null, imagePullSecrets=[], initContainers=[], nodeName=null, nodeSelector=null, overhead=null, preemptionPolicy=null, priority=null, priorityClassName=null, readinessGates=[], restartPolicy=Never, runtimeClassName=null, schedulerName=null, securityContext=null, serviceAccount=null, serviceAccountName=null, setHostnameAsFQDN=null, shareProcessNamespace=null, subdomain=null, terminationGracePeriodSeconds=null, tolerations=[], topologySpreadConstraints=[], volumes=[], additionalProperties={}), additionalProperties={}), ttlSecondsAfterFinished=300, additionalProperties={}), status=null, additionalProperties={})"; - Assert.assertEquals(jobStr,k8sTaskExecutor.getJob().toString()); - } - @Test public void testGetK8sJobStatusNormal() { JobStatus jobStatus = new JobStatus(); jobStatus.setSucceeded(1); diff --git a/pom.xml b/pom.xml index 7c0bc2cbd5..1084b0622b 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ UTF-8 UTF-8 - 2.5.6 + 2.6.1 1.8 4.12 3.1.12 diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 60bfb995d3..1380796af4 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -81,18 +81,19 @@ httpclient-4.5.13.jar httpcore-4.4.15.jar httpmime-4.5.13.jar j2objc-annotations-1.1.jar -jackson-annotations-2.10.5.jar -jackson-core-2.10.5.jar +jackson-annotations-2.13.0.jar +jackson-core-2.13.0.jar jackson-core-asl-1.9.13.jar -jackson-databind-2.10.5.jar +jackson-databind-2.13.0.jar jackson-datatype-jdk8-2.12.5.jar jackson-datatype-jsr310-2.12.5.jar jackson-mapper-asl-1.9.13.jar -jackson-module-parameter-names-2.12.5.jar +jakarta.activation-api-1.2.2.jar jakarta.annotation-api-1.3.5.jar jakarta.servlet-api-4.0.4.jar jakarta.validation-api-2.0.2.jar jakarta.websocket-api-1.1.2.jar +jakarta.xml.bind-api-2.3.3.jar jamon-runtime-2.3.1.jar java-xmlbuilder-0.4.jar javassist-3.27.0-GA.jar @@ -123,7 +124,7 @@ jna-5.10.0.jar jna-platform-5.10.0.jar joda-time-2.10.13.jar jpam-1.1.jar -jsch-0.1.42.jar +jsch-0.1.55.jar jsp-api-2.1.jar jsqlparser-2.1.jar jsr305-3.0.0.jar @@ -179,7 +180,7 @@ spring-boot-2.5.6.jar spring-boot-actuator-2.5.6.jar spring-boot-actuator-autoconfigure-2.5.6.jar spring-boot-autoconfigure-2.5.6.jar -spring-boot-configuration-processor-2.5.6.jar +spring-boot-configuration-processor-2.6.1.jar spring-boot-starter-2.5.6.jar spring-boot-starter-actuator-2.5.6.jar spring-boot-starter-aop-2.5.6.jar @@ -190,6 +191,9 @@ spring-boot-starter-logging-2.5.6.jar spring-boot-starter-quartz-2.5.6.jar spring-boot-starter-web-2.5.6.jar spring-boot-starter-cache-2.5.6.jar +spring-cloud-context-3.0.6.jar +spring-cloud-kubernetes-commons-2.0.6.jar +spring-cloud-starter-kubernetes-fabric8-config-2.0.6.jar spring-context-5.3.19.jar spring-context-support-5.3.12.jar spring-core-5.3.19.jar @@ -220,31 +224,33 @@ xmlbeans-3.1.0.jar xmlenc-0.52.jar zookeeper-3.4.14.jar Java-WebSocket-1.5.1.jar -kubernetes-client-5.8.0.jar -kubernetes-model-admissionregistration-5.8.0.jar -kubernetes-model-apiextensions-5.8.0.jar -kubernetes-model-apps-5.8.0.jar -kubernetes-model-autoscaling-5.8.0.jar -kubernetes-model-batch-5.8.0.jar -kubernetes-model-certificates-5.8.0.jar -kubernetes-model-common-5.8.0.jar -kubernetes-model-coordination-5.8.0.jar -kubernetes-model-core-5.8.0.jar -kubernetes-model-discovery-5.8.0.jar -kubernetes-model-events-5.8.0.jar -kubernetes-model-extensions-5.8.0.jar -kubernetes-model-flowcontrol-5.8.0.jar -kubernetes-model-metrics-5.8.0.jar -kubernetes-model-networking-5.8.0.jar -kubernetes-model-node-5.8.0.jar -kubernetes-model-policy-5.8.0.jar -kubernetes-model-rbac-5.8.0.jar -kubernetes-model-scheduling-5.8.0.jar -kubernetes-model-storageclass-5.8.0.jar +kubernetes-client-4.13.3.jar +kubernetes-model-admissionregistration-4.13.3.jar +kubernetes-model-apiextensions-4.13.3.jar +kubernetes-model-apps-4.13.3.jar +kubernetes-model-autoscaling-4.13.3.jar +kubernetes-model-batch-4.13.3.jar +kubernetes-model-certificates-4.13.3.jar +kubernetes-model-common-4.13.3.jar +kubernetes-model-coordination-4.13.3.jar +kubernetes-model-core-4.13.3.jar +kubernetes-model-discovery-4.13.3.jar +kubernetes-model-events-4.13.3.jar +kubernetes-model-extensions-4.13.3.jar +kubernetes-model-metrics-4.13.3.jar +kubernetes-model-networking-4.13.3.jar +kubernetes-model-node-4.13.3.jar +kubernetes-model-policy-4.13.3.jar +kubernetes-model-rbac-4.13.3.jar +kubernetes-model-settings-4.13.3.jar +kubernetes-model-scheduling-4.13.3.jar +kubernetes-model-storageclass-4.13.3.jar zjsonpatch-0.3.0.jar automaton-1.11-8.jar generex-1.0.2.jar jackson-dataformat-yaml-2.12.5.jar +jackson-module-jaxb-annotations-2.12.5.jar +jackson-module-parameter-names-2.12.5.jar logging-interceptor-3.14.9.jar okhttp-3.14.9.jar okio-1.17.2.jar