github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Support reading application properties from kubernetes config map

k8s/config
kezhenxu94 2 years ago
parent
26a21566bf
commit
dbdb695c96
No known key found for this signature in database
GPG Key ID: 8BD99F552D9F33D7
14 changed files with 598 additions and 448 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 527
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  2. 2
      dolphinscheduler-api/src/main/resources/application.yaml
  3. 116
      dolphinscheduler-bom/pom.xml
  4. 104
      dolphinscheduler-dist/release-docs/LICENSE
  5. 2
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
  6. 2
      dolphinscheduler-master/src/main/resources/application.yaml
  7. 5
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
  8. 60
      dolphinscheduler-standalone-server/pom.xml
  9. 2
      dolphinscheduler-standalone-server/src/main/resources/application.yaml
  10. 106
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
  11. 47
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/K8sUtils.java
  12. 4
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
  13. 9
      pom.xml
  14. 60
      tools/dependencies/known-dependencies.txt

527
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
View File

File diff suppressed because it is too large Load Diff

2
dolphinscheduler-api/src/main/resources/application.yaml
Unescape View File

@ -65,7 +65,7 @@ spring:
properties:
org.quartz.threadPool:threadPriority: 5
org.quartz.jobStore.isClustered: true
org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX
org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true

116
dolphinscheduler-bom/pom.xml
Unescape View File

@ -15,7 +15,6 @@
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@ -25,9 +24,9 @@
<version>dev-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-bom</artifactId>
<name>${project.artifactId}</name>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<properties>
<netty.version>4.1.53.Final</netty.version>
<spring-boot.version>2.5.6</spring-boot.version>
@ -49,7 +48,7 @@
<commons-collections4.version>4.1</commons-collections4.version>
<httpclient.version>4.5.13</httpclient.version>
<httpcore.version>4.4.15</httpcore.version>
<jackson.version>2.10.5</jackson.version>
<jackson.version>2.13.0</jackson.version>
<protostuff.version>1.7.2</protostuff.version>
<byte-buddy.version>1.9.16</byte-buddy.version>
<logback.version>1.2.11</logback.version>
@ -83,15 +82,14 @@
<commons-compress.version>1.21</commons-compress.version>
<commons-math3.version>3.1.1</commons-math3.version>
<error_prone_annotations.version>2.5.1</error_prone_annotations.version>
<kubernetes.version>5.8.0</kubernetes.version>
<hibernate-validator.version>6.2.2.Final</hibernate-validator.version>
<aws-sdk.version>1.12.160</aws-sdk.version>
<joda-time.version>2.10.13</joda-time.version>
<okhttp.version>3.14.9</okhttp.version>
<json-path.version>2.7.0</json-path.version>
<spring-cloud-dependencies.version>2020.0.6</spring-cloud-dependencies.version>
</properties>
<dependencyManagement>
<dependencies>
<!-- netty -->
@ -99,15 +97,15 @@
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>${netty.version}</version>
<scope>import</scope>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>${netty.version}</version>
</dependency>
<!-- spring -->
<dependency>
<groupId>org.springframework.boot</groupId>
@ -147,7 +145,7 @@
<version>${spring.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.java-websocket</groupId>
<artifactId>Java-WebSocket</artifactId>
@ -169,7 +167,7 @@
<artifactId>mybatis-plus-annotation</artifactId>
<version>${mybatis-plus.version}</version>
</dependency>
<!-- quartz-->
<dependency>
<groupId>org.quartz-scheduler</groupId>
@ -181,13 +179,13 @@
<artifactId>cron-utils</artifactId>
<version>${cron-utils.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>${druid.version}</version>
</dependency>
<!-- Zookeeper -->
<dependency>
<groupId>org.apache.zookeeper</groupId>
@ -199,8 +197,8 @@
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<artifactId>netty</artifactId>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>com.github.spotbugs</groupId>
@ -246,7 +244,7 @@
<artifactId>curator-test</artifactId>
<version>${curator-test.version}</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
@ -282,7 +280,7 @@
<artifactId>commons-email</artifactId>
<version>${commons-email.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
@ -309,7 +307,7 @@
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>
<!--protostuff-->
<dependency>
<groupId>io.protostuff</groupId>
@ -321,33 +319,33 @@
<artifactId>protostuff-runtime</artifactId>
<version>${protostuff.version}</version>
</dependency>
<dependency>
<groupId>net.bytebuddy</groupId>
<artifactId>byte-buddy</artifactId>
<version>${byte-buddy.version}</version>
</dependency>
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
<version>${reflections.version}</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql-connector.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc8</artifactId>
<version>${oracle-jdbc.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
@ -375,7 +373,7 @@
<artifactId>logback-core</artifactId>
<version>${logback.version}</version>
</dependency>
<!--excel poi-->
<dependency>
<groupId>org.apache.poi</groupId>
@ -387,7 +385,7 @@
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
<!-- hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@ -395,12 +393,12 @@
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<artifactId>com.sun.jersey</artifactId>
<groupId>jersey-json</groupId>
<artifactId>com.sun.jersey</artifactId>
</exclusion>
<exclusion>
<groupId>junit</groupId>
@ -427,37 +425,37 @@
<artifactId>hadoop-yarn-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>${commons-collections4.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>${postgresql.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
<version>${hive-jdbc.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
</dependency>
<dependency>
<groupId>com.github.oshi</groupId>
<artifactId>oshi-core</artifactId>
@ -477,31 +475,31 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<version>${clickhouse-jdbc.version}</version>
</dependency>
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId>
<version>${mssql-jdbc.version}</version>
</dependency>
<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-jdbc</artifactId>
<version>${presto-jdbc.version}</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<version>${servlet-api.version}</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
@ -512,49 +510,49 @@
<artifactId>springfox-swagger2</artifactId>
<version>${springfox.version}</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>${springfox.version}</version>
</dependency>
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-models</artifactId>
<version>${swagger-models.version}</version>
</dependency>
<dependency>
<groupId>com.github.xiaoymin</groupId>
<artifactId>swagger-bootstrap-ui</artifactId>
<version>${swagger.version}</version>
</dependency>
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>guava-retrying</artifactId>
<version>${guava-retry.version}</version>
</dependency>
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
<version>${activation.version}</version>
</dependency>
<dependency>
<groupId>com.sun.mail</groupId>
<artifactId>javax.mail</artifactId>
<version>${javax-mail}</version>
</dependency>
<dependency>
<groupId>net.sf.py4j</groupId>
<artifactId>py4j</artifactId>
<version>${py4j.version}</version>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
@ -575,19 +573,14 @@
<artifactId>error_prone_annotations</artifactId>
<version>${error_prone_annotations.version}</version>
</dependency>
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>kubernetes-client</artifactId>
<version>${kubernetes.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.hibernate.validator/hibernate-validator -->
<dependency>
<groupId>org.hibernate.validator</groupId>
<artifactId>hibernate-validator</artifactId>
<version>${hibernate-validator.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-emr</artifactId>
@ -598,26 +591,33 @@
<artifactId>joda-time</artifactId>
<version>${joda-time.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>${aws-sdk.version}</version>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp.version}</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>${json-path.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>${spring-cloud-dependencies.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
</project>
</project>

104
dolphinscheduler-dist/release-docs/LICENSE vendored
Unescape View File

@ -218,7 +218,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
accessors-smart 2.4.7: https://github.com/netplex/json-smart-v2, Apache 2.0
apacheds-i18n 2.0.0-M15: https://mvnrepository.com/artifact/org.apache.directory.server/apacheds-i18n/2.0.0-M15, Apache 2.0
apacheds-kerberos-codec 2.0.0-M15: https://mvnrepository.com/artifact/org.apache.directory.server/apacheds-kerberos-codec/2.0.0-M15, Apache 2.0
tomcat-embed-el 9.0.54: https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.54, Apache 2.0
tomcat-embed-el 9.0.55: https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.55, Apache 2.0
api-asn1-api 1.0.0-M20: https://mvnrepository.com/artifact/org.apache.directory.api/api-asn1-api/1.0.0-M20, Apache 2.0
api-util 1.0.0-M20: https://mvnrepository.com/artifact/org.apache.directory.api/api-util/1.0.0-M20, Apache 2.0
audience-annotations 0.5.0: https://mvnrepository.com/artifact/org.apache.yetus/audience-annotations/0.5.0, Apache 2.0
@ -284,14 +284,14 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
httpclient 4.5.13: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient/4.5.13, Apache 2.0
httpcore 4.4.15: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore/4.4.15, Apache 2.0
httpmime 4.5.13: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpmime/4.5.13, Apache 2.0
jackson-annotations 2.10.5: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.10.5, Apache 2.0
jackson-core 2.10.5: https://github.com/FasterXML/jackson-core, Apache 2.0
jackson-annotations 2.13.0: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.13.0, Apache 2.0
jackson-core 2.13.0: https://github.com/FasterXML/jackson-core, Apache 2.0
jackson-core-asl 1.9.13: https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-core-asl/1.9.13, Apache 2.0
jackson-databind 2.10.5: https://github.com/FasterXML/jackson-databind, Apache 2.0
jackson-datatype-jdk8 2.12.5: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.12.5, Apache 2.0
jackson-datatype-jsr310 2.12.5: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.12.5, Apache 2.0
jackson-databind 2.13.0: https://github.com/FasterXML/jackson-databind, Apache 2.0
jackson-datatype-jdk8 2.13.0: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.13.0, Apache 2.0
jackson-datatype-jsr310 2.13.0: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.13.0, Apache 2.0
jackson-mapper-asl 1.9.13: https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-mapper-asl/1.9.13, Apache 2.0
jackson-module-parameter-names 2.12.5: https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-parameter-names/2.12.5, Apache 2.0
jackson-module-parameter-names 2.13.0: https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-parameter-names/2.13.0, Apache 2.0
javax.jdo-3.2.0-m3: https://mvnrepository.com/artifact/org.datanucleus/javax.jdo/3.2.0-m3, Apache 2.0
java-xmlbuilder 0.4 : https://mvnrepository.com/artifact/com.jamesmurty.utils/java-xmlbuilder/0.4, Apache 2.0
jdo-api 3.0.1: https://mvnrepository.com/artifact/javax.jdo/jdo-api/3.0.1, Apache 2.0
@ -346,27 +346,27 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2, Apache 2.0
snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0
SparseBitSet 1.2: https://mvnrepository.com/artifact/com.zaxxer/SparseBitSet/1.2, Apache 2.0
spring-aop 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-aop/5.3.12, Apache 2.0
spring-aop 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-aop/5.3.13, Apache 2.0
spring-beans 5.3.19: https://mvnrepository.com/artifact/org.springframework/spring-beans/5.3.19, Apache 2.0
spring-boot 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.5.6, Apache 2.0
spring-boot-actuator 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator/2.5.6, Apache 2.0
spring-boot-actuator-autoconfigure 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator-autoconfigure/2.5.6, Apache 2.0
spring-boot-configuration-processor 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-configuration-processor/2.5.6, Apache 2.0
spring-boot-autoconfigure 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.5.6, Apache 2.0
spring-boot-starter 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.5.6, Apache 2.0
spring-boot-starter-actuator 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-actuator/2.5.6, Apache 2.0
spring-boot-starter-aop 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.5.6, Apache 2.0
spring-boot-starter-jdbc 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.5.6, Apache 2.0
spring-boot-starter-jetty 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.5.6, Apache 2.0
spring-boot-starter-json 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.5.6, Apache 2.0
spring-boot-starter-logging 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.5.6, Apache 2.0
spring-boot-starter-quartz 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-quartz/2.5.6, Apache 2.0
spring-boot-starter-web 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.5.6, Apache 2.0
spring-boot-starter-cache 2.5.6: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-cache/2.5.6, Apache 2.0
spring-boot 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.6.1, Apache 2.0
spring-boot-actuator 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator/2.6.1, Apache 2.0
spring-boot-actuator-autoconfigure 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator-autoconfigure/2.6.1, Apache 2.0
spring-boot-configuration-processor 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-configuration-processor/2.6.1, Apache 2.0
spring-boot-autoconfigure 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.6.1, Apache 2.0
spring-boot-starter 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.6.1, Apache 2.0
spring-boot-starter-actuator 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-actuator/2.6.1, Apache 2.0
spring-boot-starter-aop 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.6.1, Apache 2.0
spring-boot-starter-jdbc 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.6.1, Apache 2.0
spring-boot-starter-jetty 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.6.1, Apache 2.0
spring-boot-starter-json 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.6.1, Apache 2.0
spring-boot-starter-logging 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.6.1, Apache 2.0
spring-boot-starter-quartz 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-quartz/2.6.1, Apache 2.0
spring-boot-starter-web 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.6.1, Apache 2.0
spring-boot-starter-cache 2.6.1: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-cache/2.6.1, Apache 2.0
spring-context 5.3.19: https://mvnrepository.com/artifact/org.springframework/spring-context/5.3.19, Apache 2.0
spring-context-support 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-context-support/5.3.12, Apache 2.0
spring-context-support 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-context-support/5.3.13, Apache 2.0
spring-core 5.3.19: https://mvnrepository.com/artifact/org.springframework/spring-core/5.3.19, Apache 2.0
spring-expression 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-expression/5.3.12, Apache 2.0
spring-expression 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-expression/5.3.13, Apache 2.0
springfox-core 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-core/2.9.2, Apache 2.0
springfox-schema 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-schema/2.9.2, Apache 2.0
springfox-spi 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-spi/2.9.2, Apache 2.0
@ -374,18 +374,18 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
springfox-swagger2 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger2/2.9.2, Apache 2.0
springfox-swagger-common 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-common/2.9.2, Apache 2.0
springfox-swagger-ui 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-ui/2.9.2, Apache 2.0
spring-jcl 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.3.12, Apache 2.0
spring-jcl 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.3.13, Apache 2.0
spring-jdbc 5.3.19: https://mvnrepository.com/artifact/org.springframework/spring-jdbc/5.3.19, Apache 2.0
spring-plugin-core 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-core/1.2.0.RELEASE, Apache 2.0
spring-plugin-metadata 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-metadata/1.2.0.RELEASE, Apache 2.0
spring-tx 5.3.19: https://mvnrepository.com/artifact/org.springframework/spring-tx/5.3.19, Apache 2.0
spring-web 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-web/5.3.12, Apache 2.0
spring-webmvc 5.3.12: https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.3.12, Apache 2.0
spring-web 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-web/5.3.13, Apache 2.0
spring-webmvc 5.3.13: https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.3.13, Apache 2.0
swagger-annotations 1.5.20: https://mvnrepository.com/artifact/io.swagger/swagger-annotations/1.5.20, Apache 2.0
swagger-bootstrap-ui 1.9.3: https://mvnrepository.com/artifact/com.github.xiaoymin/swagger-bootstrap-ui/1.9.3, Apache 2.0
swagger-models 1.5.24: https://mvnrepository.com/artifact/io.swagger/swagger-models/1.5.24, Apache 2.0
tephra-api 0.6.0: https://mvnrepository.com/artifact/co.cask.tephra/tephra-api/0.6.0, Apache 2.0
tomcat-embed-el 9.0.54: https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.54, Apache 2.0
tomcat-embed-el 9.0.55: https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.55, Apache 2.0
xercesImpl 2.9.1: https://mvnrepository.com/artifact/xerces/xercesImpl/2.9.1, Apache 2.0
xmlbeans 3.1.0: https://mvnrepository.com/artifact/org.apache.xmlbeans/xmlbeans/3.1.0, Apache 2.0
xml-apis 1.3.04: https://mvnrepository.com/artifact/xml-apis/xml-apis/1.3.04, Apache 2.0 and W3C
@ -397,30 +397,30 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0
prometheus client_java(simpleclient) 0.12.0: https://github.com/prometheus/client_java, Apache 2.0
snowflake snowflake-2010: https://github.com/twitter-archive/snowflake/tree/snowflake-2010, Apache 2.0
kubernetes-client 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-client/5.8.0, Apache 2.0
kubernetes-model-admissionregistration 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-admissionregistration/5.8.0, Apache 2.0
kubernetes-model-apiextensions 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apiextensions/5.8.0, Apache 2.0
kubernetes-model-apps 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apps/5.8.0, Apache 2.0
kubernetes-model-autoscaling 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/5.8.0, Apache 2.0
kubernetes-model-batch 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/5.8.0, Apache 2.0
kubernetes-model-certificates 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-certificates/5.8.0, Apache 2.0
kubernetes-model-common 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-common/5.8.0, Apache 2.0
kubernetes-model-coordination 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-coordination/5.8.0, Apache 2.0
kubernetes-model-core 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-core/5.8.0, Apache 2.0
kubernetes-model-discovery 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-discovery/5.8.0, Apache 2.0
kubernetes-model-events 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-events/5.8.0, Apache 2.0
kubernetes-model-extensions 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-extensions/5.8.0, Apache 2.0
kubernetes-model-flowcontrol 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-flowcontrol/5.8.0, Apache 2.0
kubernetes-model-metrics 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-metrics/5.8.0, Apache 2.0
kubernetes-model-networking 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-networking/5.8.0, Apache 2.0
kubernetes-model-node 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-node/5.8.0, Apache 2.0
kubernetes-model-policy 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-policy/5.8.0, Apache 2.0
kubernetes-model-rbac 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-rbac/5.8.0, Apache 2.0
kubernetes-model-scheduling 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-scheduling/5.8.0, Apache 2.0
kubernetes-model-storageclass 5.8.0: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-storageclass/5.8.0, Apache 2.0
kubernetes-client 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-client/4.13.3, Apache 2.0
kubernetes-model-admissionregistration 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-admissionregistration/4.13.3, Apache 2.0
kubernetes-model-apiextensions 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apiextensions/4.13.3, Apache 2.0
kubernetes-model-apps 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apps/4.13.3, Apache 2.0
kubernetes-model-autoscaling 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/4.13.3, Apache 2.0
kubernetes-model-batch 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/4.13.3, Apache 2.0
kubernetes-model-certificates 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-certificates/4.13.3, Apache 2.0
kubernetes-model-common 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-common/4.13.3, Apache 2.0
kubernetes-model-coordination 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-coordination/4.13.3, Apache 2.0
kubernetes-model-core 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-core/4.13.3, Apache 2.0
kubernetes-model-discovery 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-discovery/4.13.3, Apache 2.0
kubernetes-model-events 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-events/4.13.3, Apache 2.0
kubernetes-model-extensions 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-extensions/4.13.3, Apache 2.0
kubernetes-model-flowcontrol 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-flowcontrol/4.13.3, Apache 2.0
kubernetes-model-metrics 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-metrics/4.13.3, Apache 2.0
kubernetes-model-networking 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-networking/4.13.3, Apache 2.0
kubernetes-model-node 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-node/4.13.3, Apache 2.0
kubernetes-model-policy 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-policy/4.13.3, Apache 2.0
kubernetes-model-rbac 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-rbac/4.13.3, Apache 2.0
kubernetes-model-scheduling 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-scheduling/4.13.3, Apache 2.0
kubernetes-model-storageclass 4.13.3: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-storageclass/4.13.3, Apache 2.0
zjsonpatch 0.3.0 https://mvnrepository.com/artifact/io.fabric8/zjsonpatch/0.3.0, Apache 2.0
generex 1.0.2 https://mvnrepository.com/artifact/com.github.mifmif/generex/1.0.2, Apache 2.0
jackson-dataformat-yaml 2.12.5 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.12.5, Apache 2.0
jackson-dataformat-yaml 2.13.0 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.13.0, Apache 2.0
logging-interceptor 3.14.9 https://mvnrepository.com/artifact/com.squareup.okhttp3/logging-interceptor/3.14.9, Apache 2.0
okhttp 3.14.3 https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp/3.14.3, Apache 2.0
okio 1.17.2 https://mvnrepository.com/artifact/com.squareup.okio/okio/1.17.2, Apache 2.0
@ -429,7 +429,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
jboss-logging:jar 3.4.2.Final https://mvnrepository.com/artifact/org.jboss.logging/jboss-logging/3.4.2.Final, Apache 2.0
ion-java 1.0.2 https://mvnrepository.com/artifact/software.amazon.ion/ion-java/1.0.2 Apache 2.0
jmespath-java 1.12.160 https://mvnrepository.com/artifact/com.amazonaws/jmespath-java/1.12.160 Apache 2.0
jackson-dataformat-cbor 2.12.5 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-cbor/2.12.5 Apache 2.0
jackson-dataformat-cbor 2.13.0 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-cbor/2.13.0 Apache 2.0
aws-java-sdk-emr 1.12.160 https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-emr/1.12.160 Apache 2.0
aws-java-sdk-core 1.12.160 https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-core/1.12.160 Apache 2.0
aws-java-sdk-s3 1.12.160 https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-s3/1.12.160 Apache 2.0

2
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
Unescape View File

@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
@ -79,6 +80,7 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
@Autowired
private MasterConfig masterConfig;
@Lazy
@Autowired
private WorkflowExecuteThreadPool workflowExecuteThreadPool;

2
dolphinscheduler-master/src/main/resources/application.yaml
Unescape View File

@ -56,7 +56,7 @@ spring:
properties:
org.quartz.threadPool:threadPriority: 5
org.quartz.jobStore.isClustered: true
org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX
org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true

5
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
Unescape View File

@ -266,9 +266,6 @@ public class ProcessServiceImpl implements ProcessService {
@Autowired
private CuringParamsService curingGlobalParamsService;
@Autowired
private ProcessService processService;
/**
* handle Command (construct ProcessInstance from Command) , wrapped in transaction
*
@ -1268,7 +1265,7 @@ public class ProcessServiceImpl implements ProcessService {
try {
// submit task to db
// Only want to use transaction here
task = processService.submitTask(processInstance, taskInstance);
task = submitTask(processInstance, taskInstance);
if (task != null && task.getId() != 0) {
break;
}

60
dolphinscheduler-standalone-server/pom.xml
Unescape View File

@ -15,18 +15,29 @@
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dolphinscheduler-standalone-server</artifactId>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-bom</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
@ -44,17 +55,17 @@
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-alert-server</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-log-server</artifactId>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
@ -65,20 +76,21 @@
</exclusion>
</exclusions>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-bom</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-context</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-kubernetes-commons</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
@ -96,10 +108,10 @@
<executions>
<execution>
<id>dolphinscheduler-standalone-server</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<finalName>standalone-server</finalName>
<descriptors>
@ -112,7 +124,7 @@
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>docker</id>

2
dolphinscheduler-standalone-server/src/main/resources/application.yaml
Unescape View File

@ -49,7 +49,7 @@ spring:
properties:
org.quartz.threadPool:threadPriority: 5
org.quartz.jobStore.isClustered: true
org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX
org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true

106
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
Unescape View File

@ -32,6 +32,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.NAME_LAB
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RESTART_POLICY;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_INSTANCE_ID;
import io.fabric8.kubernetes.client.KubernetesClientException;
import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
@ -58,20 +59,20 @@ import org.slf4j.Logger;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.Quantity;
import io.fabric8.kubernetes.api.model.ResourceRequirements;
import io.fabric8.kubernetes.api.model.batch.v1.Job;
import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder;
import io.fabric8.kubernetes.api.model.batch.v1.JobStatus;
import io.fabric8.kubernetes.api.model.batch.Job;
import io.fabric8.kubernetes.api.model.batch.JobBuilder;
import io.fabric8.kubernetes.api.model.batch.JobStatus;
import io.fabric8.kubernetes.client.Watch;
import io.fabric8.kubernetes.client.Watcher;
import io.fabric8.kubernetes.client.WatcherException;
/**
* K8sTaskExecutor used to submit k8s task to K8S
*/
public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
private Job job;
public K8sTaskExecutor(Logger logger, TaskExecutionContext taskRequest) {
super(logger,taskRequest);
super(logger, taskRequest);
}
public Job buildK8sJob(K8sTaskMainParameters k8STaskMainParameters) {
@ -99,7 +100,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
List<EnvVar> envVars = new ArrayList<>();
envVars.add(taskInstanceIdVar);
if (MapUtils.isNotEmpty(otherParams)) {
for (Map.Entry<String,String> entry : otherParams.entrySet()) {
for (Map.Entry<String, String> entry : otherParams.entrySet()) {
String param = entry.getKey();
String paramValue = entry.getValue();
EnvVar envVar = new EnvVar(param, paramValue, null);
@ -107,60 +108,58 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
}
}
return new JobBuilder()
.withApiVersion(API_VERSION)
.withNewMetadata()
.withName(k8sJobName)
.withLabels(labelMap)
.withNamespace(namespaceName)
.endMetadata()
.withNewSpec()
.withTtlSecondsAfterFinished(JOB_TTL_SECONDS)
.withNewTemplate()
.withNewSpec()
.addNewContainer()
.withName(k8sJobName)
.withImage(image)
.withImagePullPolicy(IMAGE_PULL_POLICY)
.withResources(new ResourceRequirements(limitRes, reqRes))
.withEnv(envVars)
.endContainer()
.withRestartPolicy(RESTART_POLICY)
.endSpec()
.endTemplate()
.withBackoffLimit(retryNum)
.endSpec()
.build();
.withApiVersion(API_VERSION)
.withNewMetadata()
.withName(k8sJobName)
.withLabels(labelMap)
.withNamespace(namespaceName)
.endMetadata()
.withNewSpec()
.withTtlSecondsAfterFinished(JOB_TTL_SECONDS)
.withNewTemplate()
.withNewSpec()
.addNewContainer()
.withName(k8sJobName)
.withImage(image)
.withImagePullPolicy(IMAGE_PULL_POLICY)
.withResources(new ResourceRequirements(limitRes, reqRes))
.withEnv(envVars)
.endContainer()
.withRestartPolicy(RESTART_POLICY)
.endSpec()
.endTemplate()
.withBackoffLimit(retryNum)
.endSpec()
.build();
}
public void registerBatchJobWatcher(Job job, String taskInstanceId, TaskResponse taskResponse, K8sTaskMainParameters k8STaskMainParameters) {
public void registerBatchJobWatcher(Job job, String taskInstanceId, TaskResponse taskResponse,
K8sTaskMainParameters k8STaskMainParameters) {
CountDownLatch countDownLatch = new CountDownLatch(1);
Watcher<Job> watcher = new Watcher<Job>() {
@Override
public void eventReceived(Action action, Job job) {
if (action != Action.ADDED) {
int jobStatus = getK8sJobStatus(job);
setTaskStatus(jobStatus,taskInstanceId, taskResponse, k8STaskMainParameters);
setTaskStatus(jobStatus, taskInstanceId, taskResponse, k8STaskMainParameters);
countDownLatch.countDown();
}
}
}
@Override
public void onClose(WatcherException e) {
logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail in k8s: %s",job.getMetadata().getName(),e.getMessage()));
public void onClose(KubernetesClientException e) {
logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail in k8s: %s", job.getMetadata().getName(),
e.getMessage()));
taskResponse.setExitStatusCode(EXIT_CODE_FAILURE);
countDownLatch.countDown();
}
@Override
public void onClose() {
logger.warn("Watch gracefully closed");
}
};
Watch watch = null;
try {
watch = k8sUtils.createBatchJobWatcher(job.getMetadata().getName(), watcher);
boolean timeoutFlag = taskRequest.getTaskTimeoutStrategy() == TaskTimeoutStrategy.FAILED
|| taskRequest.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED;
|| taskRequest.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED;
if (timeoutFlag) {
Boolean timeout = !(countDownLatch.await(taskRequest.getTaskTimeout(), TimeUnit.SECONDS));
waitTimeout(timeout);
@ -169,11 +168,11 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
}
flushLog(taskResponse);
} catch (InterruptedException e) {
logger.error("job failed in k8s: {}",e.getMessage(), e);
logger.error("job failed in k8s: {}", e.getMessage(), e);
Thread.currentThread().interrupt();
taskResponse.setExitStatusCode(EXIT_CODE_FAILURE);
} catch (Exception e) {
logger.error("job failed in k8s: {}",e.getMessage(), e);
logger.error("job failed in k8s: {}", e.getMessage(), e);
taskResponse.setExitStatusCode(EXIT_CODE_FAILURE);
} finally {
if (watch != null) {
@ -186,7 +185,8 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
public TaskResponse run(String k8sParameterStr) throws Exception {
TaskResponse result = new TaskResponse();
int taskInstanceId = taskRequest.getTaskInstanceId();
K8sTaskMainParameters k8STaskMainParameters = JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class);
K8sTaskMainParameters k8STaskMainParameters =
JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class);
try {
if (null == TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId)) {
result.setExitStatusCode(EXIT_CODE_KILL);
@ -197,7 +197,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
return result;
}
K8sTaskExecutionContext k8sTaskExecutionContext = taskRequest.getK8sTaskExecutionContext();
String configYaml = k8sTaskExecutionContext.getConfigYaml();
String configYaml = k8sTaskExecutionContext.getConfigYaml();
k8sUtils.buildClient(configYaml);
submitJob2k8s(k8sParameterStr);
registerBatchJobWatcher(job, Integer.toString(taskInstanceId), result, k8STaskMainParameters);
@ -220,7 +220,8 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
public void submitJob2k8s(String k8sParameterStr) {
int taskInstanceId = taskRequest.getTaskInstanceId();
String taskName = taskRequest.getTaskName().toLowerCase(Locale.ROOT);
K8sTaskMainParameters k8STaskMainParameters = JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class);
K8sTaskMainParameters k8STaskMainParameters =
JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class);
try {
logger.info("[K8sJobExecutor-{}-{}] start to submit job", taskName, taskInstanceId);
job = buildK8sJob(k8STaskMainParameters);
@ -236,7 +237,8 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
@Override
public void stopJobOnK8s(String k8sParameterStr) {
K8sTaskMainParameters k8STaskMainParameters = JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class);
K8sTaskMainParameters k8STaskMainParameters =
JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class);
String namespaceName = k8STaskMainParameters.getNamespaceName();
String jobName = job.getMetadata().getName();
try {
@ -260,17 +262,21 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
}
}
public void setTaskStatus(int jobStatus,String taskInstanceId, TaskResponse taskResponse, K8sTaskMainParameters k8STaskMainParameters) {
public void setTaskStatus(int jobStatus, String taskInstanceId, TaskResponse taskResponse,
K8sTaskMainParameters k8STaskMainParameters) {
if (jobStatus == EXIT_CODE_SUCCESS || jobStatus == EXIT_CODE_FAILURE) {
if (null == TaskExecutionContextCacheManager.getByTaskInstanceId(Integer.valueOf(taskInstanceId))) {
logStringBuffer.append(String.format("[K8sJobExecutor-%s] killed", job.getMetadata().getName()));
taskResponse.setExitStatusCode(EXIT_CODE_KILL);
} else if (jobStatus == EXIT_CODE_SUCCESS) {
logStringBuffer.append(String.format("[K8sJobExecutor-%s] succeed in k8s", job.getMetadata().getName()));
logStringBuffer
.append(String.format("[K8sJobExecutor-%s] succeed in k8s", job.getMetadata().getName()));
taskResponse.setExitStatusCode(EXIT_CODE_SUCCESS);
} else {
String errorMessage = k8sUtils.getPodLog(job.getMetadata().getName(), k8STaskMainParameters.getNamespaceName());
logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail in k8s: %s", job.getMetadata().getName(), errorMessage));
String errorMessage =
k8sUtils.getPodLog(job.getMetadata().getName(), k8STaskMainParameters.getNamespaceName());
logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail in k8s: %s", job.getMetadata().getName(),
errorMessage));
taskResponse.setExitStatusCode(EXIT_CODE_FAILURE);
}
}

47
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/K8sUtils.java
Unescape View File

@ -28,8 +28,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.batch.v1.Job;
import io.fabric8.kubernetes.api.model.batch.v1.JobList;
import io.fabric8.kubernetes.api.model.batch.Job;
import io.fabric8.kubernetes.api.model.batch.JobList;
import io.fabric8.kubernetes.client.Config;
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClient;
@ -37,40 +37,41 @@ import io.fabric8.kubernetes.client.Watch;
import io.fabric8.kubernetes.client.Watcher;
public class K8sUtils {
private static final Logger log = LoggerFactory.getLogger(K8sUtils.class);
private KubernetesClient client;
public void createJob(String namespace, Job job) {
try {
client.batch().v1()
.jobs()
.inNamespace(namespace)
.create(job);
client.batch()
.jobs()
.inNamespace(namespace)
.create(job);
} catch (Exception e) {
throw new TaskException("fail to create job",e);
throw new TaskException("fail to create job", e);
}
}
public void deleteJob(String jobName, String namespace) {
try {
client.batch().v1()
.jobs()
.inNamespace(namespace)
.withName(jobName)
.delete();
client.batch()
.jobs()
.inNamespace(namespace)
.withName(jobName)
.delete();
} catch (Exception e) {
throw new TaskException("fail to delete job",e);
throw new TaskException("fail to delete job", e);
}
}
public Boolean jobExist(String jobName, String namespace) {
Optional<Job> result;
try {
JobList jobList = client.batch().v1().jobs().inNamespace(namespace).list();
JobList jobList = client.batch().jobs().inNamespace(namespace).list();
List<Job> jobs = jobList.getItems();
result = jobs.stream()
.filter(job -> job.getMetadata().getName().equals(jobName))
.findFirst();
.filter(job -> job.getMetadata().getName().equals(jobName))
.findFirst();
return result.isPresent();
} catch (Exception e) {
throw new TaskException("fail to check job: ", e);
@ -79,10 +80,10 @@ public class K8sUtils {
public Watch createBatchJobWatcher(String jobName, Watcher<Job> watcher) {
try {
return client.batch().v1()
.jobs().withName(jobName).watch(watcher);
return client.batch()
.jobs().withName(jobName).watch(watcher);
} catch (Exception e) {
throw new TaskException("fail to register batch job watcher",e);
throw new TaskException("fail to register batch job watcher", e);
}
}
@ -97,9 +98,9 @@ public class K8sUtils {
}
}
return client.pods().inNamespace(namespace)
.withName(podName)
.tailingLines(LOG_LINES)
.getLog(Boolean.TRUE);
.withName(podName)
.tailingLines(LOG_LINES)
.getLog(Boolean.TRUE);
} catch (Exception e) {
log.error("fail to getPodLog", e);
log.error("response bodies : {}", e.getMessage());
@ -112,7 +113,7 @@ public class K8sUtils {
Config config = Config.fromKubeconfig(configYaml);
client = new DefaultKubernetesClient(config);
} catch (Exception e) {
throw new TaskException("fail to build k8s ApiClient",e);
throw new TaskException("fail to build k8s ApiClient", e);
}
}

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
Unescape View File

@ -16,8 +16,8 @@
*/
package org.apache.dolphinscheduler.plugin.task.api.k8s;
import io.fabric8.kubernetes.api.model.batch.v1.Job;
import io.fabric8.kubernetes.api.model.batch.v1.JobStatus;
import io.fabric8.kubernetes.api.model.batch.Job;
import io.fabric8.kubernetes.api.model.batch.JobStatus;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CLUSTER;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL;

9
pom.xml
Unescape View File

@ -62,7 +62,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring.boot.version>2.5.6</spring.boot.version>
<spring.boot.version>2.6.1</spring.boot.version>
<java.version>1.8</java.version>
<junit.version>4.12</junit.version>
<spotbugs.version>3.1.12</spotbugs.version>
@ -309,6 +309,13 @@
<artifactId>dolphinscheduler-tools</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.3.3</version>
<scope>test</scope>
</dependency>
</dependencies>
</dependencyManagement>

60
tools/dependencies/known-dependencies.txt
Unescape View File

@ -81,18 +81,19 @@ httpclient-4.5.13.jar
httpcore-4.4.15.jar
httpmime-4.5.13.jar
j2objc-annotations-1.1.jar
jackson-annotations-2.10.5.jar
jackson-core-2.10.5.jar
jackson-annotations-2.13.0.jar
jackson-core-2.13.0.jar
jackson-core-asl-1.9.13.jar
jackson-databind-2.10.5.jar
jackson-databind-2.13.0.jar
jackson-datatype-jdk8-2.12.5.jar
jackson-datatype-jsr310-2.12.5.jar
jackson-mapper-asl-1.9.13.jar
jackson-module-parameter-names-2.12.5.jar
jakarta.activation-api-1.2.2.jar
jakarta.annotation-api-1.3.5.jar
jakarta.servlet-api-4.0.4.jar
jakarta.validation-api-2.0.2.jar
jakarta.websocket-api-1.1.2.jar
jakarta.xml.bind-api-2.3.3.jar
jamon-runtime-2.3.1.jar
java-xmlbuilder-0.4.jar
javassist-3.27.0-GA.jar
@ -123,7 +124,7 @@ jna-5.10.0.jar
jna-platform-5.10.0.jar
joda-time-2.10.13.jar
jpam-1.1.jar
jsch-0.1.42.jar
jsch-0.1.55.jar
jsp-api-2.1.jar
jsqlparser-2.1.jar
jsr305-3.0.0.jar
@ -179,7 +180,7 @@ spring-boot-2.5.6.jar
spring-boot-actuator-2.5.6.jar
spring-boot-actuator-autoconfigure-2.5.6.jar
spring-boot-autoconfigure-2.5.6.jar
spring-boot-configuration-processor-2.5.6.jar
spring-boot-configuration-processor-2.6.1.jar
spring-boot-starter-2.5.6.jar
spring-boot-starter-actuator-2.5.6.jar
spring-boot-starter-aop-2.5.6.jar
@ -190,6 +191,9 @@ spring-boot-starter-logging-2.5.6.jar
spring-boot-starter-quartz-2.5.6.jar
spring-boot-starter-web-2.5.6.jar
spring-boot-starter-cache-2.5.6.jar
spring-cloud-context-3.0.6.jar
spring-cloud-kubernetes-commons-2.0.6.jar
spring-cloud-starter-kubernetes-fabric8-config-2.0.6.jar
spring-context-5.3.19.jar
spring-context-support-5.3.12.jar
spring-core-5.3.19.jar
@ -220,31 +224,33 @@ xmlbeans-3.1.0.jar
xmlenc-0.52.jar
zookeeper-3.4.14.jar
Java-WebSocket-1.5.1.jar
kubernetes-client-5.8.0.jar
kubernetes-model-admissionregistration-5.8.0.jar
kubernetes-model-apiextensions-5.8.0.jar
kubernetes-model-apps-5.8.0.jar
kubernetes-model-autoscaling-5.8.0.jar
kubernetes-model-batch-5.8.0.jar
kubernetes-model-certificates-5.8.0.jar
kubernetes-model-common-5.8.0.jar
kubernetes-model-coordination-5.8.0.jar
kubernetes-model-core-5.8.0.jar
kubernetes-model-discovery-5.8.0.jar
kubernetes-model-events-5.8.0.jar
kubernetes-model-extensions-5.8.0.jar
kubernetes-model-flowcontrol-5.8.0.jar
kubernetes-model-metrics-5.8.0.jar
kubernetes-model-networking-5.8.0.jar
kubernetes-model-node-5.8.0.jar
kubernetes-model-policy-5.8.0.jar
kubernetes-model-rbac-5.8.0.jar
kubernetes-model-scheduling-5.8.0.jar
kubernetes-model-storageclass-5.8.0.jar
kubernetes-client-4.13.3.jar
kubernetes-model-admissionregistration-4.13.3.jar
kubernetes-model-apiextensions-4.13.3.jar
kubernetes-model-apps-4.13.3.jar
kubernetes-model-autoscaling-4.13.3.jar
kubernetes-model-batch-4.13.3.jar
kubernetes-model-certificates-4.13.3.jar
kubernetes-model-common-4.13.3.jar
kubernetes-model-coordination-4.13.3.jar
kubernetes-model-core-4.13.3.jar
kubernetes-model-discovery-4.13.3.jar
kubernetes-model-events-4.13.3.jar
kubernetes-model-extensions-4.13.3.jar
kubernetes-model-metrics-4.13.3.jar
kubernetes-model-networking-4.13.3.jar
kubernetes-model-node-4.13.3.jar
kubernetes-model-policy-4.13.3.jar
kubernetes-model-rbac-4.13.3.jar
kubernetes-model-settings-4.13.3.jar
kubernetes-model-scheduling-4.13.3.jar
kubernetes-model-storageclass-4.13.3.jar
zjsonpatch-0.3.0.jar
automaton-1.11-8.jar
generex-1.0.2.jar
jackson-dataformat-yaml-2.12.5.jar
jackson-module-jaxb-annotations-2.12.5.jar
jackson-module-parameter-names-2.12.5.jar
logging-interceptor-3.14.9.jar
okhttp-3.14.9.jar
okio-1.17.2.jar

Write Preview
Loading…
Cancel
Save