From 277f137358628bd248483b6b5e865f41b3aaaa11 Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Fri, 2 Sep 2022 12:03:19 +0800 Subject: [PATCH] Add Kubernetes configmap reload to all components (#11730) --- docs/docs/en/architecture/configuration.md | 37 ++++++----- docs/docs/zh/architecture/configuration.md | 53 ++++++++------- .../dolphinscheduler-alert-server/pom.xml | 13 ++-- .../src/main/resources/application.yaml | 2 - .../src/main/resources/bootstrap.yaml | 28 ++++++++ dolphinscheduler-api/pom.xml | 5 ++ .../src/main/resources/application.yaml | 2 - .../src/main/resources/bootstrap.yaml | 28 ++++++++ dolphinscheduler-dist/release-docs/LICENSE | 2 + .../server/log/LoggerRequestProcessor.java | 17 ++--- dolphinscheduler-master/pom.xml | 64 ++++++++++--------- .../src/main/resources/application.yaml | 2 - .../src/main/resources/bootstrap.yaml | 28 ++++++++ dolphinscheduler-standalone-server/pom.xml | 4 -- .../src/main/resources/application.yaml | 2 - .../src/main/resources/bootstrap.yaml | 28 ++++++++ dolphinscheduler-worker/pom.xml | 41 ++++++------ .../src/main/resources/application.yaml | 2 - .../src/main/resources/bootstrap.yaml | 28 ++++++++ tools/dependencies/known-dependencies.txt | 11 ++++ 20 files changed, 284 insertions(+), 113 deletions(-) create mode 100644 dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/bootstrap.yaml create mode 100644 dolphinscheduler-api/src/main/resources/bootstrap.yaml create mode 100644 dolphinscheduler-master/src/main/resources/bootstrap.yaml create mode 100644 dolphinscheduler-standalone-server/src/main/resources/bootstrap.yaml create mode 100644 dolphinscheduler-worker/src/main/resources/bootstrap.yaml diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 48d8aa80ab..cfa853c896 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -17,14 +17,14 @@ The directory structure of DolphinScheduler is as follows: │ ├── licenses directory of licenses │ -├── bin directory of DolphinScheduler application commands, configrations scripts +├── bin directory of DolphinScheduler application commands, configrations scripts │   ├── dolphinscheduler-daemon.sh script to start or shut down DolphinScheduler application │   ├── env directory of scripts to load environment variables │   │   ├── dolphinscheduler_env.sh script to export environment variables [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...] when you start or stop service using script `dolphinscheduler-daemon.sh` │   │   └── install_env.sh script to export environment variables for DolphinScheduler installation when you use scripts `install.sh` `start-all.sh` `stop-all.sh` `status-all.sh` │   ├── install.sh script to auto-setup services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode -│   ├── remove-zk-node.sh script to cleanup ZooKeeper caches -│   ├── scp-hosts.sh script to copy installation files to target hosts +│   ├── remove-zk-node.sh script to cleanup ZooKeeper caches +│   ├── scp-hosts.sh script to copy installation files to target hosts │   ├── start-all.sh script to start all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode │   ├── status-all.sh script to check the status of all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode │   └── stop-all.sh script to shut down all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode @@ -34,7 +34,8 @@ The directory structure of DolphinScheduler is as follows: │   │   └── start.sh script to start DolphinScheduler alert-server │   ├── conf │   │   ├── application.yaml configurations of alert-server -│   │   ├── common.properties configurations of common-service like storage, credentials, etc. +│   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, +│   │   ├── common.properties configurations of common-service like storage, credentials, etc. │   │   ├── dolphinscheduler_env.sh script to load environment variables for alert-server │   │   └── logback-spring.xml configurations of alert-service log │   └── libs directory of alert-server libs @@ -44,17 +45,19 @@ The directory structure of DolphinScheduler is as follows: │   │   └── start.sh script to start DolphinScheduler api-server │   ├── conf │   │   ├── application.yaml configurations of api-server +│   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, │   │   ├── common.properties configurations of common-service like storage, credentials, etc. │   │   ├── dolphinscheduler_env.sh script to load environment variables for api-server │   │   └── logback-spring.xml configurations of api-service log │   ├── libs directory of api-server libs -│   └── ui directory of api-server related front-end web resources +│   └── ui directory of api-server related front-end web resources │ ├── master-server directory of DolphinScheduler master-server commands, configrations scripts and libs -│   ├── bin +│   ├── bin │   │   └── start.sh script to start DolphinScheduler master-server │   ├── conf │   │   ├── application.yaml configurations of master-server +│   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, │   │   ├── common.properties configurations of common-service like storage, credentials, etc. │   │   ├── dolphinscheduler_env.sh script to load environment variables for master-server │   │   └── logback-spring.xml configurations of master-service log @@ -65,13 +68,14 @@ The directory structure of DolphinScheduler is as follows: │   │   └── start.sh script to start DolphinScheduler standalone-server │   ├── conf │   │   ├── application.yaml configurations of standalone-server +│   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, │   │   ├── common.properties configurations of common-service like storage, credentials, etc. │   │   ├── dolphinscheduler_env.sh script to load environment variables for standalone-server │   │   ├── logback-spring.xml configurations of standalone-service log │   │   └── sql .sql files to create or upgrade DolphinScheduler metadata │   ├── libs directory of standalone-server libs │   └── ui directory of standalone-server related front-end web resources -│   +│   ├── tools directory of DolphinScheduler metadata tools commands, configrations scripts and libs │   ├── bin │   │   └── upgrade-schema.sh script to initialize or upgrade DolphinScheduler metadata @@ -80,12 +84,13 @@ The directory structure of DolphinScheduler is as follows: │   │   └── common.properties configurations of common-service like storage, credentials, etc. │   ├── libs directory of tool libs │   └── sql .sql files to create or upgrade DolphinScheduler metadata -│   +│   ├── worker-server directory of DolphinScheduler worker-server commands, configrations scripts and libs │ ├── bin │ │   └── start.sh script to start DolphinScheduler worker-server │ ├── conf │ │   ├── application.yaml configurations of worker-server +│ │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, │ │   ├── common.properties configurations of common-service like storage, credentials, etc. │ │   ├── dolphinscheduler_env.sh script to load environment variables for worker-server │ │   └── logback-spring.xml configurations of worker-service log @@ -107,14 +112,14 @@ Currently, DolphinScheduler just makes a basic config, remember to config furthe Default simplified parameters are: ```bash export DOLPHINSCHEDULER_OPTS=" --server --Xmx16g --Xms1g --Xss512k --XX:+UseConcMarkSweepGC --XX:+CMSParallelRemarkEnabled --XX:+UseFastAccessorMethods --XX:+UseCMSInitiatingOccupancyOnly +-server +-Xmx16g +-Xms1g +-Xss512k +-XX:+UseConcMarkSweepGC +-XX:+CMSParallelRemarkEnabled +-XX:+UseFastAccessorMethods +-XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 " ``` diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 9e18e7832d..fb985a96d3 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -13,13 +13,13 @@ DolphinScheduler的目录结构如下： │ ├── licenses licenses存放目录 │ -├── bin DolphinScheduler命令和环境变量配置存放目录 +├── bin DolphinScheduler命令和环境变量配置存放目录 │   ├── dolphinscheduler-daemon.sh 启动/关闭DolphinScheduler服务脚本 │   ├── env 环境变量配置存放目录 -│   │   ├── dolphinscheduler_env.sh 当使用`dolphinscheduler-daemon.sh`脚本起停服务时，运行此脚本加载环境变量配置文件 [如：JAVA_HOME,HADOOP_HOME, HIVE_HOME ...] -│   │   └── install_env.sh 当使用`install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`脚本时，运行此脚本为DolphinScheduler安装加载环境变量配置 -│   ├── install.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本自动安装服务 -│   ├── remove-zk-node.sh 清理zookeeper缓存文件脚本 +│   │   ├── dolphinscheduler_env.sh 当使用`dolphinscheduler-daemon.sh`脚本起停服务时，运行此脚本加载环境变量配置文件 [如：JAVA_HOME,HADOOP_HOME, HIVE_HOME ...] +│   │   └── install_env.sh 当使用`install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`脚本时，运行此脚本为DolphinScheduler安装加载环境变量配置 +│   ├── install.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本自动安装服务 +│   ├── remove-zk-node.sh 清理zookeeper缓存文件脚本 │   ├── scp-hosts.sh 安装文件传输脚本 │   ├── start-all.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本启动所有服务 │   ├── status-all.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本获取所有服务状态 @@ -30,7 +30,8 @@ DolphinScheduler的目录结构如下： │   │   └── start.sh DolphinScheduler alert-server启动脚本 │   ├── conf │   │   ├── application.yaml alert-server配置文件 -│   │   ├── common.properties 公共服务（存储等信息）配置文件 +│   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 +│   │   ├── common.properties 公共服务（存储等信息）配置文件 │   │   ├── dolphinscheduler_env.sh alert-server环境变量配置加载脚本 │   │   └── logback-spring.xml alert-service日志配置文件 │   └── libs alert-server依赖jar包存放目录 @@ -40,17 +41,19 @@ DolphinScheduler的目录结构如下： │   │   └── start.sh DolphinScheduler api-server启动脚本 │   ├── conf │   │   ├── application.yaml api-server配置文件 +│   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 │   │   ├── common.properties 公共服务（存储等信息）配置文件 │   │   ├── dolphinscheduler_env.sh api-server环境变量配置加载脚本 │   │   └── logback-spring.xml api-service日志配置文件 │   ├── libs api-server依赖jar包存放目录 -│   └── ui api-server相关前端WEB资源存放目录 +│   └── ui api-server相关前端WEB资源存放目录 │ ├── master-server DolphinScheduler master-server命令、配置和依赖存放目录 -│   ├── bin +│   ├── bin │   │   └── start.sh DolphinScheduler master-server启动脚本 │   ├── conf │   │   ├── application.yaml master-server配置文件 +│   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 │   │   ├── common.properties 公共服务（存储等信息）配置文件 │   │   ├── dolphinscheduler_env.sh master-server环境变量配置加载脚本 │   │   └── logback-spring.xml master-service日志配置文件 @@ -61,13 +64,14 @@ DolphinScheduler的目录结构如下： │   │   └── start.sh DolphinScheduler standalone-server启动脚本 │   ├── conf │   │   ├── application.yaml standalone-server配置文件 +│   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 │   │   ├── common.properties 公共服务（存储等信息）配置文件 │   │   ├── dolphinscheduler_env.sh standalone-server环境变量配置加载脚本 │   │   ├── logback-spring.xml standalone-service日志配置文件 │   │   └── sql DolphinScheduler元数据创建/升级sql文件 │   ├── libs standalone-server依赖jar包存放目录 │   └── ui standalone-server相关前端WEB资源存放目录 -│   +│   ├── tools DolphinScheduler元数据工具命令、配置和依赖存放目录 │   ├── bin │   │   └── upgrade-schema.sh DolphinScheduler元数据创建/升级脚本 @@ -76,12 +80,13 @@ DolphinScheduler的目录结构如下： │   │   └── common.properties 公共服务（存储等信息）配置文件 │   ├── libs 元数据工具依赖jar包存放目录 │   └── sql DolphinScheduler元数据创建/升级sql文件 -│   +│   ├── worker-server DolphinScheduler worker-server命令、配置和依赖存放目录 │ ├── bin │ │   └── start.sh DolphinScheduler worker-server启动脚本 │ ├── conf │ │   ├── application.yaml worker-server配置文件 +│ │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 │ │   ├── common.properties 公共服务（存储等信息）配置文件 │ │   ├── dolphinscheduler_env.sh worker-server环境变量配置加载脚本 │ │   └── logback-spring.xml worker-service日志配置文件 @@ -93,21 +98,21 @@ DolphinScheduler的目录结构如下： # 配置文件详解 ## dolphinscheduler-daemon.sh [启动/关闭DolphinScheduler服务脚本] -dolphinscheduler-daemon.sh脚本负责DolphinScheduler的启动&关闭. +dolphinscheduler-daemon.sh脚本负责DolphinScheduler的启动&关闭. start-all.sh/stop-all.sh最终也是通过dolphinscheduler-daemon.sh对集群进行启动/关闭操作. 目前DolphinScheduler只是做了一个基本的设置,JVM参数请根据各自资源的实际情况自行设置. 默认简化参数如下: ```bash export DOLPHINSCHEDULER_OPTS=" --server --Xmx16g --Xms1g --Xss512k --XX:+UseConcMarkSweepGC --XX:+CMSParallelRemarkEnabled --XX:+UseFastAccessorMethods --XX:+UseCMSInitiatingOccupancyOnly +-server +-Xmx16g +-Xms1g +-Xss512k +-XX:+UseConcMarkSweepGC +-XX:+CMSParallelRemarkEnabled +-XX:+UseFastAccessorMethods +-XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 " ``` @@ -155,7 +160,7 @@ DolphinScheduler使用Zookeeper进行集群管理、容错、事件监听等功 默认配置如下： -|参数 |默认值| 描述| +|参数 |默认值| 描述| |--|--|--| |registry.zookeeper.namespace|dolphinscheduler|Zookeeper集群使用的namespace| |registry.zookeeper.connect-string|localhost:2181| Zookeeper集群连接信息| @@ -213,7 +218,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置 ## Api-server相关配置 位置：`api-server/conf/application.yaml` -|参数 |默认值| 描述| +|参数 |默认值| 描述| |--|--|--| |server.port|12345|api服务通讯端口| |server.servlet.session.timeout|120m|session超时时间| @@ -241,7 +246,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置 ## Master Server相关配置 位置：`master-server/conf/application.yaml` -|参数 |默认值| 描述| +|参数 |默认值| 描述| |--|--|--| |master.listen-port|5678|master监听端口| |master.fetch-command-num|10|master拉取command数量| @@ -262,7 +267,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置 ## Worker Server相关配置 位置：`worker-server/conf/application.yaml` -|参数 |默认值| 描述| +|参数 |默认值| 描述| |--|--|--| |worker.listen-port|1234|worker监听端口| |worker.exec-threads|100|worker工作线程数量,用于限制并行的任务实例数量| @@ -280,7 +285,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置 ## Alert Server相关配置 位置：`alert-server/conf/application.yaml` -|参数 |默认值| 描述| +|参数 |默认值| 描述| |--|--|--| |server.port|50053|Alert Server监听端口| |alert.port|50052|alert监听端口| diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index 6ed92bf63d..587619f9f1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -15,8 +15,7 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - - 4.0.0 @@ -25,8 +24,8 @@ dev-SNAPSHOT dolphinscheduler-alert-server - ${project.artifactId} jar + ${project.artifactId} @@ -93,6 +92,12 @@ org.codehaus.janino janino + + + org.springframework.cloud + spring-cloud-starter-kubernetes-fabric8-config + + org.mockito mockito-core @@ -123,10 +128,10 @@ dolphinscheduler-alert-server - package single + package alert-server diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 40cb4c7ce5..1a65013090 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -16,8 +16,6 @@ # spring: - application: - name: alert-server jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/bootstrap.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/bootstrap.yaml new file mode 100644 index 0000000000..347cdd07bb --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/bootstrap.yaml @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + application: + name: alert-server + cloud: + kubernetes: + reload: + enabled: true + mode: event + config: + sources: + - name: ${spring.application.name} diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 74de5d9b1f..1ded902ced 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -181,6 +181,11 @@ spring-ldap + + org.springframework.cloud + spring-cloud-starter-kubernetes-fabric8-config + + com.h2database h2 diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 874e638c12..9a8381454b 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -28,8 +28,6 @@ server: max-http-form-post-size: 5000000 spring: - application: - name: api-server banner: charset: UTF-8 jackson: diff --git a/dolphinscheduler-api/src/main/resources/bootstrap.yaml b/dolphinscheduler-api/src/main/resources/bootstrap.yaml new file mode 100644 index 0000000000..04804ab540 --- /dev/null +++ b/dolphinscheduler-api/src/main/resources/bootstrap.yaml @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + application: + name: api-server + cloud: + kubernetes: + reload: + enabled: true + mode: event + config: + sources: + - name: ${spring.application.name} diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 52c7758455..4b4ea79e77 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -521,6 +521,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt. oshi-core 6.1.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/6.1.1, MIT unirest-java 3.7.04-standalone: https://mvnrepository.com/artifact/com.konghq/unirest-java/3.7.04, MIT classgraph 4.8.83: https://mvnrepository.com/artifact/io.github.classgraph/classgraph, MIT + bcpkix-jdk15on 1.68: https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on MIT + bcprov-jdk15on 1.68: https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on MIT ======================================================================== MPL 1.1 licenses diff --git a/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java index ea60fe918c..490f6d76eb 100644 --- a/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -59,8 +59,6 @@ import org.springframework.stereotype.Component; import io.netty.channel.Channel; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - /** * logger request process logic */ @@ -80,7 +78,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { public void process(Channel channel, Command command) { logger.info("received command : {}", command); - //request task log command type + // request task log command type final CommandType commandType = command.getType(); switch (commandType) { case GET_LOG_BYTES_REQUEST: @@ -120,7 +118,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { final int MaxResponseLogSize = 65535; int totalLogByteSize = 0; for (String line : lines) { - //If a single line of log is exceed max response size, cut off the line + // If a single line of log is exceed max response size, cut off the line final int lineByteSize = line.getBytes(StandardCharsets.UTF_8).length; if (lineByteSize >= MaxResponseLogSize) { builder.append(line, 0, MaxResponseLogSize) @@ -136,7 +134,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { break; } } - RollViewLogResponseCommand rollViewLogRequestResponse = new RollViewLogResponseCommand(builder.toString()); + RollViewLogResponseCommand rollViewLogRequestResponse = + new RollViewLogResponseCommand(builder.toString()); channel.writeAndFlush(rollViewLogRequestResponse.convert2Command(command.getOpaque())); break; case REMOVE_TAK_LOG_REQUEST: @@ -161,7 +160,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { channel.writeAndFlush(removeTaskLogResponse.convert2Command(command.getOpaque())); break; case GET_APP_ID_REQUEST: - GetAppIdRequestCommand getAppIdRequestCommand = JSONUtils.parseObject(command.getBody(), GetAppIdRequestCommand.class); + GetAppIdRequestCommand getAppIdRequestCommand = + JSONUtils.parseObject(command.getBody(), GetAppIdRequestCommand.class); String logPath = getAppIdRequestCommand.getLogPath(); if (!checkPathSecurity(logPath)) { throw new IllegalArgumentException("Illegal path"); @@ -203,8 +203,9 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { * @return byte array of file */ private byte[] getFileContentBytes(String filePath) { - try (InputStream in = new FileInputStream(filePath); - ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + try ( + InputStream in = new FileInputStream(filePath); + ByteArrayOutputStream bos = new ByteArrayOutputStream()) { byte[] buf = new byte[1024]; int len; while ((len = in.read(buf)) != -1) { diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 558741fea1..d204d987fd 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -18,18 +18,29 @@ ~ under the License. ~ --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-master + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + org.apache.dolphinscheduler @@ -66,12 +77,12 @@ spring-boot-starter-cache - log4j-api org.apache.logging.log4j + log4j-api - log4j-to-slf4j org.apache.logging.log4j + log4j-to-slf4j @@ -93,8 +104,8 @@ slf4j-log4j12 - servlet-api javax.servlet + servlet-api org.codehaus.jackson @@ -118,20 +129,20 @@ hadoop-mapreduce-client-shuffle - jersey-client com.sun.jersey + jersey-client - jersey-core com.sun.jersey + jersey-core - jaxb-api javax.xml.bind + jaxb-api - log4j log4j + log4j @@ -149,8 +160,8 @@ jdk.tools - servlet-api javax.servlet + servlet-api javax.servlet @@ -214,16 +225,16 @@ jsp-api - jersey-json com.sun.jersey + jersey-json - jersey-server com.sun.jersey + jersey-server - jersey-core com.sun.jersey + jersey-core @@ -240,25 +251,20 @@ org.apache.dolphinscheduler dolphinscheduler-worker - test + test + org.apache.dolphinscheduler dolphinscheduler-log-server - - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - + + org.springframework.cloud + spring-cloud-starter-kubernetes-fabric8-config + + + @@ -278,10 +284,10 @@ dolphinscheduler-master-server - package single + package master-server diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index 4a00768278..1e5ba5d9b9 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -17,8 +17,6 @@ spring: banner: charset: UTF-8 - application: - name: master-server jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" diff --git a/dolphinscheduler-master/src/main/resources/bootstrap.yaml b/dolphinscheduler-master/src/main/resources/bootstrap.yaml new file mode 100644 index 0000000000..896590736c --- /dev/null +++ b/dolphinscheduler-master/src/main/resources/bootstrap.yaml @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + application: + name: master-server + cloud: + kubernetes: + reload: + enabled: true + mode: event + config: + sources: + - name: ${spring.application.name} diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml index 8b69742f9f..7bc450fd8a 100644 --- a/dolphinscheduler-standalone-server/pom.xml +++ b/dolphinscheduler-standalone-server/pom.xml @@ -77,10 +77,6 @@ - - org.springframework.cloud - spring-cloud-starter - org.springframework.cloud spring-cloud-starter-kubernetes-fabric8-config diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 02bd42b32b..5b4c6fdac3 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -16,8 +16,6 @@ # spring: - application: - name: standalone-server jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" diff --git a/dolphinscheduler-standalone-server/src/main/resources/bootstrap.yaml b/dolphinscheduler-standalone-server/src/main/resources/bootstrap.yaml new file mode 100644 index 0000000000..6d20924c47 --- /dev/null +++ b/dolphinscheduler-standalone-server/src/main/resources/bootstrap.yaml @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + application: + name: standalone-server + cloud: + kubernetes: + reload: + enabled: true + mode: event + config: + sources: + - name: ${spring.application.name} diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml index d648d1d410..e558deb9c7 100644 --- a/dolphinscheduler-worker/pom.xml +++ b/dolphinscheduler-worker/pom.xml @@ -18,18 +18,29 @@ ~ under the License. ~ --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-worker + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + org.apache.dolphinscheduler @@ -66,8 +77,8 @@ spring-boot-starter-tomcat - log4j-to-slf4j org.apache.logging.log4j + log4j-to-slf4j @@ -93,19 +104,13 @@ org.apache.dolphinscheduler dolphinscheduler-log-server - - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - + + org.springframework.cloud + spring-cloud-starter-kubernetes-fabric8-config + + + @@ -124,10 +129,10 @@ dolphinscheduler-worker-server - package single + package worker-server diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index fb8a1cb704..4effad23b1 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -17,8 +17,6 @@ spring: banner: charset: UTF-8 - application: - name: worker-server jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" diff --git a/dolphinscheduler-worker/src/main/resources/bootstrap.yaml b/dolphinscheduler-worker/src/main/resources/bootstrap.yaml new file mode 100644 index 0000000000..f89a64c473 --- /dev/null +++ b/dolphinscheduler-worker/src/main/resources/bootstrap.yaml @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + application: + name: worker-server + cloud: + kubernetes: + reload: + enabled: true + mode: event + config: + sources: + - name: ${spring.application.name} diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index b28957fd7a..57c4fb1c55 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -21,6 +21,8 @@ aws-java-sdk-emr-1.12.160.jar aws-java-sdk-kms-1.12.160.jar aws-java-sdk-s3-1.12.160.jar aws-java-sdk-sagemaker-1.12.160.jar +bcpkix-jdk15on-1.68.jar +bcprov-jdk15on-1.68.jar bonecp-0.8.0.RELEASE.jar byte-buddy-1.9.16.jar caffeine-2.9.3.jar @@ -243,7 +245,13 @@ spring-boot-starter-json-2.7.3.jar spring-boot-starter-logging-2.7.3.jar spring-boot-starter-quartz-2.7.3.jar spring-boot-starter-web-2.7.3.jar +spring-cloud-commons-3.1.3.jar +spring-cloud-context-3.1.3.jar +spring-cloud-kubernetes-commons-2.1.3.jar +spring-cloud-kubernetes-fabric8-autoconfig-2.1.3.jar +spring-cloud-kubernetes-fabric8-config-2.1.3.jar spring-cloud-starter-3.1.3.jar +spring-cloud-starter-bootstrap-3.1.3.jar spring-cloud-starter-kubernetes-fabric8-config-2.1.3.jar spring-context-5.3.19.jar spring-context-support-5.3.22.jar @@ -254,6 +262,9 @@ spring-jdbc-5.3.19.jar spring-ldap-1.1.2.jar spring-plugin-core-2.0.0.RELEASE.jar spring-plugin-metadata-2.0.0.RELEASE.jar +spring-retry-1.3.3.jar +spring-security-crypto-5.7.3.jar +spring-security-rsa-1.0.10.RELEASE.jar spring-tx-5.3.19.jar spring-web-5.3.22.jar spring-webmvc-5.3.22.jar