github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Add Kubernetes configmap reload to all components (#11730)

3.1.0-release
kezhenxu94 2 years ago committed by GitHub
parent
b3cc8a55b9
commit
277f137358
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 284 additions and 113 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 37
      docs/docs/en/architecture/configuration.md
  2. 53
      docs/docs/zh/architecture/configuration.md
  3. 13
      dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
  4. 2
      dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
  5. 28
      dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/bootstrap.yaml
  6. 5
      dolphinscheduler-api/pom.xml
  7. 2
      dolphinscheduler-api/src/main/resources/application.yaml
  8. 28
      dolphinscheduler-api/src/main/resources/bootstrap.yaml
  9. 2
      dolphinscheduler-dist/release-docs/LICENSE
  10. 17
      dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
  11. 64
      dolphinscheduler-master/pom.xml
  12. 2
      dolphinscheduler-master/src/main/resources/application.yaml
  13. 28
      dolphinscheduler-master/src/main/resources/bootstrap.yaml
  14. 4
      dolphinscheduler-standalone-server/pom.xml
  15. 2
      dolphinscheduler-standalone-server/src/main/resources/application.yaml
  16. 28
      dolphinscheduler-standalone-server/src/main/resources/bootstrap.yaml
  17. 41
      dolphinscheduler-worker/pom.xml
  18. 2
      dolphinscheduler-worker/src/main/resources/application.yaml
  19. 28
      dolphinscheduler-worker/src/main/resources/bootstrap.yaml
  20. 11
      tools/dependencies/known-dependencies.txt

37
docs/docs/en/architecture/configuration.md
Unescape View File

@ -17,14 +17,14 @@ The directory structure of DolphinScheduler is as follows:
│
├── licenses directory of licenses
│
├── bin directory of DolphinScheduler application commands, configrations scripts
├── bin directory of DolphinScheduler application commands, configrations scripts
│   ├── dolphinscheduler-daemon.sh script to start or shut down DolphinScheduler application
│   ├── env directory of scripts to load environment variables
│   │   ├── dolphinscheduler_env.sh script to export environment variables [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...] when you start or stop service using script `dolphinscheduler-daemon.sh`
│   │   └── install_env.sh script to export environment variables for DolphinScheduler installation when you use scripts `install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`
│   ├── install.sh script to auto-setup services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
│   ├── remove-zk-node.sh script to cleanup ZooKeeper caches
│   ├── scp-hosts.sh script to copy installation files to target hosts
│   ├── remove-zk-node.sh script to cleanup ZooKeeper caches
│   ├── scp-hosts.sh script to copy installation files to target hosts
│   ├── start-all.sh script to start all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
│   ├── status-all.sh script to check the status of all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
│   └── stop-all.sh script to shut down all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
@ -34,7 +34,8 @@ The directory structure of DolphinScheduler is as follows:
│   │   └── start.sh script to start DolphinScheduler alert-server
│   ├── conf
│   │   ├── application.yaml configurations of alert-server
│   │   ├── common.properties configurations of common-service like storage, credentials, etc.
│   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this,
│   │   ├── common.properties configurations of common-service like storage, credentials, etc.
│   │   ├── dolphinscheduler_env.sh script to load environment variables for alert-server
│   │   └── logback-spring.xml configurations of alert-service log
│   └── libs directory of alert-server libs
@ -44,17 +45,19 @@ The directory structure of DolphinScheduler is as follows:
│   │   └── start.sh script to start DolphinScheduler api-server
│   ├── conf
│   │   ├── application.yaml configurations of api-server
│   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this,
│   │   ├── common.properties configurations of common-service like storage, credentials, etc.
│   │   ├── dolphinscheduler_env.sh script to load environment variables for api-server
│   │   └── logback-spring.xml configurations of api-service log
│   ├── libs directory of api-server libs
│   └── ui directory of api-server related front-end web resources
│   └── ui directory of api-server related front-end web resources
│
├── master-server directory of DolphinScheduler master-server commands, configrations scripts and libs
│   ├── bin
│   ├── bin
│   │   └── start.sh script to start DolphinScheduler master-server
│   ├── conf
│   │   ├── application.yaml configurations of master-server
│   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this,
│   │   ├── common.properties configurations of common-service like storage, credentials, etc.
│   │   ├── dolphinscheduler_env.sh script to load environment variables for master-server
│   │   └── logback-spring.xml configurations of master-service log
@ -65,13 +68,14 @@ The directory structure of DolphinScheduler is as follows:
│   │   └── start.sh script to start DolphinScheduler standalone-server
│   ├── conf
│   │   ├── application.yaml configurations of standalone-server
│   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this,
│   │   ├── common.properties configurations of common-service like storage, credentials, etc.
│   │   ├── dolphinscheduler_env.sh script to load environment variables for standalone-server
│   │   ├── logback-spring.xml configurations of standalone-service log
│   │   └── sql .sql files to create or upgrade DolphinScheduler metadata
│   ├── libs directory of standalone-server libs
│   └── ui directory of standalone-server related front-end web resources
│  
│  
├── tools directory of DolphinScheduler metadata tools commands, configrations scripts and libs
│   ├── bin
│   │   └── upgrade-schema.sh script to initialize or upgrade DolphinScheduler metadata
@ -80,12 +84,13 @@ The directory structure of DolphinScheduler is as follows:
│   │   └── common.properties configurations of common-service like storage, credentials, etc.
│   ├── libs directory of tool libs
│   └── sql .sql files to create or upgrade DolphinScheduler metadata
│  
│  
├── worker-server directory of DolphinScheduler worker-server commands, configrations scripts and libs
│ ├── bin
│ │   └── start.sh script to start DolphinScheduler worker-server
│ ├── conf
│ │   ├── application.yaml configurations of worker-server
│ │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this,
│ │   ├── common.properties configurations of common-service like storage, credentials, etc.
│ │   ├── dolphinscheduler_env.sh script to load environment variables for worker-server
│ │   └── logback-spring.xml configurations of worker-service log
@ -107,14 +112,14 @@ Currently, DolphinScheduler just makes a basic config, remember to config furthe
Default simplified parameters are:
```bash
export DOLPHINSCHEDULER_OPTS="
-server
-Xmx16g
-Xms1g
-Xss512k
-XX:+UseConcMarkSweepGC
-XX:+CMSParallelRemarkEnabled
-XX:+UseFastAccessorMethods
-XX:+UseCMSInitiatingOccupancyOnly
-server
-Xmx16g
-Xms1g
-Xss512k
-XX:+UseConcMarkSweepGC
-XX:+CMSParallelRemarkEnabled
-XX:+UseFastAccessorMethods
-XX:+UseCMSInitiatingOccupancyOnly
-XX:CMSInitiatingOccupancyFraction=70
"
```

53
docs/docs/zh/architecture/configuration.md
Unescape View File

@ -13,13 +13,13 @@ DolphinScheduler的目录结构如下：
│
├── licenses licenses存放目录
│
├── bin DolphinScheduler命令和环境变量配置存放目录
├── bin DolphinScheduler命令和环境变量配置存放目录
│   ├── dolphinscheduler-daemon.sh 启动/关闭DolphinScheduler服务脚本
│   ├── env 环境变量配置存放目录
│   │   ├── dolphinscheduler_env.sh 当使用`dolphinscheduler-daemon.sh`脚本起停服务时，运行此脚本加载环境变量配置文件 [如：JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]
│   │   └── install_env.sh 当使用`install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`脚本时，运行此脚本为DolphinScheduler安装加载环境变量配置
│   ├── install.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本自动安装服务
│   ├── remove-zk-node.sh 清理zookeeper缓存文件脚本
│   │   ├── dolphinscheduler_env.sh 当使用`dolphinscheduler-daemon.sh`脚本起停服务时，运行此脚本加载环境变量配置文件 [如：JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]
│   │   └── install_env.sh 当使用`install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`脚本时，运行此脚本为DolphinScheduler安装加载环境变量配置
│   ├── install.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本自动安装服务
│   ├── remove-zk-node.sh 清理zookeeper缓存文件脚本
│   ├── scp-hosts.sh 安装文件传输脚本
│   ├── start-all.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本启动所有服务
│   ├── status-all.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本获取所有服务状态
@ -30,7 +30,8 @@ DolphinScheduler的目录结构如下：
│   │   └── start.sh DolphinScheduler alert-server启动脚本
│   ├── conf
│   │   ├── application.yaml alert-server配置文件
│   │   ├── common.properties 公共服务（存储等信息）配置文件
│   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改
│   │   ├── common.properties 公共服务（存储等信息）配置文件
│   │   ├── dolphinscheduler_env.sh alert-server环境变量配置加载脚本
│   │   └── logback-spring.xml alert-service日志配置文件
│   └── libs alert-server依赖jar包存放目录
@ -40,17 +41,19 @@ DolphinScheduler的目录结构如下：
│   │   └── start.sh DolphinScheduler api-server启动脚本
│   ├── conf
│   │   ├── application.yaml api-server配置文件
│   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改
│   │   ├── common.properties 公共服务（存储等信息）配置文件
│   │   ├── dolphinscheduler_env.sh api-server环境变量配置加载脚本
│   │   └── logback-spring.xml api-service日志配置文件
│   ├── libs api-server依赖jar包存放目录
│   └── ui api-server相关前端WEB资源存放目录
│   └── ui api-server相关前端WEB资源存放目录
│
├── master-server DolphinScheduler master-server命令、配置和依赖存放目录
│   ├── bin
│   ├── bin
│   │   └── start.sh DolphinScheduler master-server启动脚本
│   ├── conf
│   │   ├── application.yaml master-server配置文件
│   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改
│   │   ├── common.properties 公共服务（存储等信息）配置文件
│   │   ├── dolphinscheduler_env.sh master-server环境变量配置加载脚本
│   │   └── logback-spring.xml master-service日志配置文件
@ -61,13 +64,14 @@ DolphinScheduler的目录结构如下：
│   │   └── start.sh DolphinScheduler standalone-server启动脚本
│   ├── conf
│   │   ├── application.yaml standalone-server配置文件
│   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改
│   │   ├── common.properties 公共服务（存储等信息）配置文件
│   │   ├── dolphinscheduler_env.sh standalone-server环境变量配置加载脚本
│   │   ├── logback-spring.xml standalone-service日志配置文件
│   │   └── sql DolphinScheduler元数据创建/升级sql文件
│   ├── libs standalone-server依赖jar包存放目录
│   └── ui standalone-server相关前端WEB资源存放目录
│  
│  
├── tools DolphinScheduler元数据工具命令、配置和依赖存放目录
│   ├── bin
│   │   └── upgrade-schema.sh DolphinScheduler元数据创建/升级脚本
@ -76,12 +80,13 @@ DolphinScheduler的目录结构如下：
│   │   └── common.properties 公共服务（存储等信息）配置文件
│   ├── libs 元数据工具依赖jar包存放目录
│   └── sql DolphinScheduler元数据创建/升级sql文件
│  
│  
├── worker-server DolphinScheduler worker-server命令、配置和依赖存放目录
│ ├── bin
│ │   └── start.sh DolphinScheduler worker-server启动脚本
│ ├── conf
│ │   ├── application.yaml worker-server配置文件
│ │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改
│ │   ├── common.properties 公共服务（存储等信息）配置文件
│ │   ├── dolphinscheduler_env.sh worker-server环境变量配置加载脚本
│ │   └── logback-spring.xml worker-service日志配置文件
@ -93,21 +98,21 @@ DolphinScheduler的目录结构如下：
# 配置文件详解
## dolphinscheduler-daemon.sh [启动/关闭DolphinScheduler服务脚本]
dolphinscheduler-daemon.sh脚本负责DolphinScheduler的启动&关闭.
dolphinscheduler-daemon.sh脚本负责DolphinScheduler的启动&关闭.
start-all.sh/stop-all.sh最终也是通过dolphinscheduler-daemon.sh对集群进行启动/关闭操作.
目前DolphinScheduler只是做了一个基本的设置,JVM参数请根据各自资源的实际情况自行设置.
默认简化参数如下:
```bash
export DOLPHINSCHEDULER_OPTS="
-server
-Xmx16g
-Xms1g
-Xss512k
-XX:+UseConcMarkSweepGC
-XX:+CMSParallelRemarkEnabled
-XX:+UseFastAccessorMethods
-XX:+UseCMSInitiatingOccupancyOnly
-server
-Xmx16g
-Xms1g
-Xss512k
-XX:+UseConcMarkSweepGC
-XX:+CMSParallelRemarkEnabled
-XX:+UseFastAccessorMethods
-XX:+UseCMSInitiatingOccupancyOnly
-XX:CMSInitiatingOccupancyFraction=70
"
```
@ -155,7 +160,7 @@ DolphinScheduler使用Zookeeper进行集群管理、容错、事件监听等功
默认配置如下：
|参数 |默认值| 描述|
|参数 |默认值| 描述|
|--|--|--|
|registry.zookeeper.namespace|dolphinscheduler|Zookeeper集群使用的namespace|
|registry.zookeeper.connect-string|localhost:2181| Zookeeper集群连接信息|
@ -213,7 +218,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置
## Api-server相关配置
位置：`api-server/conf/application.yaml`
|参数 |默认值| 描述|
|参数 |默认值| 描述|
|--|--|--|
|server.port|12345|api服务通讯端口|
|server.servlet.session.timeout|120m|session超时时间|
@ -241,7 +246,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置
## Master Server相关配置
位置：`master-server/conf/application.yaml`
|参数 |默认值| 描述|
|参数 |默认值| 描述|
|--|--|--|
|master.listen-port|5678|master监听端口|
|master.fetch-command-num|10|master拉取command数量|
@ -262,7 +267,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置
## Worker Server相关配置
位置：`worker-server/conf/application.yaml`
|参数 |默认值| 描述|
|参数 |默认值| 描述|
|--|--|--|
|worker.listen-port|1234|worker监听端口|
|worker.exec-threads|100|worker工作线程数量,用于限制并行的任务实例数量|
@ -280,7 +285,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置
## Alert Server相关配置
位置：`alert-server/conf/application.yaml`
|参数 |默认值| 描述|
|参数 |默认值| 描述|
|--|--|--|
|server.port|50053|Alert Server监听端口|
|alert.port|50052|alert监听端口|

13
dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
Unescape View File

@ -15,8 +15,7 @@
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@ -25,8 +24,8 @@
<version>dev-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-alert-server</artifactId>
<name>${project.artifactId}</name>
<packaging>jar</packaging>
<name>${project.artifactId}</name>
<dependencies>
<!-- dolphinscheduler -->
@ -93,6 +92,12 @@
<groupId>org.codehaus.janino</groupId>
<artifactId>janino</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
@ -123,10 +128,10 @@
<executions>
<execution>
<id>dolphinscheduler-alert-server</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<finalName>alert-server</finalName>
<descriptors>

2
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
Unescape View File

@ -16,8 +16,6 @@
#
spring:
application:
name: alert-server
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"

28
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/bootstrap.yaml
Unescape View File

@ -0,0 +1,28 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
spring:
application:
name: alert-server
cloud:
kubernetes:
reload:
enabled: true
mode: event
config:
sources:
- name: ${spring.application.name}

5
dolphinscheduler-api/pom.xml
Unescape View File

@ -181,6 +181,11 @@
<artifactId>spring-ldap</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>

2
dolphinscheduler-api/src/main/resources/application.yaml
Unescape View File

@ -28,8 +28,6 @@ server:
max-http-form-post-size: 5000000
spring:
application:
name: api-server
banner:
charset: UTF-8
jackson:

28
dolphinscheduler-api/src/main/resources/bootstrap.yaml
Unescape View File

@ -0,0 +1,28 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
spring:
application:
name: api-server
cloud:
kubernetes:
reload:
enabled: true
mode: event
config:
sources:
- name: ${spring.application.name}

2
dolphinscheduler-dist/release-docs/LICENSE vendored
Unescape View File

@ -521,6 +521,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
oshi-core 6.1.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/6.1.1, MIT
unirest-java 3.7.04-standalone: https://mvnrepository.com/artifact/com.konghq/unirest-java/3.7.04, MIT
classgraph 4.8.83: https://mvnrepository.com/artifact/io.github.classgraph/classgraph, MIT
bcpkix-jdk15on 1.68: https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on MIT
bcprov-jdk15on 1.68: https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on MIT
========================================================================
MPL 1.1 licenses

17
dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
Unescape View File

@ -59,8 +59,6 @@ import org.springframework.stereotype.Component;
import io.netty.channel.Channel;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* logger request process logic
*/
@ -80,7 +78,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
public void process(Channel channel, Command command) {
logger.info("received command : {}", command);
//request task log command type
// request task log command type
final CommandType commandType = command.getType();
switch (commandType) {
case GET_LOG_BYTES_REQUEST:
@ -120,7 +118,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
final int MaxResponseLogSize = 65535;
int totalLogByteSize = 0;
for (String line : lines) {
//If a single line of log is exceed max response size, cut off the line
// If a single line of log is exceed max response size, cut off the line
final int lineByteSize = line.getBytes(StandardCharsets.UTF_8).length;
if (lineByteSize >= MaxResponseLogSize) {
builder.append(line, 0, MaxResponseLogSize)
@ -136,7 +134,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
break;
}
}
RollViewLogResponseCommand rollViewLogRequestResponse = new RollViewLogResponseCommand(builder.toString());
RollViewLogResponseCommand rollViewLogRequestResponse =
new RollViewLogResponseCommand(builder.toString());
channel.writeAndFlush(rollViewLogRequestResponse.convert2Command(command.getOpaque()));
break;
case REMOVE_TAK_LOG_REQUEST:
@ -161,7 +160,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
channel.writeAndFlush(removeTaskLogResponse.convert2Command(command.getOpaque()));
break;
case GET_APP_ID_REQUEST:
GetAppIdRequestCommand getAppIdRequestCommand = JSONUtils.parseObject(command.getBody(), GetAppIdRequestCommand.class);
GetAppIdRequestCommand getAppIdRequestCommand =
JSONUtils.parseObject(command.getBody(), GetAppIdRequestCommand.class);
String logPath = getAppIdRequestCommand.getLogPath();
if (!checkPathSecurity(logPath)) {
throw new IllegalArgumentException("Illegal path");
@ -203,8 +203,9 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
* @return byte array of file
*/
private byte[] getFileContentBytes(String filePath) {
try (InputStream in = new FileInputStream(filePath);
ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
try (
InputStream in = new FileInputStream(filePath);
ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
byte[] buf = new byte[1024];
int len;
while ((len = in.read(buf)) != -1) {

64
dolphinscheduler-master/pom.xml
Unescape View File

@ -18,18 +18,29 @@
~ under the License.
~
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dolphinscheduler-master</artifactId>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-bom</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
@ -66,12 +77,12 @@
<artifactId>spring-boot-starter-cache</artifactId>
<exclusions>
<exclusion>
<artifactId>log4j-api</artifactId>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
<exclusion>
<artifactId>log4j-to-slf4j</artifactId>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
@ -93,8 +104,8 @@
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
@ -118,20 +129,20 @@
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
</exclusion>
<exclusion>
<artifactId>jersey-client</artifactId>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
<artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<artifactId>jaxb-api</artifactId>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
@ -149,8 +160,8 @@
<artifactId>jdk.tools</artifactId>
</exclusion>
<exclusion>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
@ -214,16 +225,16 @@
<artifactId>jsp-api</artifactId>
</exclusion>
<exclusion>
<artifactId>jersey-json</artifactId>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<artifactId>jersey-server</artifactId>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
</exclusions>
</dependency>
@ -240,25 +251,20 @@
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-worker</artifactId>
<scope>test</scope> <!-- master should never depend on worker, this is only for tests -->
<scope>test</scope>
<!-- master should never depend on worker, this is only for tests -->
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-log-server</artifactId>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-bom</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
@ -278,10 +284,10 @@
<executions>
<execution>
<id>dolphinscheduler-master-server</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<finalName>master-server</finalName>
<descriptors>

2
dolphinscheduler-master/src/main/resources/application.yaml
Unescape View File

@ -17,8 +17,6 @@
spring:
banner:
charset: UTF-8
application:
name: master-server
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"

28
dolphinscheduler-master/src/main/resources/bootstrap.yaml
Unescape View File

@ -0,0 +1,28 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
spring:
application:
name: master-server
cloud:
kubernetes:
reload:
enabled: true
mode: event
config:
sources:
- name: ${spring.application.name}

4
dolphinscheduler-standalone-server/pom.xml
Unescape View File

@ -77,10 +77,6 @@
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>

2
dolphinscheduler-standalone-server/src/main/resources/application.yaml
Unescape View File

@ -16,8 +16,6 @@
#
spring:
application:
name: standalone-server
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"

28
dolphinscheduler-standalone-server/src/main/resources/bootstrap.yaml
Unescape View File

@ -0,0 +1,28 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
spring:
application:
name: standalone-server
cloud:
kubernetes:
reload:
enabled: true
mode: event
config:
sources:
- name: ${spring.application.name}

41
dolphinscheduler-worker/pom.xml
Unescape View File

@ -18,18 +18,29 @@
~ under the License.
~
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dolphinscheduler-worker</artifactId>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-bom</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
@ -66,8 +77,8 @@
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
<exclusion>
<artifactId>log4j-to-slf4j</artifactId>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
@ -93,19 +104,13 @@
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-log-server</artifactId>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-bom</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
@ -124,10 +129,10 @@
<executions>
<execution>
<id>dolphinscheduler-worker-server</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<finalName>worker-server</finalName>
<descriptors>

2
dolphinscheduler-worker/src/main/resources/application.yaml
Unescape View File

@ -17,8 +17,6 @@
spring:
banner:
charset: UTF-8
application:
name: worker-server
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"

28
dolphinscheduler-worker/src/main/resources/bootstrap.yaml
Unescape View File

@ -0,0 +1,28 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
spring:
application:
name: worker-server
cloud:
kubernetes:
reload:
enabled: true
mode: event
config:
sources:
- name: ${spring.application.name}

11
tools/dependencies/known-dependencies.txt
Unescape View File

@ -21,6 +21,8 @@ aws-java-sdk-emr-1.12.160.jar
aws-java-sdk-kms-1.12.160.jar
aws-java-sdk-s3-1.12.160.jar
aws-java-sdk-sagemaker-1.12.160.jar
bcpkix-jdk15on-1.68.jar
bcprov-jdk15on-1.68.jar
bonecp-0.8.0.RELEASE.jar
byte-buddy-1.9.16.jar
caffeine-2.9.3.jar
@ -243,7 +245,13 @@ spring-boot-starter-json-2.7.3.jar
spring-boot-starter-logging-2.7.3.jar
spring-boot-starter-quartz-2.7.3.jar
spring-boot-starter-web-2.7.3.jar
spring-cloud-commons-3.1.3.jar
spring-cloud-context-3.1.3.jar
spring-cloud-kubernetes-commons-2.1.3.jar
spring-cloud-kubernetes-fabric8-autoconfig-2.1.3.jar
spring-cloud-kubernetes-fabric8-config-2.1.3.jar
spring-cloud-starter-3.1.3.jar
spring-cloud-starter-bootstrap-3.1.3.jar
spring-cloud-starter-kubernetes-fabric8-config-2.1.3.jar
spring-context-5.3.19.jar
spring-context-support-5.3.22.jar
@ -254,6 +262,9 @@ spring-jdbc-5.3.19.jar
spring-ldap-1.1.2.jar
spring-plugin-core-2.0.0.RELEASE.jar
spring-plugin-metadata-2.0.0.RELEASE.jar
spring-retry-1.3.3.jar
spring-security-crypto-5.7.3.jar
spring-security-rsa-1.0.10.RELEASE.jar
spring-tx-5.3.19.jar
spring-web-5.3.22.jar
spring-webmvc-5.3.22.jar

Write Preview
Loading…
Cancel
Save