Browse Source

[Improvement][Docker/K8s] Support more configurations and service accesses, reduce the duplication configurations, improve image for python and the compatibility of python home (#5393)

* [Improvement][Docker] Support more configurations and improve image for python

* [Improvement][K8s] Support more configurations, more service accesses, and reduce the duplication configurations

* [Improvement][Python] Improve the compatibility of python home

* [Improvement][Install] Fix install config and install.sh

* [Improvement][Install] Fix workflow ut

* [Improvement][Docker] Optimize docker-swarm/check script

* [Improvement][DB] Update default username and password of database

* Update comments in master/worker.properties

* Specify version for all images

* [Improvement][Docker] Optimize PS1 and WORKDIR

* [Improvement][SQL] Reduce heap size to 64m in create-dolphinscheduler.sh and upgrade-dolphinscheduler.sh

* [Fix-5431][K8s] Fix master and worker cannot get the right address with custom DNS
pull/3/MERGE
Shiwen Cheng 4 years ago committed by GitHub
parent
commit
b537f4eb26
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 7
      .github/workflows/ci_ut.yml
  2. 12
      docker/README.md
  3. 41
      docker/build/Dockerfile
  4. 488
      docker/build/README.md
  5. 488
      docker/build/README_zh_CN.md
  6. 11
      docker/build/conf/dolphinscheduler/application-api.properties.tpl
  7. 66
      docker/build/conf/dolphinscheduler/common.properties.tpl
  8. 8
      docker/build/conf/dolphinscheduler/datasource.properties.tpl
  9. 25
      docker/build/conf/dolphinscheduler/master.properties.tpl
  10. 18
      docker/build/conf/dolphinscheduler/worker.properties.tpl
  11. 83
      docker/build/startup-init-conf.sh
  12. 51
      docker/build/startup.sh
  13. 43
      docker/docker-swarm/check
  14. 122
      docker/docker-swarm/config.env.sh
  15. 110
      docker/docker-swarm/docker-compose.yml
  16. 105
      docker/docker-swarm/docker-stack.yml
  17. 8
      docker/kubernetes/dolphinscheduler/Chart.yaml
  18. 364
      docker/kubernetes/dolphinscheduler/README.md
  19. 28
      docker/kubernetes/dolphinscheduler/templates/NOTES.txt
  20. 178
      docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
  21. 8
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
  22. 7
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
  23. 14
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml
  24. 15
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
  25. 14
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
  26. 109
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
  27. 165
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
  28. 3
      docker/kubernetes/dolphinscheduler/templates/ingress.yaml
  29. 3
      docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml
  30. 3
      docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml
  31. 3
      docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml
  32. 36
      docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml
  33. 3
      docker/kubernetes/dolphinscheduler/templates/secret-external-database.yaml
  34. 5
      docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml
  35. 178
      docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
  36. 229
      docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
  37. 8
      docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml
  38. 31
      docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml
  39. 10
      docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml
  40. 10
      docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml
  41. 99
      docker/kubernetes/dolphinscheduler/values.yaml
  42. 2
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java
  43. 2
      dolphinscheduler-alert/src/main/resources/alert.properties
  44. 2
      dolphinscheduler-api/src/main/resources/application-api.properties
  45. 34
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  46. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java
  47. 56
      dolphinscheduler-common/src/main/resources/common.properties
  48. 15
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java
  49. 8
      dolphinscheduler-dao/src/main/resources/datasource.properties
  50. 22
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
  51. 6
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
  52. 64
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
  53. 11
      dolphinscheduler-server/src/main/resources/config/install_config.conf
  54. 23
      dolphinscheduler-server/src/main/resources/master.properties
  55. 20
      dolphinscheduler-server/src/main/resources/worker.properties
  56. 82
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java
  57. 58
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java
  58. 8
      install.sh
  59. 3
      pom.xml
  60. 2
      script/create-dolphinscheduler.sh
  61. 40
      script/dolphinscheduler-daemon.sh
  62. 2
      script/upgrade-dolphinscheduler.sh

7
.github/workflows/ci_ut.yml

@ -47,10 +47,11 @@ jobs:
${{ runner.os }}-maven- ${{ runner.os }}-maven-
- name: Bootstrap database - name: Bootstrap database
run: | run: |
sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml sed -i "/image: bitnami\/postgresql/a\ ports:\n - 5432:5432" $(pwd)/docker/docker-swarm/docker-compose.yml
docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up --no-start --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql sed -i "/image: bitnami\/zookeeper/a\ ports:\n - 2181:2181" $(pwd)/docker/docker-swarm/docker-compose.yml
sudo cp $(pwd)/sql/dolphinscheduler_postgre.sql $(docker volume inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" | awk -F "\"" '{print $4}')
docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql
until docker logs docker-swarm_dolphinscheduler-postgresql_1 2>&1 | grep 'listening on IPv4 address'; do echo "waiting for postgresql ready ..."; sleep 1; done
docker run --rm --network docker-swarm_dolphinscheduler -v $(pwd)/sql/dolphinscheduler_postgre.sql:/docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql bitnami/postgresql:latest bash -c "PGPASSWORD=root psql -h docker-swarm_dolphinscheduler-postgresql_1 -U root -d dolphinscheduler -v ON_ERROR_STOP=1 -f /docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql"
- name: Set up JDK 1.8 - name: Set up JDK 1.8
uses: actions/setup-java@v1 uses: actions/setup-java@v1
with: with:

12
docker/README.md

@ -1 +1,11 @@
# DolphinScheduler for Docker # DolphinScheduler for Docker and Kubernetes
### QuickStart in Docker
[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/docker-deployment.html)
[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](https://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/docker-deployment.html)
### QuickStart in Kubernetes
[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/kubernetes-deployment.html)
[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](https://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/kubernetes-deployment.html)

41
docker/build/Dockerfile

@ -15,28 +15,35 @@
# limitations under the License. # limitations under the License.
# #
FROM openjdk:8-jdk-alpine FROM openjdk:8-jre-slim-buster
ARG VERSION ARG VERSION
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ Asia/Shanghai ENV TZ Asia/Shanghai
ENV LANG C.UTF-8 ENV LANG C.UTF-8
ENV DOCKER true ENV DOCKER true
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
# 1. install command/library/software # 1. install command/library/software
# If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: # If install slowly, you can replcae debian's mirror with new mirror, Example:
# RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories # RUN { \
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories # echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster main contrib non-free"; \
RUN apk update && \ # echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster-updates main contrib non-free"; \
apk add --no-cache tzdata dos2unix bash python2 python3 supervisor procps sudo shadow tini postgresql-client && \ # echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster-backports main contrib non-free"; \
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ # echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security buster/updates main contrib non-free"; \
apk del tzdata && \ # } > /etc/apt/sources.list
rm -rf /var/cache/apk/* RUN apt-get update && \
apt-get install -y --no-install-recommends tzdata dos2unix python supervisor procps psmisc netcat sudo tini && \
echo "Asia/Shanghai" > /etc/timezone && \
rm -f /etc/localtime && \
dpkg-reconfigure tzdata && \
rm -rf /var/lib/apt/lists/* /tmp/*
# 2. add dolphinscheduler # 2. add dolphinscheduler
ADD ./apache-dolphinscheduler-${VERSION}-bin.tar.gz /opt/ ADD ./apache-dolphinscheduler-${VERSION}-bin.tar.gz /opt/
RUN ln -s /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler RUN ln -s -r /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler WORKDIR /opt/apache-dolphinscheduler-${VERSION}-bin
# 3. add configuration and modify permissions and set soft links # 3. add configuration and modify permissions and set soft links
COPY ./checkpoint.sh /root/checkpoint.sh COPY ./checkpoint.sh /root/checkpoint.sh
@ -44,19 +51,21 @@ COPY ./startup-init-conf.sh /root/startup-init-conf.sh
COPY ./startup.sh /root/startup.sh COPY ./startup.sh /root/startup.sh
COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor.d/ COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor/conf.d/
COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/ COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/
RUN dos2unix /root/checkpoint.sh && \ RUN sed -i 's/*.conf$/*.ini/' /etc/supervisor/supervisord.conf && \
dos2unix /root/checkpoint.sh && \
dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup-init-conf.sh && \
dos2unix /root/startup.sh && \ dos2unix /root/startup.sh && \
dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \
dos2unix /opt/dolphinscheduler/bin/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \
rm -rf /bin/sh && \ rm -f /bin/sh && \
ln -s /bin/bash /bin/sh && \ ln -s /bin/bash /bin/sh && \
mkdir -p /var/mail /tmp/xls && \ mkdir -p /tmp/xls && \
echo PS1=\'\\w \\$ \' >> ~/.bashrc && \
echo "Set disable_coredump false" >> /etc/sudo.conf echo "Set disable_coredump false" >> /etc/sudo.conf
# 4. expose port # 4. expose port
EXPOSE 5678 1234 12345 50051 50052 EXPOSE 5678 1234 12345 50051 50052
ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] ENTRYPOINT ["/usr/bin/tini", "--", "/root/startup.sh"]

488
docker/build/README.md

@ -1,488 +0,0 @@
## What is DolphinScheduler?
DolphinScheduler is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.
GitHub URL: https://github.com/apache/dolphinscheduler
Official Website: https://dolphinscheduler.apache.org
![DolphinScheduler](https://dolphinscheduler.apache.org/img/hlogo_colorful.svg)
[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md)
[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md)
## Prerequisites
- [Docker](https://docs.docker.com/engine/) 1.13.1+
- [Docker Compose](https://docs.docker.com/compose/) 1.11.0+
## How to use this docker image
#### You can start a dolphinscheduler by docker-compose (recommended)
```
$ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d
```
The default **postgres** user `root`, postgres password `root` and database `dolphinscheduler` are created in the `docker-compose.yml`.
The default **zookeeper** is created in the `docker-compose.yml`.
Access the Web UI: http://192.168.xx.xx:12345/dolphinscheduler
The default username is `admin` and the default password is `dolphinscheduler123`
> **Tip**: For quick start in docker, you can create a tenant named `ds` and associate the user `admin` with the tenant `ds`
#### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`**
You can specify **existing postgres and zookeeper service**. Example:
```
$ docker run -d --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 12345:12345 \
apache/dolphinscheduler:latest all
```
Access the Web UI:http://192.168.xx.xx:12345/dolphinscheduler
#### Or start a standalone dolphinscheduler server
You can start a standalone dolphinscheduler server.
* Create a **local volume** for resource storage, For example:
```
docker volume create dolphinscheduler-resource-local
```
* Start a **master server**, For example:
```
$ docker run -d --name dolphinscheduler-master \
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
apache/dolphinscheduler:latest master-server
```
* Start a **worker server** (including **logger server**), For example:
```
$ docker run -d --name dolphinscheduler-worker \
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-e ALERT_LISTEN_HOST="dolphinscheduler-alert" \
-v dolphinscheduler-resource-local:/dolphinscheduler \
apache/dolphinscheduler:latest worker-server
```
* Start a **api server**, For example:
```
$ docker run -d --name dolphinscheduler-api \
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-v dolphinscheduler-resource-local:/dolphinscheduler \
-p 12345:12345 \
apache/dolphinscheduler:latest api-server
```
* Start a **alert server**, For example:
```
$ docker run -d --name dolphinscheduler-alert \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
apache/dolphinscheduler:latest alert-server
```
**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
## How to build a docker image
You can build a docker image in A Unix-like operating system, You can also build it in Windows operating system.
In Unix-Like, Example:
```bash
$ cd path/dolphinscheduler
$ sh ./docker/build/hooks/build
```
In Windows, Example:
```bat
C:\dolphinscheduler>.\docker\build\hooks\build.bat
```
Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script files if you don't understand
## Environment Variables
The DolphinScheduler Docker container is configured through environment variables, and the default value will be used if an environment variable is not set.
**`DATABASE_TYPE`**
This environment variable sets the type for database. The default value is `postgresql`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_DRIVER`**
This environment variable sets the type for database. The default value is `org.postgresql.Driver`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_HOST`**
This environment variable sets the host for database. The default value is `127.0.0.1`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_PORT`**
This environment variable sets the port for database. The default value is `5432`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_USERNAME`**
This environment variable sets the username for database. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_PASSWORD`**
This environment variable sets the password for database. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_DATABASE`**
This environment variable sets the database for database. The default value is `dolphinscheduler`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_PARAMS`**
This environment variable sets the database for database. The default value is `characterEncoding=utf8`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`HADOOP_HOME`**
This environment variable sets `HADOOP_HOME`. The default value is `/opt/soft/hadoop`.
**`HADOOP_CONF_DIR`**
This environment variable sets `HADOOP_CONF_DIR`. The default value is `/opt/soft/hadoop/etc/hadoop`.
**`SPARK_HOME1`**
This environment variable sets `SPARK_HOME1`. The default value is `/opt/soft/spark1`.
**`SPARK_HOME2`**
This environment variable sets `SPARK_HOME2`. The default value is `/opt/soft/spark2`.
**`PYTHON_HOME`**
This environment variable sets `PYTHON_HOME`. The default value is `/usr`.
**`JAVA_HOME`**
This environment variable sets `JAVA_HOME`. The default value is `/usr/lib/jvm/java-1.8-openjdk`.
**`HIVE_HOME`**
This environment variable sets `HIVE_HOME`. The default value is `/opt/soft/hive`.
**`FLINK_HOME`**
This environment variable sets `FLINK_HOME`. The default value is `/opt/soft/flink`.
**`DATAX_HOME`**
This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax`.
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler`
**`DOLPHINSCHEDULER_OPTS`**
This environment variable sets java options. The default value is empty.
**`RESOURCE_STORAGE_TYPE`**
This environment variable sets resource storage type for dolphinscheduler like `HDFS`, `S3`, `NONE`. The default value is `HDFS`.
**`RESOURCE_UPLOAD_PATH`**
This environment variable sets resource store path on HDFS/S3 for resource storage. The default value is `/dolphinscheduler`.
**`FS_DEFAULT_FS`**
This environment variable sets fs.defaultFS for resource storage like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler`. The default value is `file:///`.
**`FS_S3A_ENDPOINT`**
This environment variable sets s3 endpoint for resource storage. The default value is `s3.xxx.amazonaws.com`.
**`FS_S3A_ACCESS_KEY`**
This environment variable sets s3 access key for resource storage. The default value is `xxxxxxx`.
**`FS_S3A_SECRET_KEY`**
This environment variable sets s3 secret key for resource storage. The default value is `xxxxxxx`.
**`ZOOKEEPER_QUORUM`**
This environment variable sets zookeeper quorum for `master-server` and `worker-serverr`. The default value is `127.0.0.1:2181`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`.
**`ZOOKEEPER_ROOT`**
This environment variable sets zookeeper root directory for dolphinscheduler. The default value is `/dolphinscheduler`.
**`MASTER_EXEC_THREADS`**
This environment variable sets exec thread num for `master-server`. The default value is `100`.
**`MASTER_EXEC_TASK_NUM`**
This environment variable sets exec task num for `master-server`. The default value is `20`.
**`MASTER_HEARTBEAT_INTERVAL`**
This environment variable sets heartbeat interval for `master-server`. The default value is `10`.
**`MASTER_TASK_COMMIT_RETRYTIMES`**
This environment variable sets task commit retry times for `master-server`. The default value is `5`.
**`MASTER_TASK_COMMIT_INTERVAL`**
This environment variable sets task commit interval for `master-server`. The default value is `1000`.
**`MASTER_MAX_CPULOAD_AVG`**
This environment variable sets max cpu load avg for `master-server`. The default value is `100`.
**`MASTER_RESERVED_MEMORY`**
This environment variable sets reserved memory for `master-server`. The default value is `0.1`.
**`MASTER_LISTEN_PORT`**
This environment variable sets port for `master-server`. The default value is `5678`.
**`WORKER_EXEC_THREADS`**
This environment variable sets exec thread num for `worker-server`. The default value is `100`.
**`WORKER_HEARTBEAT_INTERVAL`**
This environment variable sets heartbeat interval for `worker-server`. The default value is `10`.
**`WORKER_MAX_CPULOAD_AVG`**
This environment variable sets max cpu load avg for `worker-server`. The default value is `100`.
**`WORKER_RESERVED_MEMORY`**
This environment variable sets reserved memory for `worker-server`. The default value is `0.1`.
**`WORKER_LISTEN_PORT`**
This environment variable sets port for `worker-server`. The default value is `1234`.
**`WORKER_GROUPS`**
This environment variable sets groups for `worker-server`. The default value is `default`.
**`WORKER_HOST_WEIGHT`**
This environment variable sets weight for `worker-server`. The default value is `100`.
**`ALERT_LISTEN_HOST`**
This environment variable sets the host of `alert-server` for `worker-server`. The default value is `127.0.0.1`.
**`ALERT_PLUGIN_DIR`**
This environment variable sets the alert plugin directory for `alert-server`. The default value is `lib/plugin/alert`.
## Initialization scripts
If you would like to do additional initialization in an image derived from this one, add one or more environment variable under `/root/start-init-conf.sh`, and modify template files in `/opt/dolphinscheduler/conf/*.tpl`.
For example, to add an environment variable `API_SERVER_PORT` in `/root/start-init-conf.sh`:
```
export API_SERVER_PORT=5555
```
and to modify `/opt/dolphinscheduler/conf/application-api.properties.tpl` template file, add server port:
```
server.port=${API_SERVER_PORT}
```
`/root/start-init-conf.sh` will dynamically generate config file:
```sh
echo "generate dolphinscheduler config"
ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
eval "cat << EOF
$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
EOF
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done
```
## FAQ
### How to stop dolphinscheduler by docker-compose?
Stop containers:
```
docker-compose stop
```
Stop containers and remove containers, networks and volumes:
```
docker-compose down -v
```
### How to deploy dolphinscheduler on Docker Swarm?
Assuming that the Docker Swarm cluster has been created (If there is no Docker Swarm cluster, please refer to [create-swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/))
Start a stack named dolphinscheduler
```
docker stack deploy -c docker-stack.yml dolphinscheduler
```
Stop and remove the stack named dolphinscheduler
```
docker stack rm dolphinscheduler
```
### How to use MySQL as the DolphinScheduler's database instead of PostgreSQL?
> Because of the commercial license, we cannot directly use the driver and client of MySQL.
>
> If you want to use MySQL, you can build a new image based on the `apache/dolphinscheduler` image as follows.
1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`)
2. Create a new `Dockerfile` to add MySQL driver and client:
```
FROM apache/dolphinscheduler:latest
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
RUN apk add --update --no-cache mysql-client
```
3. Build a new docker image including MySQL driver and client:
```
docker build -t apache/dolphinscheduler:mysql .
```
4. Modify all `image` fields to `apache/dolphinscheduler:mysql` in `docker-compose.yml`
> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml`
5. Comment the `dolphinscheduler-postgresql` block in `docker-compose.yml`
6. Add `dolphinscheduler-mysql` service in `docker-compose.yml` (**Optional**, you can directly use a external MySQL database)
7. Modify all DATABASE environments in `docker-compose.yml`
```
DATABASE_TYPE: mysql
DATABASE_DRIVER: com.mysql.jdbc.Driver
DATABASE_HOST: dolphinscheduler-mysql
DATABASE_PORT: 3306
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
DATABASE_PARAMS: useUnicode=true&characterEncoding=UTF-8
```
> If you have added `dolphinscheduler-mysql` service in `docker-compose.yml`, just set `DATABASE_HOST` to `dolphinscheduler-mysql`
8. Run a dolphinscheduler (See **How to use this docker image**)
### How to support MySQL datasource in `Datasource manage`?
> Because of the commercial license, we cannot directly use the driver of MySQL.
>
> If you want to add MySQL datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows.
1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`)
2. Create a new `Dockerfile` to add MySQL driver:
```
FROM apache/dolphinscheduler:latest
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
```
3. Build a new docker image including MySQL driver:
```
docker build -t apache/dolphinscheduler:mysql-driver .
```
4. Modify all `image` fields to `apache/dolphinscheduler:mysql-driver` in `docker-compose.yml`
> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml`
5. Run a dolphinscheduler (See **How to use this docker image**)
6. Add a MySQL datasource in `Datasource manage`
### How to support Oracle datasource in `Datasource manage`?
> Because of the commercial license, we cannot directly use the driver of Oracle.
>
> If you want to add Oracle datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows.
1. Download the Oracle driver [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`)
2. Create a new `Dockerfile` to add Oracle driver:
```
FROM apache/dolphinscheduler:latest
COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib
```
3. Build a new docker image including Oracle driver:
```
docker build -t apache/dolphinscheduler:oracle-driver .
```
4. Modify all `image` fields to `apache/dolphinscheduler:oracle-driver` in `docker-compose.yml`
> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml`
5. Run a dolphinscheduler (See **How to use this docker image**)
6. Add a Oracle datasource in `Datasource manage`
For more information please refer to the [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) documentation.

488
docker/build/README_zh_CN.md

@ -1,488 +0,0 @@
## DolphinScheduler是什么?
一个分布式易扩展的可视化DAG工作流任务调度系统。致力于解决数据处理流程中错综复杂的依赖关系,使调度系统在数据处理流程中`开箱即用`。
GitHub URL: https://github.com/apache/dolphinscheduler
Official Website: https://dolphinscheduler.apache.org
![DolphinScheduler](https://dolphinscheduler.apache.org/img/hlogo_colorful.svg)
[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md)
[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md)
## 先决条件
- [Docker](https://docs.docker.com/engine/) 1.13.1+
- [Docker Compose](https://docs.docker.com/compose/) 1.11.0+
## 如何使用docker镜像
#### 以 docker-compose 的方式启动dolphinscheduler(推荐)
```
$ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d
```
在`docker-compose.yml`文件中,默认的创建`Postgres`的用户、密码和数据库,默认值分别为:`root`、`root`、`dolphinscheduler`。
同时,默认的`Zookeeper`也会在`docker-compose.yml`文件中被创建。
访问前端页面:http://192.168.xx.xx:12345/dolphinscheduler
默认的用户是`admin`,默认的密码是`dolphinscheduler123`
> **提示**: 为了在docker中快速开始,你可以创建一个名为`ds`的租户,并将这个租户`ds`关联到用户`admin`
#### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
你可以指定已经存在的 **`Postgres`** 和 **`Zookeeper`** 服务. 如下:
```
$ docker run -d --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 12345:12345 \
apache/dolphinscheduler:latest all
```
访问前端页面:http://192.168.xx.xx:12345/dolphinscheduler
#### 或者运行dolphinscheduler中的部分服务
你能够运行dolphinscheduler中的部分服务。
* 创建一个 **本地卷** 用于资源存储,如下:
```
docker volume create dolphinscheduler-resource-local
```
* 启动一个 **master server**, 如下:
```
$ docker run -d --name dolphinscheduler-master \
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
apache/dolphinscheduler:latest master-server
```
* 启动一个 **worker server** (包括 **logger server**), 如下:
```
$ docker run -d --name dolphinscheduler-worker \
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-e ALERT_LISTEN_HOST="dolphinscheduler-alert" \
-v dolphinscheduler-resource-local:/dolphinscheduler \
apache/dolphinscheduler:latest worker-server
```
* 启动一个 **api server**, 如下:
```
$ docker run -d --name dolphinscheduler-api \
-e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-v dolphinscheduler-resource-local:/dolphinscheduler \
-p 12345:12345 \
apache/dolphinscheduler:latest api-server
```
* 启动一个 **alert server**, 如下:
```
$ docker run -d --name dolphinscheduler-alert \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
apache/dolphinscheduler:latest alert-server
```
**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`
## 如何构建一个docker镜像
你能够在类Unix系统和Windows系统中构建一个docker镜像。
类Unix系统, 如下:
```bash
$ cd path/dolphinscheduler
$ sh ./docker/build/hooks/build
```
Windows系统, 如下:
```bat
C:\dolphinscheduler>.\docker\build\hooks\build.bat
```
如果你不理解这些脚本 `./docker/build/hooks/build` `./docker/build/hooks/build.bat`,请阅读里面的内容。
## 环境变量
DolphinScheduler Docker 容器通过环境变量进行配置,缺省时将会使用默认值
**`DATABASE_TYPE`**
配置`database`的`TYPE`, 默认值 `postgresql`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_DRIVER`**
配置`database`的`DRIVER`, 默认值 `org.postgresql.Driver`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_HOST`**
配置`database`的`HOST`, 默认值 `127.0.0.1`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_PORT`**
配置`database`的`PORT`, 默认值 `5432`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_USERNAME`**
配置`database`的`USERNAME`, 默认值 `root`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_PASSWORD`**
配置`database`的`PASSWORD`, 默认值 `root`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_DATABASE`**
配置`database`的`DATABASE`, 默认值 `dolphinscheduler`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_PARAMS`**
配置`database`的`PARAMS`, 默认值 `characterEncoding=utf8`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`HADOOP_HOME`**
配置`dolphinscheduler`的`HADOOP_HOME`,默认值 `/opt/soft/hadoop`
**`HADOOP_CONF_DIR`**
配置`dolphinscheduler`的`HADOOP_CONF_DIR`,默认值 `/opt/soft/hadoop/etc/hadoop`
**`SPARK_HOME1`**
配置`dolphinscheduler`的`SPARK_HOME1`,默认值 `/opt/soft/spark1`
**`SPARK_HOME2`**
配置`dolphinscheduler`的`SPARK_HOME2`,默认值 `/opt/soft/spark2`
**`PYTHON_HOME`**
配置`dolphinscheduler`的`PYTHON_HOME`,默认值 `/usr`
**`JAVA_HOME`**
配置`dolphinscheduler`的`JAVA_HOME`,默认值 `/usr/lib/jvm/java-1。8-openjdk`
**`HIVE_HOME`**
配置`dolphinscheduler`的`HIVE_HOME`,默认值 `/opt/soft/hive`
**`FLINK_HOME`**
配置`dolphinscheduler`的`FLINK_HOME`,默认值 `/opt/soft/flink`
**`DATAX_HOME`**
配置`dolphinscheduler`的`DATAX_HOME`,默认值 `/opt/soft/datax`
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler`
**`DOLPHINSCHEDULER_OPTS`**
配置`dolphinscheduler`的`java options`,默认值 `""`
**`RESOURCE_STORAGE_TYPE`**
配置`dolphinscheduler`的资源存储类型,可选项为 `HDFS`、`S3`、`NONE`,默认值 `HDFS`
**`RESOURCE_UPLOAD_PATH`**
配置`HDFS/S3`上的资源存储路径,默认值 `/dolphinscheduler`
**`FS_DEFAULT_FS`**
配置资源存储的文件系统协议,如 `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler`,默认值 `file:///`
**`FS_S3A_ENDPOINT`**
当`RESOURCE_STORAGE_TYPE=S3`时,需要配置`S3`的访问路径,默认值 `s3.xxx.amazonaws.com`
**`FS_S3A_ACCESS_KEY`**
当`RESOURCE_STORAGE_TYPE=S3`时,需要配置`S3`的`s3 access key`,默认值 `xxxxxxx`
**`FS_S3A_SECRET_KEY`**
当`RESOURCE_STORAGE_TYPE=S3`时,需要配置`S3`的`s3 secret key`,默认值 `xxxxxxx`
**`ZOOKEEPER_QUORUM`**
配置`master-server`和`worker-serverr`的`Zookeeper`地址, 默认值 `127.0.0.1:2181`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`ZOOKEEPER_ROOT`**
配置`dolphinscheduler`在`zookeeper`中数据存储的根目录,默认值 `/dolphinscheduler`
**`MASTER_EXEC_THREADS`**
配置`master-server`中的执行线程数量,默认值 `100`
**`MASTER_EXEC_TASK_NUM`**
配置`master-server`中的执行任务数量,默认值 `20`
**`MASTER_HEARTBEAT_INTERVAL`**
配置`master-server`中的心跳交互时间,默认值 `10`
**`MASTER_TASK_COMMIT_RETRYTIMES`**
配置`master-server`中的任务提交重试次数,默认值 `5`
**`MASTER_TASK_COMMIT_INTERVAL`**
配置`master-server`中的任务提交交互时间,默认值 `1000`
**`MASTER_MAX_CPULOAD_AVG`**
配置`master-server`中的CPU中的`load average`值,默认值 `100`
**`MASTER_RESERVED_MEMORY`**
配置`master-server`的保留内存,默认值 `0.1`
**`MASTER_LISTEN_PORT`**
配置`master-server`的端口,默认值 `5678`
**`WORKER_EXEC_THREADS`**
配置`worker-server`中的执行线程数量,默认值 `100`
**`WORKER_HEARTBEAT_INTERVAL`**
配置`worker-server`中的心跳交互时间,默认值 `10`
**`WORKER_MAX_CPULOAD_AVG`**
配置`worker-server`中的CPU中的最大`load average`值,默认值 `100`
**`WORKER_RESERVED_MEMORY`**
配置`worker-server`的保留内存,默认值 `0.1`
**`WORKER_LISTEN_PORT`**
配置`worker-server`的端口,默认值 `1234`
**`WORKER_GROUPS`**
配置`worker-server`的分组,默认值 `default`
**`WORKER_HOST_WEIGHT`**
配置`worker-server`的权重,默认之`100`。
**`ALERT_LISTEN_HOST`**
配置`worker-server`的告警主机,即`alert-server`的主机名,默认值 `127.0.0.1`
**`ALERT_PLUGIN_DIR`**
配置`alert-server`的告警插件目录,默认值 `lib/plugin/alert`
## 初始化脚本
如果你想在编译的时候或者运行的时候附加一些其它的操作及新增一些环境变量,你可以在`/root/start-init-conf.sh`文件中进行修改,同时如果涉及到配置文件的修改,请在`/opt/dolphinscheduler/conf/*.tpl`中修改相应的配置文件
例如,在`/root/start-init-conf.sh`添加一个环境变量`API_SERVER_PORT`:
```
export API_SERVER_PORT=5555
```
当添加以上环境变量后,你应该在相应的模板文件`/opt/dolphinscheduler/conf/application-api.properties.tpl`中添加这个环境变量配置:
```
server.port=${API_SERVER_PORT}
```
`/root/start-init-conf.sh`将根据模板文件动态的生成配置文件:
```sh
echo "generate dolphinscheduler config"
ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
eval "cat << EOF
$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
EOF
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done
```
## FAQ
### 如何通过 docker-compose 停止 dolphinscheduler?
停止所有容器:
```
docker-compose stop
```
停止所有容器并移除所有容器,网络和存储卷:
```
docker-compose down -v
```
### 如何在 Docker Swarm 上部署 dolphinscheduler?
假设 Docker Swarm 集群已经部署(如果还没有创建 Docker Swarm 集群,请参考 [create-swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/))
启动名为 dolphinscheduler 的 stack
```
docker stack deploy -c docker-stack.yml dolphinscheduler
```
启动并移除名为 dolphinscheduler 的 stack
```
docker stack rm dolphinscheduler
```
### 如何用 MySQL 替代 PostgreSQL 作为 DolphinScheduler 的数据库?
> 由于商业许可证的原因,我们不能直接使用 MySQL 的驱动包和客户端.
>
> 如果你要使用 MySQL, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建.
1. 下载 MySQL 驱动包 [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (要求 `>=5.1.47`)
2. 创建一个新的 `Dockerfile`,用于添加 MySQL 的驱动包和客户端:
```
FROM apache/dolphinscheduler:latest
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
RUN apk add --update --no-cache mysql-client
```
3. 构建一个包含 MySQL 的驱动包和客户端的新镜像:
```
docker build -t apache/dolphinscheduler:mysql .
```
4. 修改 `docker-compose.yml` 文件中的所有 image 字段为 `apache/dolphinscheduler:mysql`
> 如果你想在 Docker Swarm 上部署 dolphinscheduler,你需要修改 `docker-stack.yml`
5. 注释 `docker-compose.yml` 文件中的 `dolphinscheduler-postgresql`
6. 在 `docker-compose.yml` 文件中添加 `dolphinscheduler-mysql` 服务(**可选**,你可以直接使用一个外部的 MySQL 数据库)
7. 修改 `docker-compose.yml` 文件中的所有 DATABASE 环境变量
```
DATABASE_TYPE: mysql
DATABASE_DRIVER: com.mysql.jdbc.Driver
DATABASE_HOST: dolphinscheduler-mysql
DATABASE_PORT: 3306
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
DATABASE_PARAMS: useUnicode=true&characterEncoding=UTF-8
```
> 如果你已经添加了 `dolphinscheduler-mysql` 服务,设置 `DATABASE_HOST``dolphinscheduler-mysql` 即可
8. 运行 dolphinscheduler (详见**如何使用docker镜像**)
### 如何在数据源中心支持 MySQL 数据源?
> 由于商业许可证的原因,我们不能直接使用 MySQL 的驱动包.
>
> 如果你要添加 MySQL 数据源, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建.
1. 下载 MySQL 驱动包 [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (要求 `>=5.1.47`)
2. 创建一个新的 `Dockerfile`,用于添加 MySQL 驱动包:
```
FROM apache/dolphinscheduler:latest
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
```
3. 构建一个包含 MySQL 驱动包的新镜像:
```
docker build -t apache/dolphinscheduler:mysql-driver .
```
4. 将 `docker-compose.yml` 文件中的所有 image 字段 修改为 `apache/dolphinscheduler:mysql-driver`
> 如果你想在 Docker Swarm 上部署 dolphinscheduler,你需要修改 `docker-stack.yml`
5. 运行 dolphinscheduler (详见**如何使用docker镜像**)
6. 在数据源中心添加一个 MySQL 数据源
### 如何在数据源中心支持 Oracle 数据源?
> 由于商业许可证的原因,我们不能直接使用 Oracle 的驱动包.
>
> 如果你要添加 Oracle 数据源, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建.
1. 下载 Oracle 驱动包 [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`)
2. 创建一个新的 `Dockerfile`,用于添加 Oracle 驱动包:
```
FROM apache/dolphinscheduler:latest
COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib
```
3. 构建一个包含 Oracle 驱动包的新镜像:
```
docker build -t apache/dolphinscheduler:oracle-driver .
```
4. 将 `docker-compose.yml` 文件中的所有 image 字段 修改为 `apache/dolphinscheduler:oracle-driver`
> 如果你想在 Docker Swarm 上部署 dolphinscheduler,你需要修改 `docker-stack.yml`
5. 运行 dolphinscheduler (详见**如何使用docker镜像**)
6. 在数据源中心添加一个 Oracle 数据源
更多信息请查看 [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) 文档.

11
docker/build/conf/dolphinscheduler/application-api.properties.tpl

@ -41,12 +41,21 @@ server.jetty.max-http-form-post-size=5000000
# messages encoding # messages encoding
spring.messages.encoding=UTF-8 spring.messages.encoding=UTF-8
# i18n classpath folder , file prefix messages, if have many files, use "," seperator # i18n classpath folder, file prefix messages. if have many files, use "," seperator
spring.messages.basename=i18n/messages spring.messages.basename=i18n/messages
# Authentication types (supported types: PASSWORD) # Authentication types (supported types: PASSWORD)
security.authentication.type=PASSWORD security.authentication.type=PASSWORD
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
#traffic.control.global.switch=true
#traffic.control.max.global.qps.rate=500
#traffic.control.tenant.switch=true
#traffic.control.default.tenant.qps.rate=10
#traffic.control.customize.tenant.qps.rate={'tenant1':11,'tenant2':20}
#============================================================================ #============================================================================
# LDAP Config # LDAP Config
# mock ldap server from https://www.forumsys.com/tutorials/integration-how-to/ldap/online-ldap-test-server/ # mock ldap server from https://www.forumsys.com/tutorials/integration-how-to/ldap/online-ldap-test-server/

66
docker/build/conf/dolphinscheduler/common.properties.tpl

@ -15,34 +15,37 @@
# limitations under the License. # limitations under the License.
# #
# resource storage type : HDFS, S3, NONE # user data local directory path, please make sure the directory exists and have read write permissions
data.basedir.path=${DATA_BASEDIR_PATH}
# resource storage type: HDFS, S3, NONE
resource.storage.type=${RESOURCE_STORAGE_TYPE} resource.storage.type=${RESOURCE_STORAGE_TYPE}
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.upload.path=${RESOURCE_UPLOAD_PATH} resource.upload.path=${RESOURCE_UPLOAD_PATH}
# user data local directory path, please make sure the directory exists and have read write permissions # whether to startup kerberos
data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} hadoop.security.authentication.startup.state=${HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE}
# whether kerberos starts
hadoop.security.authentication.startup.state=false
# java.security.krb5.conf path # java.security.krb5.conf path
java.security.krb5.conf.path=/opt/krb5.conf java.security.krb5.conf.path=${JAVA_SECURITY_KRB5_CONF_PATH}
# login user from keytab username # login user from keytab username
login.user.keytab.username=hdfs-mycluster@ESZ.COM login.user.keytab.username=${LOGIN_USER_KEYTAB_USERNAME}
# login user from keytab path # login user from keytab path
login.user.keytab.path=/opt/hdfs.headless.keytab login.user.keytab.path=${LOGIN_USER_KEYTAB_PATH}
# kerberos expire time, the unit is hour
kerberos.expire.time=${KERBEROS_EXPIRE_TIME}
#resource.view.suffixs # resource view suffixs
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
# if resource.storage.type=HDFS, the user need to have permission to create directories under the HDFS root path # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
hdfs.root.user=hdfs hdfs.root.user=${HDFS_ROOT_USER}
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
fs.defaultFS=${FS_DEFAULT_FS} fs.defaultFS=${FS_DEFAULT_FS}
# if resource.storage.type=S3, s3 endpoint # if resource.storage.type=S3, s3 endpoint
@ -54,25 +57,32 @@ fs.s3a.access.key=${FS_S3A_ACCESS_KEY}
# if resource.storage.type=S3, s3 secret key # if resource.storage.type=S3, s3 secret key
fs.s3a.secret.key=${FS_S3A_SECRET_KEY} fs.s3a.secret.key=${FS_S3A_SECRET_KEY}
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty # resourcemanager port, the default value is 8088 if not specified
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx resource.manager.httpaddress.port=${RESOURCE_MANAGER_HTTPADDRESS_PORT}
# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname. # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s yarn.resourcemanager.ha.rm.ids=${YARN_RESOURCEMANAGER_HA_RM_IDS}
# job history status url when application number threshold is reached(default 10000,maybe it was set to 1000) # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s yarn.application.status.address=${YARN_APPLICATION_STATUS_ADDRESS}
# system env path, If you want to set your own path, you need to set this env file to an absolute path # job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh yarn.job.history.status.address=${YARN_JOB_HISTORY_STATUS_ADDRESS}
development.state=false
# kerberos tgt expire time, unit is hours # datasource encryption enable
kerberos.expire.time=2 datasource.encryption.enable=${DATASOURCE_ENCRYPTION_ENABLE}
# datasource encryption salt # datasource encryption salt
datasource.encryption.enable=false datasource.encryption.salt=${DATASOURCE_ENCRYPTION_SALT}
datasource.encryption.salt=!@#$%^&*
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
sudo.enable=${SUDO_ENABLE}
# Network IP gets priority, default inner outer # network IP gets priority, default: inner outer
#dolphin.scheduler.network.priority.strategy=default #dolphin.scheduler.network.priority.strategy=default
# system env path
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
# development state
development.state=false

8
docker/build/conf/dolphinscheduler/datasource.properties.tpl

@ -24,14 +24,14 @@ spring.datasource.password=${DATABASE_PASSWORD}
# postgresql # postgresql
#spring.datasource.driver-class-name=org.postgresql.Driver #spring.datasource.driver-class-name=org.postgresql.Driver
#spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler #spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
#spring.datasource.username=test #spring.datasource.username=root
#spring.datasource.password=test #spring.datasource.password=root
# mysql # mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 #spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
#spring.datasource.username=xxxx #spring.datasource.username=ds_user
#spring.datasource.password=xxxx #spring.datasource.password=dolphinscheduler
# connection configuration # connection configuration
#spring.datasource.initialSize=5 #spring.datasource.initialSize=5

25
docker/build/conf/dolphinscheduler/master.properties.tpl

@ -15,29 +15,32 @@
# limitations under the License. # limitations under the License.
# #
# master execute thread number # master listen port
#master.listen.port=5678
# master execute thread number to limit process instances in parallel
master.exec.threads=${MASTER_EXEC_THREADS} master.exec.threads=${MASTER_EXEC_THREADS}
# master execute task number in parallel # master execute task number in parallel per process instance
master.exec.task.num=${MASTER_EXEC_TASK_NUM} master.exec.task.num=${MASTER_EXEC_TASK_NUM}
# master dispatch task number # master dispatch task number per batch
#master.dispatch.task.num=3 master.dispatch.task.num=${MASTER_DISPATCH_TASK_NUM}
# master host selector to select a suitable worker, default value: LowerWeight. Optional values include Random, RoundRobin, LowerWeight
master.host.selector=${MASTER_HOST_SELECTOR}
# master heartbeat interval # master heartbeat interval, the unit is second
master.heartbeat.interval=${MASTER_HEARTBEAT_INTERVAL} master.heartbeat.interval=${MASTER_HEARTBEAT_INTERVAL}
# master commit task retry times # master commit task retry times
master.task.commit.retryTimes=${MASTER_TASK_COMMIT_RETRYTIMES} master.task.commit.retryTimes=${MASTER_TASK_COMMIT_RETRYTIMES}
# master commit task interval # master commit task interval, the unit is millisecond
master.task.commit.interval=${MASTER_TASK_COMMIT_INTERVAL} master.task.commit.interval=${MASTER_TASK_COMMIT_INTERVAL}
# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 # master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
master.max.cpuload.avg=${MASTER_MAX_CPULOAD_AVG} master.max.cpuload.avg=${MASTER_MAX_CPULOAD_AVG}
# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. # master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
master.reserved.memory=${MASTER_RESERVED_MEMORY} master.reserved.memory=${MASTER_RESERVED_MEMORY}
# master listen port
master.listen.port=${MASTER_LISTEN_PORT}

18
docker/build/conf/dolphinscheduler/worker.properties.tpl

@ -15,10 +15,13 @@
# limitations under the License. # limitations under the License.
# #
# worker execute thread number # worker listener port
#worker.listen.port=1234
# worker execute thread number to limit task instances in parallel
worker.exec.threads=${WORKER_EXEC_THREADS} worker.exec.threads=${WORKER_EXEC_THREADS}
# worker heartbeat interval # worker heartbeat interval, the unit is second
worker.heartbeat.interval=${WORKER_HEARTBEAT_INTERVAL} worker.heartbeat.interval=${WORKER_HEARTBEAT_INTERVAL}
# worker host weight to dispatch tasks, default value 100 # worker host weight to dispatch tasks, default value 100
@ -27,17 +30,14 @@ worker.host.weight=${WORKER_HOST_WEIGHT}
# worker tenant auto create # worker tenant auto create
worker.tenant.auto.create=true worker.tenant.auto.create=true
# only less than cpu avg load, worker server can work. default value -1: the number of cpu cores * 2 # worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2
worker.max.cpuload.avg=${WORKER_MAX_CPULOAD_AVG} worker.max.cpuload.avg=${WORKER_MAX_CPULOAD_AVG}
# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G. # worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G
worker.reserved.memory=${WORKER_RESERVED_MEMORY} worker.reserved.memory=${WORKER_RESERVED_MEMORY}
# worker listener port # default worker groups separated by comma, like 'worker.groups=default,test'
worker.listen.port=${WORKER_LISTEN_PORT}
# default worker groups, if this worker belongs different groups, you can config the following like that 'worker.groups=default,test'
worker.groups=${WORKER_GROUPS} worker.groups=${WORKER_GROUPS}
# alert server listener host # alert server listen host
alert.listen.host=${ALERT_LISTEN_HOST} alert.listen.host=${ALERT_LISTEN_HOST}

83
docker/build/startup-init-conf.sh

@ -23,7 +23,7 @@ echo "init env variables"
# Define parameters default value # Define parameters default value
#============================================================================ #============================================================================
# Database Source # Database
#============================================================================ #============================================================================
export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"} export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"}
export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"} export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"}
@ -34,64 +34,91 @@ export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"}
export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"} export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"}
export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
#============================================================================
# ZooKeeper
#============================================================================
export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"}
export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"}
#============================================================================ #============================================================================
# Common # Common
#============================================================================ #============================================================================
# common opts
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
# common env
export DATA_BASEDIR_PATH=${DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"}
export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"}
export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"}
export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
export HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=${HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE:-"false"}
export JAVA_SECURITY_KRB5_CONF_PATH=${JAVA_SECURITY_KRB5_CONF_PATH:-"/opt/krb5.conf"}
export LOGIN_USER_KEYTAB_USERNAME=${LOGIN_USER_KEYTAB_USERNAME:-"hdfs@HADOOP.COM"}
export LOGIN_USER_KEYTAB_PATH=${LOGIN_USER_KEYTAB_PATH:-"/opt/hdfs.keytab"}
export KERBEROS_EXPIRE_TIME=${KERBEROS_EXPIRE_TIME:-"2"}
export HDFS_ROOT_USER=${HDFS_ROOT_USER:-"hdfs"}
export RESOURCE_MANAGER_HTTPADDRESS_PORT=${RESOURCE_MANAGER_HTTPADDRESS_PORT:-"8088"}
export YARN_RESOURCEMANAGER_HA_RM_IDS=${YARN_RESOURCEMANAGER_HA_RM_IDS:-""}
export YARN_APPLICATION_STATUS_ADDRESS=${YARN_APPLICATION_STATUS_ADDRESS:-"http://ds1:%s/ws/v1/cluster/apps/%s"}
export YARN_JOB_HISTORY_STATUS_ADDRESS=${YARN_JOB_HISTORY_STATUS_ADDRESS:-"http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"}
export DATASOURCE_ENCRYPTION_ENABLE=${DATASOURCE_ENCRYPTION_ENABLE:-"false"}
export DATASOURCE_ENCRYPTION_SALT=${DATASOURCE_ENCRYPTION_SALT:-"!@#$%^&*"}
export SUDO_ENABLE=${SUDO_ENABLE:-"true"}
# dolphinscheduler env # dolphinscheduler env
export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"} export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"} export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"}
export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"} export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"}
export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"} export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"}
export PYTHON_HOME=${PYTHON_HOME:-"/usr"} export PYTHON_HOME=${PYTHON_HOME:-"/usr/bin/python"}
export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"} export JAVA_HOME=${JAVA_HOME:-"/usr/local/openjdk-8"}
export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"} export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"}
export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"} export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"}
export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"} export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"}
# common env
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"}
export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"}
export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"}
export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
#============================================================================
# Zookeeper
#============================================================================
export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"}
export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"}
#============================================================================ #============================================================================
# Master Server # Master Server
#============================================================================ #============================================================================
export MASTER_SERVER_OPTS=${MASTER_SERVER_OPTS:-"-Xms1g -Xmx1g -Xmn512m"}
export MASTER_EXEC_THREADS=${MASTER_EXEC_THREADS:-"100"} export MASTER_EXEC_THREADS=${MASTER_EXEC_THREADS:-"100"}
export MASTER_EXEC_TASK_NUM=${MASTER_EXEC_TASK_NUM:-"20"} export MASTER_EXEC_TASK_NUM=${MASTER_EXEC_TASK_NUM:-"20"}
export MASTER_DISPATCH_TASK_NUM=${MASTER_DISPATCH_TASK_NUM:-"3"}
export MASTER_HOST_SELECTOR=${MASTER_HOST_SELECTOR:-"LowerWeight"}
export MASTER_HEARTBEAT_INTERVAL=${MASTER_HEARTBEAT_INTERVAL:-"10"} export MASTER_HEARTBEAT_INTERVAL=${MASTER_HEARTBEAT_INTERVAL:-"10"}
export MASTER_TASK_COMMIT_RETRYTIMES=${MASTER_TASK_COMMIT_RETRYTIMES:-"5"} export MASTER_TASK_COMMIT_RETRYTIMES=${MASTER_TASK_COMMIT_RETRYTIMES:-"5"}
export MASTER_TASK_COMMIT_INTERVAL=${MASTER_TASK_COMMIT_INTERVAL:-"1000"} export MASTER_TASK_COMMIT_INTERVAL=${MASTER_TASK_COMMIT_INTERVAL:-"1000"}
export MASTER_MAX_CPULOAD_AVG=${MASTER_MAX_CPULOAD_AVG:-"100"} export MASTER_MAX_CPULOAD_AVG=${MASTER_MAX_CPULOAD_AVG:-"-1"}
export MASTER_RESERVED_MEMORY=${MASTER_RESERVED_MEMORY:-"0.1"} export MASTER_RESERVED_MEMORY=${MASTER_RESERVED_MEMORY:-"0.3"}
export MASTER_LISTEN_PORT=${MASTER_LISTEN_PORT:-"5678"}
#============================================================================ #============================================================================
# Worker Server # Worker Server
#============================================================================ #============================================================================
export WORKER_SERVER_OPTS=${WORKER_SERVER_OPTS:-"-Xms1g -Xmx1g -Xmn512m"}
export WORKER_EXEC_THREADS=${WORKER_EXEC_THREADS:-"100"} export WORKER_EXEC_THREADS=${WORKER_EXEC_THREADS:-"100"}
export WORKER_HEARTBEAT_INTERVAL=${WORKER_HEARTBEAT_INTERVAL:-"10"} export WORKER_HEARTBEAT_INTERVAL=${WORKER_HEARTBEAT_INTERVAL:-"10"}
export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"100"}
export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.1"}
export WORKER_LISTEN_PORT=${WORKER_LISTEN_PORT:-"1234"}
export WORKER_GROUPS=${WORKER_GROUPS:-"default"}
export WORKER_HOST_WEIGHT=${WORKER_HOST_WEIGHT:-"100"} export WORKER_HOST_WEIGHT=${WORKER_HOST_WEIGHT:-"100"}
export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"} export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"-1"}
export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.3"}
export WORKER_GROUPS=${WORKER_GROUPS:-"default"}
export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"localhost"}
#============================================================================ #============================================================================
# Alert Server # Alert Server
#============================================================================ #============================================================================
export ALERT_SERVER_OPTS=${ALERT_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"} export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"}
#============================================================================
# Api Server
#============================================================================
export API_SERVER_OPTS=${API_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
#============================================================================
# Logger Server
#============================================================================
export LOGGER_SERVER_OPTS=${LOGGER_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
echo "generate dolphinscheduler config" echo "generate dolphinscheduler config"
ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
eval "cat << EOF eval "cat << EOF
@ -100,9 +127,9 @@ EOF
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done done
# generate dolphinscheduler env only in docker # generate dolphinscheduler env
DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh
if [ -z "${KUBERNETES_SERVICE_HOST}" ] && [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then if [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then
eval "cat << EOF eval "cat << EOF
$(cat ${DOLPHINSCHEDULER_ENV_PATH}.tpl) $(cat ${DOLPHINSCHEDULER_ENV_PATH}.tpl)
EOF EOF

51
docker/build/startup.sh

@ -27,7 +27,7 @@ export LOGGER_START_ENABLED=false
# wait database # wait database
waitDatabase() { waitDatabase() {
echo "test ${DATABASE_TYPE} service" echo "try to connect ${DATABASE_TYPE} ..."
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
local counter=$((counter+1)) local counter=$((counter+1))
if [ $counter == 30 ]; then if [ $counter == 30 ]; then
@ -37,21 +37,7 @@ waitDatabase() {
echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 5 sleep 5
done done
echo "${DATABASE_TYPE} connection is ok"
echo "connect ${DATABASE_TYPE} service"
if [ ${DATABASE_TYPE} = "mysql" ]; then
v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1)
if [ "$(echo ${v} | grep 'ERROR' | wc -l)" -eq 1 ]; then
echo "Error: Can't connect to database...${v}"
exit 1
fi
else
v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then
echo "Error: Can't connect to database...${v}"
exit 1
fi
fi
} }
# init database # init database
@ -60,38 +46,9 @@ initDatabase() {
${DOLPHINSCHEDULER_HOME}/script/create-dolphinscheduler.sh ${DOLPHINSCHEDULER_HOME}/script/create-dolphinscheduler.sh
} }
# check ds version
checkDSVersion() {
if [ ${DATABASE_TYPE} = "mysql" ]; then
v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "SELECT * FROM public.t_ds_version" 2>/dev/null)
else
v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "SELECT * FROM public.t_ds_version" 2>/dev/null)
fi
if [ -n "$v" ]; then
echo "ds version: $v"
return 0
else
return 1
fi
}
# check init database
checkInitDatabase() {
echo "check init database"
while ! checkDSVersion; do
local counter=$((counter+1))
if [ $counter == 30 ]; then
echo "Error: Couldn't check init database."
exit 1
fi
echo "Trying to check init database. Attempt $counter."
sleep 5
done
}
# wait zk # wait zk
waitZK() { waitZK() {
echo "connect remote zookeeper" echo "try to connect zookeeper ..."
echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
while ! nc -z ${line%:*} ${line#*:}; do while ! nc -z ${line%:*} ${line#*:}; do
local counter=$((counter+1)) local counter=$((counter+1))
@ -103,6 +60,7 @@ waitZK() {
sleep 5 sleep 5
done done
done done
echo "zookeeper connection is ok"
} }
# print usage # print usage
@ -150,7 +108,6 @@ case "$1" in
;; ;;
(alert-server) (alert-server)
waitDatabase waitDatabase
checkInitDatabase
export ALERT_START_ENABLED=true export ALERT_START_ENABLED=true
;; ;;
(help) (help)

43
docker/docker-swarm/check

@ -15,21 +15,28 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
echo "------ dolphinscheduler check - server - status -------"
sleep 60 echo "------- check server status -------"
server_num=$(docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) cd $(dirname $0)
if [ $server_num -eq 5 ] until [ $(docker-compose top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) -eq 5 ]; do
then counter=$((counter+1))
echo "Server all start successfully" if [ $counter -eq 30 ]; then
else echo "error: all servers start failed"
echo "Server start failed "$server_num exit 1
exit 1 fi
fi echo "waiting for all servers started ..."
ready=`curl http://127.0.0.1:12345/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` sleep 5
if [ $ready -eq 1 ] done
then echo "all servers start successfully"
echo "Servers is ready"
else counter=0
echo "Servers is not ready" until docker logs docker-swarm_dolphinscheduler-api_1 2>&1 | grep '0.0.0.0:12345' &>/dev/null; do
exit 1 counter=$((counter+1))
fi if [ $counter -eq 30 ]; then
echo "error: api server is not ready"
exit 1
fi
echo "waiting for api server ready ..."
sleep 5
done
echo "api server is ready"

122
docker/docker-swarm/config.env.sh

@ -0,0 +1,122 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#============================================================================
# Database
#============================================================================
# postgresql
DATABASE_TYPE=postgresql
DATABASE_DRIVER=org.postgresql.Driver
DATABASE_HOST=dolphinscheduler-postgresql
DATABASE_PORT=5432
DATABASE_USERNAME=root
DATABASE_PASSWORD=root
DATABASE_DATABASE=dolphinscheduler
DATABASE_PARAMS=characterEncoding=utf8
# mysql
# DATABASE_TYPE=mysql
# DATABASE_DRIVER=com.mysql.jdbc.Driver
# DATABASE_HOST=dolphinscheduler-mysql
# DATABASE_PORT=3306
# DATABASE_USERNAME=root
# DATABASE_PASSWORD=root
# DATABASE_DATABASE=dolphinscheduler
# DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8
#============================================================================
# ZooKeeper
#============================================================================
ZOOKEEPER_QUORUM=dolphinscheduler-zookeeper:2181
ZOOKEEPER_ROOT=/dolphinscheduler
#============================================================================
# Common
#============================================================================
# common opts
DOLPHINSCHEDULER_OPTS=
# common env
DATA_BASEDIR_PATH=/tmp/dolphinscheduler
RESOURCE_STORAGE_TYPE=HDFS
RESOURCE_UPLOAD_PATH=/dolphinscheduler
FS_DEFAULT_FS=file:///
FS_S3A_ENDPOINT=s3.xxx.amazonaws.com
FS_S3A_ACCESS_KEY=xxxxxxx
FS_S3A_SECRET_KEY=xxxxxxx
HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=false
JAVA_SECURITY_KRB5_CONF_PATH=/opt/krb5.conf
LOGIN_USER_KEYTAB_USERNAME=hdfs@HADOOP.COM
LOGIN_USER_KEYTAB_PATH=/opt/hdfs.keytab
KERBEROS_EXPIRE_TIME=2
HDFS_ROOT_USER=hdfs
RESOURCE_MANAGER_HTTPADDRESS_PORT=8088
YARN_RESOURCEMANAGER_HA_RM_IDS=
YARN_APPLICATION_STATUS_ADDRESS=http://ds1:%s/ws/v1/cluster/apps/%s
YARN_JOB_HISTORY_STATUS_ADDRESS=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
DATASOURCE_ENCRYPTION_ENABLE=false
DATASOURCE_ENCRYPTION_SALT=!@#$%^&*
SUDO_ENABLE=true
# dolphinscheduler env
HADOOP_HOME=/opt/soft/hadoop
HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
SPARK_HOME1=/opt/soft/spark1
SPARK_HOME2=/opt/soft/spark2
PYTHON_HOME=/usr/bin/python
JAVA_HOME=/usr/local/openjdk-8
HIVE_HOME=/opt/soft/hive
FLINK_HOME=/opt/soft/flink
DATAX_HOME=/opt/soft/datax
#============================================================================
# Master Server
#============================================================================
MASTER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m
MASTER_EXEC_THREADS=100
MASTER_EXEC_TASK_NUM=20
MASTER_DISPATCH_TASK_NUM=3
MASTER_HOST_SELECTOR=LowerWeight
MASTER_HEARTBEAT_INTERVAL=10
MASTER_TASK_COMMIT_RETRYTIMES=5
MASTER_TASK_COMMIT_INTERVAL=1000
MASTER_MAX_CPULOAD_AVG=-1
MASTER_RESERVED_MEMORY=0.3
#============================================================================
# Worker Server
#============================================================================
WORKER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m
WORKER_EXEC_THREADS=100
WORKER_HEARTBEAT_INTERVAL=10
WORKER_HOST_WEIGHT=100
WORKER_MAX_CPULOAD_AVG=-1
WORKER_RESERVED_MEMORY=0.3
WORKER_GROUPS=default
ALERT_LISTEN_HOST=dolphinscheduler-alert
#============================================================================
# Alert Server
#============================================================================
ALERT_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m
ALERT_PLUGIN_DIR=lib/plugin/alert
#============================================================================
# Api Server
#============================================================================
API_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m
#============================================================================
# Logger Server
#============================================================================
LOGGER_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m

110
docker/docker-swarm/docker-compose.yml

@ -19,10 +19,7 @@ version: "3.1"
services: services:
dolphinscheduler-postgresql: dolphinscheduler-postgresql:
image: bitnami/postgresql:latest image: bitnami/postgresql:11.11.0
container_name: dolphinscheduler-postgresql
ports:
- 5432:5432
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
POSTGRESQL_USERNAME: root POSTGRESQL_USERNAME: root
@ -30,16 +27,12 @@ services:
POSTGRESQL_DATABASE: dolphinscheduler POSTGRESQL_DATABASE: dolphinscheduler
volumes: volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql - dolphinscheduler-postgresql:/bitnami/postgresql
- dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d
restart: unless-stopped restart: unless-stopped
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-zookeeper: dolphinscheduler-zookeeper:
image: bitnami/zookeeper:latest image: bitnami/zookeeper:3.6.2
container_name: dolphinscheduler-zookeeper
ports:
- 2181:2181
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes" ALLOW_ANONYMOUS_LOGIN: "yes"
@ -51,26 +44,13 @@ services:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-api: dolphinscheduler-api:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:1.3.6-SNAPSHOT
container_name: dolphinscheduler-api
command: api-server command: api-server
ports: ports:
- 12345:12345 - 12345:12345
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m" env_file: config.env.sh
DATABASE_TYPE: postgresql
DATABASE_DRIVER: org.postgresql.Driver
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
DATABASE_PARAMS: characterEncoding=utf8
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
RESOURCE_STORAGE_TYPE: HDFS
RESOURCE_UPLOAD_PATH: /dolphinscheduler
FS_DEFAULT_FS: file:///
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
interval: 30s interval: 30s
@ -81,29 +61,18 @@ services:
- dolphinscheduler-zookeeper - dolphinscheduler-zookeeper
volumes: volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler - dolphinscheduler-resource-local:/dolphinscheduler
restart: unless-stopped restart: unless-stopped
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-alert: dolphinscheduler-alert:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:1.3.6-SNAPSHOT
container_name: dolphinscheduler-alert
command: alert-server command: alert-server
ports:
- 50052:50052
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
ALERT_PLUGIN_DIR: lib/plugin/alert env_file: config.env.sh
DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
DATABASE_TYPE: postgresql
DATABASE_DRIVER: org.postgresql.Driver
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
DATABASE_PARAMS: characterEncoding=utf8
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"] test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30s interval: 30s
@ -118,31 +87,11 @@ services:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-master: dolphinscheduler-master:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:1.3.6-SNAPSHOT
container_name: dolphinscheduler-master
command: master-server command: master-server
ports:
- 5678:5678
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
MASTER_EXEC_THREADS: "100" env_file: config.env.sh
MASTER_EXEC_TASK_NUM: "20"
MASTER_HEARTBEAT_INTERVAL: "10"
MASTER_TASK_COMMIT_RETRYTIMES: "5"
MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
DATABASE_TYPE: postgresql
DATABASE_DRIVER: org.postgresql.Driver
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
DATABASE_PARAMS: characterEncoding=utf8
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"] test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
interval: 30s interval: 30s
@ -153,49 +102,17 @@ services:
- dolphinscheduler-zookeeper - dolphinscheduler-zookeeper
volumes: volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
restart: unless-stopped restart: unless-stopped
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-worker: dolphinscheduler-worker:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:1.3.6-SNAPSHOT
container_name: dolphinscheduler-worker
command: worker-server command: worker-server
ports:
- 1234:1234
- 50051:50051
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
WORKER_EXEC_THREADS: "100" env_file: config.env.sh
WORKER_HEARTBEAT_INTERVAL: "10"
WORKER_MAX_CPULOAD_AVG: "100"
WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUPS: "default"
WORKER_HOST_WEIGHT: "100"
ALERT_LISTEN_HOST: dolphinscheduler-alert
HADOOP_HOME: "/opt/soft/hadoop"
HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop"
SPARK_HOME1: "/opt/soft/spark1"
SPARK_HOME2: "/opt/soft/spark2"
#PYTHON_HOME: "/opt/soft/python"
JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
HIVE_HOME: "/opt/soft/hive"
FLINK_HOME: "/opt/soft/flink"
DATAX_HOME: "/opt/soft/datax"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
DATABASE_TYPE: postgresql
DATABASE_DRIVER: org.postgresql.Driver
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
DATABASE_PARAMS: characterEncoding=utf8
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
RESOURCE_STORAGE_TYPE: HDFS
RESOURCE_UPLOAD_PATH: /dolphinscheduler
FS_DEFAULT_FS: file:///
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
interval: 30s interval: 30s
@ -207,6 +124,7 @@ services:
volumes: volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler - dolphinscheduler-resource-local:/dolphinscheduler
restart: unless-stopped restart: unless-stopped
networks: networks:
@ -218,8 +136,8 @@ networks:
volumes: volumes:
dolphinscheduler-postgresql: dolphinscheduler-postgresql:
dolphinscheduler-postgresql-initdb:
dolphinscheduler-zookeeper: dolphinscheduler-zookeeper:
dolphinscheduler-worker-data: dolphinscheduler-worker-data:
dolphinscheduler-logs: dolphinscheduler-logs:
dolphinscheduler-shared-local:
dolphinscheduler-resource-local: dolphinscheduler-resource-local:

105
docker/docker-swarm/docker-stack.yml

@ -19,9 +19,7 @@ version: "3.1"
services: services:
dolphinscheduler-postgresql: dolphinscheduler-postgresql:
image: bitnami/postgresql:latest image: bitnami/postgresql:11.11.0
ports:
- 5432:5432
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
POSTGRESQL_USERNAME: root POSTGRESQL_USERNAME: root
@ -36,9 +34,7 @@ services:
replicas: 1 replicas: 1
dolphinscheduler-zookeeper: dolphinscheduler-zookeeper:
image: bitnami/zookeeper:latest image: bitnami/zookeeper:3.6.2
ports:
- 2181:2181
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes" ALLOW_ANONYMOUS_LOGIN: "yes"
@ -52,25 +48,13 @@ services:
replicas: 1 replicas: 1
dolphinscheduler-api: dolphinscheduler-api:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:1.3.6-SNAPSHOT
command: api-server command: api-server
ports: ports:
- 12345:12345 - 12345:12345
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m" env_file: config.env.sh
DATABASE_TYPE: postgresql
DATABASE_DRIVER: org.postgresql.Driver
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
DATABASE_PARAMS: characterEncoding=utf8
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
RESOURCE_STORAGE_TYPE: HDFS
RESOURCE_UPLOAD_PATH: /dolphinscheduler
FS_DEFAULT_FS: file:///
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
interval: 30s interval: 30s
@ -78,6 +62,8 @@ services:
retries: 3 retries: 3
volumes: volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks: networks:
- dolphinscheduler - dolphinscheduler
deploy: deploy:
@ -85,22 +71,11 @@ services:
replicas: 1 replicas: 1
dolphinscheduler-alert: dolphinscheduler-alert:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:1.3.6-SNAPSHOT
command: alert-server command: alert-server
ports:
- 50052:50052
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
ALERT_PLUGIN_DIR: lib/plugin/alert env_file: config.env.sh
DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
DATABASE_TYPE: postgresql
DATABASE_DRIVER: org.postgresql.Driver
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
DATABASE_PARAMS: characterEncoding=utf8
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"] test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30s interval: 30s
@ -115,30 +90,11 @@ services:
replicas: 1 replicas: 1
dolphinscheduler-master: dolphinscheduler-master:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:1.3.6-SNAPSHOT
command: master-server command: master-server
ports:
- 5678:5678
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
MASTER_EXEC_THREADS: "100" env_file: config.env.sh
MASTER_EXEC_TASK_NUM: "20"
MASTER_HEARTBEAT_INTERVAL: "10"
MASTER_TASK_COMMIT_RETRYTIMES: "5"
MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
DATABASE_TYPE: postgresql
DATABASE_DRIVER: org.postgresql.Driver
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
DATABASE_PARAMS: characterEncoding=utf8
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"] test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
interval: 30s interval: 30s
@ -146,6 +102,7 @@ services:
retries: 3 retries: 3
volumes: volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
networks: networks:
- dolphinscheduler - dolphinscheduler
deploy: deploy:
@ -153,43 +110,11 @@ services:
replicas: 1 replicas: 1
dolphinscheduler-worker: dolphinscheduler-worker:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:1.3.6-SNAPSHOT
command: worker-server command: worker-server
ports:
- 1234:1234
- 50051:50051
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
WORKER_EXEC_THREADS: "100" env_file: config.env.sh
WORKER_HEARTBEAT_INTERVAL: "10"
WORKER_MAX_CPULOAD_AVG: "100"
WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUPS: "default"
WORKER_HOST_WEIGHT: "100"
ALERT_LISTEN_HOST: dolphinscheduler-alert
HADOOP_HOME: "/opt/soft/hadoop"
HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop"
SPARK_HOME1: "/opt/soft/spark1"
SPARK_HOME2: "/opt/soft/spark2"
#PYTHON_HOME: "/opt/soft/python"
JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
HIVE_HOME: "/opt/soft/hive"
FLINK_HOME: "/opt/soft/flink"
DATAX_HOME: "/opt/soft/datax"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
DATABASE_TYPE: postgresql
DATABASE_DRIVER: org.postgresql.Driver
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
DATABASE_PARAMS: characterEncoding=utf8
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
RESOURCE_STORAGE_TYPE: HDFS
RESOURCE_UPLOAD_PATH: /dolphinscheduler
FS_DEFAULT_FS: file:///
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
interval: 30s interval: 30s
@ -198,6 +123,8 @@ services:
volumes: volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks: networks:
- dolphinscheduler - dolphinscheduler
deploy: deploy:
@ -213,3 +140,5 @@ volumes:
dolphinscheduler-zookeeper: dolphinscheduler-zookeeper:
dolphinscheduler-worker-data: dolphinscheduler-worker-data:
dolphinscheduler-logs: dolphinscheduler-logs:
dolphinscheduler-shared-local:
dolphinscheduler-resource-local:

8
docker/kubernetes/dolphinscheduler/Chart.yaml

@ -35,18 +35,18 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes # This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version. # to the chart and its templates, including the app version.
version: 1.2.0 version: 2.0.0
# This is the version number of the application being deployed. This version number should be # This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. # incremented each time you make changes to the application.
appVersion: 1.4.0 appVersion: 1.3.6-SNAPSHOT
dependencies: dependencies:
- name: postgresql - name: postgresql
version: 10.x.x version: 10.3.18
repository: https://charts.bitnami.com/bitnami repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled condition: postgresql.enabled
- name: zookeeper - name: zookeeper
version: 6.x.x version: 6.5.3
repository: https://charts.bitnami.com/bitnami repository: https://charts.bitnami.com/bitnami
condition: zookeeper.enabled condition: zookeeper.enabled

364
docker/kubernetes/dolphinscheduler/README.md

@ -1,364 +0,0 @@
# DolphinScheduler
[DolphinScheduler](https://dolphinscheduler.apache.org) is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.
## Introduction
This chart bootstraps a [DolphinScheduler](https://dolphinscheduler.apache.org) distributed deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
## Prerequisites
- [Helm](https://helm.sh/) 3.1.0+
- [Kubernetes](https://kubernetes.io/) 1.12+
- PV provisioner support in the underlying infrastructure
## Installing the Chart
To install the chart with the release name `dolphinscheduler`:
```bash
$ git clone https://github.com/apache/dolphinscheduler.git
$ cd dolphinscheduler/docker/kubernetes/dolphinscheduler
$ helm repo add bitnami https://charts.bitnami.com/bitnami
$ helm dependency update .
$ helm install dolphinscheduler .
```
To install the chart with a namespace named `test`:
```bash
$ helm install dolphinscheduler . -n test
```
> **Tip**: If a namespace named `test` is used, the option `-n test` needs to be added to the `helm` and `kubectl` command
These commands deploy DolphinScheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
> **Tip**: List all releases using `helm list`
## Access DolphinScheduler UI
If `ingress.enabled` in `values.yaml` is set to `true`, you just access `http://${ingress.host}/dolphinscheduler` in browser.
> **Tip**: If there is a problem with ingress access, please contact the Kubernetes administrator and refer to the [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/)
Otherwise, you need to execute port-forward command like:
```bash
$ kubectl port-forward --address 0.0.0.0 svc/dolphinscheduler-api 12345:12345
$ kubectl port-forward --address 0.0.0.0 -n test svc/dolphinscheduler-api 12345:12345 # with test namespace
```
> **Tip**: If the error of `unable to do port forwarding: socat not found` appears, you need to install `socat` at first
And then access the web: http://192.168.xx.xx:12345/dolphinscheduler
The default username is `admin` and the default password is `dolphinscheduler123`
> **Tip**: For quick start in docker, you can create a tenant named `ds` and associate the user `admin` with the tenant `ds`
## Uninstalling the Chart
To uninstall/delete the `dolphinscheduler` deployment:
```bash
$ helm uninstall dolphinscheduler
```
The command removes all the Kubernetes components but PVC's associated with the chart and deletes the release.
To delete the PVC's associated with `dolphinscheduler`:
```bash
$ kubectl delete pvc -l app.kubernetes.io/instance=dolphinscheduler
```
> **Note**: Deleting the PVC's will delete all data as well. Please be cautious before doing it.
## Configuration
The Configuration file is `values.yaml`, and the following tables lists the configurable parameters of the DolphinScheduler chart and their default values.
| Parameter | Description | Default |
| --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------- |
| `timezone` | World time and date for cities in all time zones | `Asia/Shanghai` |
| | | |
| `image.repository` | Docker image repository for the DolphinScheduler | `apache/dolphinscheduler` |
| `image.tag` | Docker image version for the DolphinScheduler | `latest` |
| `image.pullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` |
| `image.pullSecret` | Image pull secret. An optional reference to secret in the same namespace to use for pulling any of the images | `nil` |
| | | |
| `postgresql.enabled` | If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL | `true` |
| `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` |
| `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` |
| `postgresql.postgresqlDatabase` | The database for internal PostgreSQL | `dolphinscheduler` |
| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` |
| `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` |
| `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database type will use it | `postgresql` |
| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database driver will use it | `org.postgresql.Driver` |
| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database host will use it | `localhost` |
| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database port will use it | `5432` |
| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database username will use it | `root` |
| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database password will use it | `root` |
| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database database will use it | `dolphinscheduler` |
| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database params will use it | `characterEncoding=utf8` |
| | | |
| `zookeeper.enabled` | If not exists external Zookeeper, by default, the DolphinScheduler will use a internal Zookeeper | `true` |
| `zookeeper.fourlwCommandsWhitelist` | A list of comma separated Four Letter Words commands to use | `srvr,ruok,wchs,cons` |
| `zookeeper.service.port` | ZooKeeper port | `2181` |
| `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` |
| `zookeeper.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` |
| `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `zookeeper.zookeeperRoot` | Specify dolphinscheduler root directory in Zookeeper | `/dolphinscheduler` |
| `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enabled` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` |
| `externalZookeeper.zookeeperRoot` | If exists external Zookeeper, and set `zookeeper.enabled` value to false. Specify dolphinscheduler root directory in Zookeeper | `/dolphinscheduler` |
| | | |
| `common.configmap.DOLPHINSCHEDULER_ENV` | System env path, self configuration, please read `values.yaml` | `[]` |
| `common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | User data directory path, self configuration, please make sure the directory exists and have read write permissions | `/tmp/dolphinscheduler` |
| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource storage type: HDFS, S3, NONE | `HDFS` |
| `common.configmap.RESOURCE_UPLOAD_PATH` | Resource store on HDFS/S3 path, please make sure the directory exists on hdfs and have read write permissions | `/dolphinscheduler` |
| `common.configmap.FS_DEFAULT_FS` | Resource storage file system like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler` | `file:///` |
| `common.configmap.FS_S3A_ENDPOINT` | S3 endpoint when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `s3.xxx.amazonaws.com` |
| `common.configmap.FS_S3A_ACCESS_KEY` | S3 access key when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `xxxxxxx` |
| `common.configmap.FS_S3A_SECRET_KEY` | S3 secret key when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `xxxxxxx` |
| `common.fsFileResourcePersistence.enabled` | Set `common.fsFileResourcePersistence.enabled` to `true` to mount a new file resource volume for `api` and `worker` | `false` |
| `common.fsFileResourcePersistence.accessModes` | `PersistentVolumeClaim` Access Modes, must be `ReadWriteMany` | `[ReadWriteMany]` |
| `common.fsFileResourcePersistence.storageClassName` | Resource Persistent Volume Storage Class, must support the access mode: ReadWriteMany | `-` |
| `common.fsFileResourcePersistence.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | |
| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `master.annotations` | The `annotations` for master server | `{}` |
| `master.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `master.tolerations` | If specified, the pod's tolerations | `{}` |
| `master.resources` | The `resource` limit and request config for master server | `{}` |
| `master.configmap.DOLPHINSCHEDULER_OPTS` | The java options for master server | `""` |
| `master.configmap.MASTER_EXEC_THREADS` | Master execute thread number | `100` |
| `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` |
| `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` |
| `master.configmap.MASTER_TASK_COMMIT_RETRYTIMES` | Master commit task retry times | `5` |
| `master.configmap.MASTER_TASK_COMMIT_INTERVAL` | Master commit task interval | `1000` |
| `master.configmap.MASTER_MAX_CPULOAD_AVG` | Only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 | `100` |
| `master.configmap.MASTER_RESERVED_MEMORY` | Only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G | `0.1` |
| `master.configmap.MASTER_LISTEN_PORT` | Master listen port | `5678` |
| `master.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `master.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `master.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
| `master.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
| `master.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
| `master.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
| `master.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
| `master.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
| `master.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
| `master.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
| `master.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
| `master.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
| `master.persistentVolumeClaim.enabled` | Set `master.persistentVolumeClaim.enabled` to `true` to mount a new volume for `master` | `false` |
| `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
| `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | |
| `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `worker.annotations` | The `annotations` for worker server | `{}` |
| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `worker.tolerations` | If specified, the pod's tolerations | `{}` |
| `worker.resources` | The `resource` limit and request config for worker server | `{}` |
| `worker.configmap.DOLPHINSCHEDULER_OPTS` | The java options for worker server | `""` |
| `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread number | `100` |
| `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` |
| `worker.configmap.WORKER_MAX_CPULOAD_AVG` | Only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 | `100` |
| `worker.configmap.WORKER_RESERVED_MEMORY` | Only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G | `0.1` |
| `worker.configmap.WORKER_LISTEN_PORT` | Worker listen port | `1234` |
| `worker.configmap.WORKER_GROUPS` | Worker groups | `default` |
| `worker.configmap.WORKER_HOST_WEIGHT` | Worker host weight | `100` |
| `worker.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `worker.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `worker.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
| `worker.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
| `worker.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
| `worker.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
| `worker.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
| `worker.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
| `worker.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
| `worker.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
| `worker.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
| `worker.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
| `worker.persistentVolumeClaim.enabled` | Set `worker.persistentVolumeClaim.enabled` to `true` to enable `persistentVolumeClaim` for `worker` | `false` |
| `worker.persistentVolumeClaim.dataPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.dataPersistentVolume.enabled` to `true` to mount a data volume for `worker` | `false` |
| `worker.persistentVolumeClaim.dataPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
| `worker.persistentVolumeClaim.dataPersistentVolume.storageClassName` | `Worker` data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `worker.persistentVolumeClaim.dataPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| `worker.persistentVolumeClaim.logsPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.logsPersistentVolume.enabled` to `true` to mount a logs volume for `worker` | `false` |
| `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
| `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | |
| `alert.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
| `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
| `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
| `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` |
| `alert.annotations` | The `annotations` for alert server | `{}` |
| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `alert.tolerations` | If specified, the pod's tolerations | `{}` |
| `alert.resources` | The `resource` limit and request config for alert server | `{}` |
| `alert.configmap.DOLPHINSCHEDULER_OPTS` | The java options for alert server | `""` |
| `alert.configmap.ALERT_PLUGIN_DIR` | Alert plugin directory | `lib/plugin/alert` |
| `alert.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `alert.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `alert.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
| `alert.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
| `alert.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
| `alert.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
| `alert.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
| `alert.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
| `alert.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
| `alert.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
| `alert.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
| `alert.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
| `alert.persistentVolumeClaim.enabled` | Set `alert.persistentVolumeClaim.enabled` to `true` to mount a new volume for `alert` | `false` |
| `alert.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
| `alert.persistentVolumeClaim.storageClassName` | `Alert` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `alert.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | |
| `api.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
| `api.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
| `api.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
| `api.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` |
| `api.annotations` | The `annotations` for api server | `{}` |
| `api.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `api.tolerations` | If specified, the pod's tolerations | `{}` |
| `api.resources` | The `resource` limit and request config for api server | `{}` |
| `api.configmap.DOLPHINSCHEDULER_OPTS` | The java options for api server | `""` |
| `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
| `api.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
| `api.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
| `api.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
| `api.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
| `api.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
| `api.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
| `api.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
| `api.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
| `api.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
| `api.persistentVolumeClaim.enabled` | Set `api.persistentVolumeClaim.enabled` to `true` to mount a new volume for `api` | `false` |
| `api.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
| `api.persistentVolumeClaim.storageClassName` | `api` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `api.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | |
| `ingress.enabled` | Enable ingress | `false` |
| `ingress.host` | Ingress host | `dolphinscheduler.org` |
| `ingress.path` | Ingress path | `/dolphinscheduler` |
| `ingress.tls.enabled` | Enable ingress tls | `false` |
| `ingress.tls.secretName` | Ingress tls secret name | `dolphinscheduler-tls` |
## FAQ
### How to use MySQL as the DolphinScheduler's database instead of PostgreSQL?
> Because of the commercial license, we cannot directly use the driver and client of MySQL.
>
> If you want to use MySQL, you can build a new image based on the `apache/dolphinscheduler` image as follows.
1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`)
2. Create a new `Dockerfile` to add MySQL driver and client:
```
FROM apache/dolphinscheduler:latest
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
RUN apk add --update --no-cache mysql-client
```
3. Build a new docker image including MySQL driver and client:
```
docker build -t apache/dolphinscheduler:mysql .
```
4. Push the docker image `apache/dolphinscheduler:mysql` to a docker registry
5. Modify image `repository` and update `tag` to `mysql` in `values.yaml`
6. Modify postgresql `enabled` to `false`
7. Modify externalDatabase (especially modify `host`, `username` and `password`):
```
externalDatabase:
type: "mysql"
driver: "com.mysql.jdbc.Driver"
host: "localhost"
port: "3306"
username: "root"
password: "root"
database: "dolphinscheduler"
params: "useUnicode=true&characterEncoding=UTF-8"
```
8. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**)
### How to support MySQL datasource in `Datasource manage`?
> Because of the commercial license, we cannot directly use the driver of MySQL.
>
> If you want to add MySQL datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows.
1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`)
2. Create a new `Dockerfile` to add MySQL driver:
```
FROM apache/dolphinscheduler:latest
COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib
```
3. Build a new docker image including MySQL driver:
```
docker build -t apache/dolphinscheduler:mysql-driver .
```
4. Push the docker image `apache/dolphinscheduler:mysql-driver` to a docker registry
5. Modify image `repository` and update `tag` to `mysql-driver` in `values.yaml`
6. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**)
7. Add a MySQL datasource in `Datasource manage`
### How to support Oracle datasource in `Datasource manage`?
> Because of the commercial license, we cannot directly use the driver of Oracle.
>
> If you want to add Oracle datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows.
1. Download the Oracle driver [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`)
2. Create a new `Dockerfile` to add Oracle driver:
```
FROM apache/dolphinscheduler:latest
COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib
```
3. Build a new docker image including Oracle driver:
```
docker build -t apache/dolphinscheduler:oracle-driver .
```
4. Push the docker image `apache/dolphinscheduler:oracle-driver` to a docker registry
5. Modify image `repository` and update `tag` to `oracle-driver` in `values.yaml`
6. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**)
7. Add a Oracle datasource in `Datasource manage`
For more information please refer to the [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) documentation.

28
docker/kubernetes/dolphinscheduler/templates/NOTES.txt

@ -17,16 +17,34 @@
** Please be patient while the chart DolphinScheduler {{ .Chart.AppVersion }} is being deployed ** ** Please be patient while the chart DolphinScheduler {{ .Chart.AppVersion }} is being deployed **
Access DolphinScheduler by: Access DolphinScheduler UI URL by:
{{- if .Values.ingress.enabled }} {{- if .Values.ingress.enabled }}
DolphinScheduler URL: http://{{ .Values.ingress.host }}/dolphinscheduler DolphinScheduler UI URL: http{{ if .Values.ingress.tls.enabled }}s{{ end }}://{{ .Values.ingress.host }}/dolphinscheduler
{{- else }} {{- else if eq .Values.api.service.type "ClusterIP" }}
kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-api 12345:12345 kubectl port-forward -n {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-api 12345:12345
DolphinScheduler URL: http://127.0.0.1:12345/dolphinscheduler DolphinScheduler UI URL: http://127.0.0.1:12345/dolphinscheduler
{{- else if eq .Values.api.service.type "NodePort" }}
NODE_IP=$(kubectl get no -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
NODE_PORT=$(kubectl get svc {{ template "dolphinscheduler.fullname" . }}-api -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}")
echo http://$NODE_IP:$NODE_PORT/dolphinscheduler
DolphinScheduler UI URL: http://$NODE_IP:$NODE_PORT/dolphinscheduler
{{- else if eq .Values.api.service.type "LoadBalancer" }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get svc {{ template "dolphinscheduler.fullname" . }}-api -n {{ .Release.Namespace }} -w'
SERVICE_IP=$(kubectl get svc {{ template "dolphinscheduler.fullname" . }}-api -n {{ .Release.Namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
echo http://$SERVICE_IP:12345/dolphinscheduler
DolphinScheduler UI URL: http://$SERVICE_IP:12345/dolphinscheduler
{{- end }} {{- end }}

178
docker/kubernetes/dolphinscheduler/templates/_helpers.tpl

@ -30,7 +30,52 @@ If release name contains chart name it will be used as a full name.
Create a default docker image fullname. Create a default docker image fullname.
*/}} */}}
{{- define "dolphinscheduler.image.fullname" -}} {{- define "dolphinscheduler.image.fullname" -}}
{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}} {{- .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion -}}
{{- end -}}
{{/*
Create a default common labels.
*/}}
{{- define "dolphinscheduler.common.labels" -}}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/version: {{ .Chart.AppVersion }}
{{- end -}}
{{/*
Create a master labels.
*/}}
{{- define "dolphinscheduler.master.labels" -}}
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
app.kubernetes.io/component: master
{{ include "dolphinscheduler.common.labels" . }}
{{- end -}}
{{/*
Create a worker labels.
*/}}
{{- define "dolphinscheduler.worker.labels" -}}
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
app.kubernetes.io/component: worker
{{ include "dolphinscheduler.common.labels" . }}
{{- end -}}
{{/*
Create an alert labels.
*/}}
{{- define "dolphinscheduler.alert.labels" -}}
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
app.kubernetes.io/component: alert
{{ include "dolphinscheduler.common.labels" . }}
{{- end -}}
{{/*
Create an api labels.
*/}}
{{- define "dolphinscheduler.api.labels" -}}
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
app.kubernetes.io/component: api
{{ include "dolphinscheduler.common.labels" . }}
{{- end -}} {{- end -}}
{{/* {{/*
@ -58,3 +103,134 @@ Create a default fully qualified zookkeeper quorum.
{{- $port := default "2181" (.Values.zookeeper.service.port | toString) -}} {{- $port := default "2181" (.Values.zookeeper.service.port | toString) -}}
{{- printf "%s:%s" (include "dolphinscheduler.zookeeper.fullname" .) $port -}} {{- printf "%s:%s" (include "dolphinscheduler.zookeeper.fullname" .) $port -}}
{{- end -}} {{- end -}}
{{/*
Create a database environment variables.
*/}}
{{- define "dolphinscheduler.database.env_vars" -}}
- name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
name: {{ template "dolphinscheduler.postgresql.fullname" . }}
key: postgresql-password
{{- else }}
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
key: database-password
{{- end }}
- name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
{{- end -}}
{{/*
Create a zookeeper environment variables.
*/}}
{{- define "dolphinscheduler.zookeeper.env_vars" -}}
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: ZOOKEEPER_ROOT
{{- if .Values.zookeeper.enabled }}
value: {{ .Values.zookeeper.zookeeperRoot }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperRoot }}
{{- end }}
{{- end -}}
{{/*
Create a common fs_s3a environment variables.
*/}}
{{- define "dolphinscheduler.fs_s3a.env_vars" -}}
{{- if eq (default "HDFS" .Values.common.configmap.RESOURCE_STORAGE_TYPE) "S3" -}}
- name: FS_S3A_SECRET_KEY
valueFrom:
secretKeyRef:
key: fs-s3a-secret-key
name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
{{- end -}}
{{- end -}}
{{/*
Create a sharedStoragePersistence volume.
*/}}
{{- define "dolphinscheduler.sharedStorage.volume" -}}
{{- if .Values.common.sharedStoragePersistence.enabled -}}
- name: {{ include "dolphinscheduler.fullname" . }}-shared
persistentVolumeClaim:
claimName: {{ include "dolphinscheduler.fullname" . }}-shared
{{- end -}}
{{- end -}}
{{/*
Create a sharedStoragePersistence volumeMount.
*/}}
{{- define "dolphinscheduler.sharedStorage.volumeMount" -}}
{{- if .Values.common.sharedStoragePersistence.enabled -}}
- mountPath: {{ .Values.common.sharedStoragePersistence.mountPath | quote }}
name: {{ include "dolphinscheduler.fullname" . }}-shared
{{- end -}}
{{- end -}}
{{/*
Create a fsFileResourcePersistence volume.
*/}}
{{- define "dolphinscheduler.fsFileResource.volume" -}}
{{- if .Values.common.fsFileResourcePersistence.enabled -}}
- name: {{ include "dolphinscheduler.fullname" . }}-fs-file
persistentVolumeClaim:
claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file
{{- end -}}
{{- end -}}
{{/*
Create a fsFileResourcePersistence volumeMount.
*/}}
{{- define "dolphinscheduler.fsFileResource.volumeMount" -}}
{{- if .Values.common.fsFileResourcePersistence.enabled -}}
- mountPath: {{ default "/dolphinscheduler" .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
name: {{ include "dolphinscheduler.fullname" . }}-fs-file
{{- end -}}
{{- end -}}

8
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml

@ -21,9 +21,9 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-alert name: {{ include "dolphinscheduler.fullname" . }}-alert
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data: data:
DOLPHINSCHEDULER_OPTS: {{ .Values.alert.configmap.DOLPHINSCHEDULER_OPTS | quote }} {{- range $key, $value := .Values.alert.configmap }}
ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }} {{ $key }}: {{ $value | quote }}
{{- end }}
{{- end }} {{- end }}

7
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml

@ -21,8 +21,9 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-api name: {{ include "dolphinscheduler.fullname" . }}-api
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data: data:
DOLPHINSCHEDULER_OPTS: {{ .Values.api.configmap.DOLPHINSCHEDULER_OPTS | quote }} {{- range $key, $value := .Values.api.configmap }}
{{ $key }}: {{ $value | quote }}
{{- end }}
{{- end }} {{- end }}

14
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml

@ -21,17 +21,9 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-common name: {{ include "dolphinscheduler.fullname" . }}-common
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data: data:
DOLPHINSCHEDULER_ENV: |- {{- range $key, $value := (omit .Values.common.configmap "FS_S3A_SECRET_KEY") }}
{{- range .Values.common.configmap.DOLPHINSCHEDULER_ENV }} {{ $key }}: {{ $value | quote }}
{{ . }}
{{- end }} {{- end }}
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }}
RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }}
RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }}
FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }}
FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }}
{{- end }} {{- end }}

15
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml

@ -21,16 +21,9 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-master name: {{ include "dolphinscheduler.fullname" . }}-master
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data: data:
DOLPHINSCHEDULER_OPTS: {{ .Values.master.configmap.DOLPHINSCHEDULER_OPTS | quote }} {{- range $key, $value := .Values.master.configmap }}
MASTER_EXEC_THREADS: {{ .Values.master.configmap.MASTER_EXEC_THREADS | quote }} {{ $key }}: {{ $value | quote }}
MASTER_EXEC_TASK_NUM: {{ .Values.master.configmap.MASTER_EXEC_TASK_NUM | quote }} {{- end }}
MASTER_HEARTBEAT_INTERVAL: {{ .Values.master.configmap.MASTER_HEARTBEAT_INTERVAL | quote }}
MASTER_TASK_COMMIT_RETRYTIMES: {{ .Values.master.configmap.MASTER_TASK_COMMIT_RETRYTIMES | quote }}
MASTER_TASK_COMMIT_INTERVAL: {{ .Values.master.configmap.MASTER_TASK_COMMIT_INTERVAL | quote }}
MASTER_MAX_CPULOAD_AVG: {{ .Values.master.configmap.MASTER_MAX_CPULOAD_AVG | quote }}
MASTER_RESERVED_MEMORY: {{ .Values.master.configmap.MASTER_RESERVED_MEMORY | quote }}
MASTER_LISTEN_PORT: {{ .Values.master.configmap.MASTER_LISTEN_PORT | quote }}
{{- end }} {{- end }}

14
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml

@ -21,15 +21,9 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-worker name: {{ include "dolphinscheduler.fullname" . }}-worker
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data: data:
DOLPHINSCHEDULER_OPTS: {{ .Values.worker.configmap.DOLPHINSCHEDULER_OPTS | quote }} {{- range $key, $value := .Values.worker.configmap }}
WORKER_EXEC_THREADS: {{ .Values.worker.configmap.WORKER_EXEC_THREADS | quote }} {{ $key }}: {{ $value | quote }}
WORKER_HEARTBEAT_INTERVAL: {{ .Values.worker.configmap.WORKER_HEARTBEAT_INTERVAL | quote }} {{- end }}
WORKER_MAX_CPULOAD_AVG: {{ .Values.worker.configmap.WORKER_MAX_CPULOAD_AVG | quote }}
WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }}
WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }}
WORKER_GROUPS: {{ .Values.worker.configmap.WORKER_GROUPS | quote }}
WORKER_HOST_WEIGHT: {{ .Values.worker.configmap.WORKER_HOST_WEIGHT | quote }}
{{- end }} {{- end }}

109
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@ -19,18 +19,12 @@ kind: Deployment
metadata: metadata:
name: {{ include "dolphinscheduler.fullname" . }}-alert name: {{ include "dolphinscheduler.fullname" . }}-alert
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert {{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: alert
spec: spec:
replicas: {{ .Values.alert.replicas }} replicas: {{ .Values.alert.replicas }}
selector: selector:
matchLabels: matchLabels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert {{- include "dolphinscheduler.alert.labels" . | nindent 6 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: alert
strategy: strategy:
type: {{ .Values.alert.strategy.type | quote }} type: {{ .Values.alert.strategy.type | quote }}
rollingUpdate: rollingUpdate:
@ -39,23 +33,23 @@ spec:
template: template:
metadata: metadata:
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert {{- include "dolphinscheduler.alert.labels" . | nindent 8 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: alert
{{- if .Values.alert.annotations }} {{- if .Values.alert.annotations }}
annotations: annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }} {{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }} {{- end }}
spec: spec:
{{- if .Values.alert.affinity }} {{- if .Values.alert.affinity }}
affinity: {{- toYaml .Values.alert.affinity | nindent 8 }} affinity:
{{- toYaml .Values.alert.affinity | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.alert.nodeSelector }} {{- if .Values.alert.nodeSelector }}
nodeSelector: {{- toYaml .Values.alert.nodeSelector | nindent 8 }} nodeSelector:
{{- toYaml .Values.alert.nodeSelector | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.alert.tolerations }} {{- if .Values.alert.tolerations }}
tolerations: {{- toYaml . | nindent 8 }} tolerations:
{{- toYaml .Values.alert.tolerations | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.image.pullSecret }} {{- if .Values.image.pullSecret }}
imagePullSecrets: imagePullSecrets:
@ -73,84 +67,20 @@ spec:
env: env:
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: DOLPHINSCHEDULER_OPTS {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
valueFrom: envFrom:
configMapKeyRef: - configMapRef:
key: DOLPHINSCHEDULER_OPTS name: {{ include "dolphinscheduler.fullname" . }}-common
name: {{ include "dolphinscheduler.fullname" . }}-alert - configMapRef:
- name: ALERT_PLUGIN_DIR name: {{ include "dolphinscheduler.fullname" . }}-alert
valueFrom:
configMapKeyRef:
key: ALERT_PLUGIN_DIR
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
name: {{ template "dolphinscheduler.postgresql.fullname" . }}
key: postgresql-password
{{- else }}
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
key: database-password
{{- end }}
- name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
{{- if .Values.alert.resources }} {{- if .Values.alert.resources }}
resources: resources:
limits: {{- toYaml .Values.alert.resources | nindent 12 }}
memory: {{ .Values.alert.resources.limits.memory }}
cpu: {{ .Values.alert.resources.limits.cpu }}
requests:
memory: {{ .Values.alert.resources.requests.memory }}
cpu: {{ .Values.alert.resources.requests.cpu }}
{{- end }} {{- end }}
{{- if .Values.alert.livenessProbe.enabled }} {{- if .Values.alert.livenessProbe.enabled }}
livenessProbe: livenessProbe:
exec: exec:
command: command: ["bash", "/root/checkpoint.sh", "AlertServer"]
- sh
- /root/checkpoint.sh
- AlertServer
initialDelaySeconds: {{ .Values.alert.livenessProbe.initialDelaySeconds }} initialDelaySeconds: {{ .Values.alert.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.alert.livenessProbe.periodSeconds }} periodSeconds: {{ .Values.alert.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.alert.livenessProbe.timeoutSeconds }} timeoutSeconds: {{ .Values.alert.livenessProbe.timeoutSeconds }}
@ -160,10 +90,7 @@ spec:
{{- if .Values.alert.readinessProbe.enabled }} {{- if .Values.alert.readinessProbe.enabled }}
readinessProbe: readinessProbe:
exec: exec:
command: command: ["bash", "/root/checkpoint.sh", "AlertServer"]
- sh
- /root/checkpoint.sh
- AlertServer
initialDelaySeconds: {{ .Values.alert.readinessProbe.initialDelaySeconds }} initialDelaySeconds: {{ .Values.alert.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.alert.readinessProbe.periodSeconds }} periodSeconds: {{ .Values.alert.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.alert.readinessProbe.timeoutSeconds }} timeoutSeconds: {{ .Values.alert.readinessProbe.timeoutSeconds }}

165
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@ -19,18 +19,12 @@ kind: Deployment
metadata: metadata:
name: {{ include "dolphinscheduler.fullname" . }}-api name: {{ include "dolphinscheduler.fullname" . }}-api
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api {{- include "dolphinscheduler.api.labels" . | nindent 4 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: api
spec: spec:
replicas: {{ .Values.api.replicas }} replicas: {{ .Values.api.replicas }}
selector: selector:
matchLabels: matchLabels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api {{- include "dolphinscheduler.api.labels" . | nindent 6 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: api
strategy: strategy:
type: {{ .Values.api.strategy.type | quote }} type: {{ .Values.api.strategy.type | quote }}
rollingUpdate: rollingUpdate:
@ -39,23 +33,23 @@ spec:
template: template:
metadata: metadata:
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api {{- include "dolphinscheduler.api.labels" . | nindent 8 }}
app.kubernetes.io/instance: {{ .Release.Name }} {{- if .Values.api.annotations }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: api
{{- if .Values.alert.annotations }}
annotations: annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }} {{- toYaml .Values.api.annotations | nindent 8 }}
{{- end }} {{- end }}
spec: spec:
{{- if .Values.api.affinity }} {{- if .Values.api.affinity }}
affinity: {{- toYaml .Values.api.affinity | nindent 8 }} affinity:
{{- toYaml .Values.api.affinity | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.api.nodeSelector }} {{- if .Values.api.nodeSelector }}
nodeSelector: {{- toYaml .Values.api.nodeSelector | nindent 8 }} nodeSelector:
{{- toYaml .Values.api.nodeSelector | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.api.tolerations }} {{- if .Values.api.tolerations }}
tolerations: {{- toYaml . | nindent 8 }} tolerations:
{{- toYaml .Values.api.tolerations | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.image.pullSecret }} {{- if .Values.image.pullSecret }}
imagePullSecrets: imagePullSecrets:
@ -73,120 +67,22 @@ spec:
env: env:
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: DOLPHINSCHEDULER_OPTS {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
valueFrom: {{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
configMapKeyRef: {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
key: DOLPHINSCHEDULER_OPTS envFrom:
name: {{ include "dolphinscheduler.fullname" . }}-api - configMapRef:
- name: DATABASE_TYPE name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if .Values.postgresql.enabled }} - configMapRef:
value: "postgresql" name: {{ include "dolphinscheduler.fullname" . }}-api
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
name: {{ template "dolphinscheduler.postgresql.fullname" . }}
key: postgresql-password
{{- else }}
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
key: database-password
{{- end }}
- name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: ZOOKEEPER_ROOT
{{- if .Values.zookeeper.enabled }}
value: {{ .Values.zookeeper.zookeeperRoot }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperRoot }}
{{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }}
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
secretKeyRef:
key: fs-s3a-secret-key
name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
{{- end }}
{{- if .Values.api.resources }} {{- if .Values.api.resources }}
resources: resources:
limits: {{- toYaml .Values.api.resources | nindent 12 }}
memory: {{ .Values.api.resources.limits.memory | quote }}
cpu: {{ .Values.api.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.api.resources.requests.memory | quote }}
cpu: {{ .Values.api.resources.requests.cpu | quote }}
{{- end }} {{- end }}
{{- if .Values.api.livenessProbe.enabled }} {{- if .Values.api.livenessProbe.enabled }}
livenessProbe: livenessProbe:
tcpSocket: exec:
port: 12345 command: ["bash", "/root/checkpoint.sh", "ApiApplicationServer"]
initialDelaySeconds: {{ .Values.api.livenessProbe.initialDelaySeconds }} initialDelaySeconds: {{ .Values.api.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.api.livenessProbe.periodSeconds }} periodSeconds: {{ .Values.api.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.api.livenessProbe.timeoutSeconds }} timeoutSeconds: {{ .Values.api.livenessProbe.timeoutSeconds }}
@ -195,8 +91,8 @@ spec:
{{- end }} {{- end }}
{{- if .Values.api.readinessProbe.enabled }} {{- if .Values.api.readinessProbe.enabled }}
readinessProbe: readinessProbe:
tcpSocket: exec:
port: 12345 command: ["bash", "/root/checkpoint.sh", "ApiApplicationServer"]
initialDelaySeconds: {{ .Values.api.readinessProbe.initialDelaySeconds }} initialDelaySeconds: {{ .Values.api.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.api.readinessProbe.periodSeconds }} periodSeconds: {{ .Values.api.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.api.readinessProbe.timeoutSeconds }} timeoutSeconds: {{ .Values.api.readinessProbe.timeoutSeconds }}
@ -206,10 +102,8 @@ spec:
volumeMounts: volumeMounts:
- mountPath: "/opt/dolphinscheduler/logs" - mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-api name: {{ include "dolphinscheduler.fullname" . }}-api
{{- if .Values.common.fsFileResourcePersistence.enabled }} {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
- mountPath: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
name: {{ include "dolphinscheduler.fullname" . }}-fs-file
{{- end }}
volumes: volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-api - name: {{ include "dolphinscheduler.fullname" . }}-api
{{- if .Values.api.persistentVolumeClaim.enabled }} {{- if .Values.api.persistentVolumeClaim.enabled }}
@ -218,8 +112,5 @@ spec:
{{- else }} {{- else }}
emptyDir: {} emptyDir: {}
{{- end }} {{- end }}
{{- if .Values.common.fsFileResourcePersistence.enabled }} {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
- name: {{ include "dolphinscheduler.fullname" . }}-fs-file {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
persistentVolumeClaim:
claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file
{{- end }}

3
docker/kubernetes/dolphinscheduler/templates/ingress.yaml

@ -27,8 +27,7 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }} name: {{ include "dolphinscheduler.fullname" . }}
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }} app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
rules: rules:
- host: {{ .Values.ingress.host }} - host: {{ .Values.ingress.host }}

3
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml

@ -21,8 +21,7 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-alert name: {{ include "dolphinscheduler.fullname" . }}-alert
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
accessModes: accessModes:
{{- range .Values.alert.persistentVolumeClaim.accessModes }} {{- range .Values.alert.persistentVolumeClaim.accessModes }}

3
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml

@ -21,8 +21,7 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-api name: {{ include "dolphinscheduler.fullname" . }}-api
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
accessModes: accessModes:
{{- range .Values.api.persistentVolumeClaim.accessModes }} {{- range .Values.api.persistentVolumeClaim.accessModes }}

3
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml

@ -21,8 +21,7 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-fs-file name: {{ include "dolphinscheduler.fullname" . }}-fs-file
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-file app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-file
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
annotations: annotations:
"helm.sh/resource-policy": keep "helm.sh/resource-policy": keep
spec: spec:

36
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml

@ -0,0 +1,36 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if .Values.common.sharedStoragePersistence.enabled }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-shared
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-shared
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
annotations:
"helm.sh/resource-policy": keep
spec:
accessModes:
{{- range .Values.common.sharedStoragePersistence.accessModes }}
- {{ . | quote }}
{{- end }}
storageClassName: {{ .Values.common.sharedStoragePersistence.storageClassName | quote }}
resources:
requests:
storage: {{ .Values.common.sharedStoragePersistence.storage | quote }}
{{- end }}

3
docker/kubernetes/dolphinscheduler/templates/secret-external-database.yaml

@ -21,8 +21,7 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-externaldb name: {{ include "dolphinscheduler.fullname" . }}-externaldb
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-externaldb app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-externaldb
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
type: Opaque type: Opaque
data: data:
database-password: {{ .Values.externalDatabase.password | b64enc | quote }} database-password: {{ .Values.externalDatabase.password | b64enc | quote }}

5
docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml

@ -14,15 +14,14 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }} {{- if eq (default "HDFS" .Values.common.configmap.RESOURCE_STORAGE_TYPE) "S3" }}
apiVersion: v1 apiVersion: v1
kind: Secret kind: Secret
metadata: metadata:
name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
type: Opaque type: Opaque
data: data:
fs-s3a-secret-key: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | b64enc | quote }} fs-s3a-secret-key: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | b64enc | quote }}

178
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@ -19,40 +19,34 @@ kind: StatefulSet
metadata: metadata:
name: {{ include "dolphinscheduler.fullname" . }}-master name: {{ include "dolphinscheduler.fullname" . }}-master
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master {{- include "dolphinscheduler.master.labels" . | nindent 4 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: master
spec: spec:
podManagementPolicy: {{ .Values.master.podManagementPolicy }} podManagementPolicy: {{ .Values.master.podManagementPolicy }}
replicas: {{ .Values.master.replicas }} replicas: {{ .Values.master.replicas }}
selector: selector:
matchLabels: matchLabels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master {{- include "dolphinscheduler.master.labels" . | nindent 6 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: master
serviceName: {{ template "dolphinscheduler.fullname" . }}-master-headless serviceName: {{ template "dolphinscheduler.fullname" . }}-master-headless
template: template:
metadata: metadata:
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master {{- include "dolphinscheduler.master.labels" . | nindent 8 }}
app.kubernetes.io/instance: {{ .Release.Name }} {{- if .Values.master.annotations }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: master
{{- if .Values.alert.annotations }}
annotations: annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }} {{- toYaml .Values.master.annotations | nindent 8 }}
{{- end }} {{- end }}
spec: spec:
{{- if .Values.master.affinity }} {{- if .Values.master.affinity }}
affinity: {{- toYaml .Values.master.affinity | nindent 8 }} affinity:
{{- toYaml .Values.master.affinity | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.master.nodeSelector }} {{- if .Values.master.nodeSelector }}
nodeSelector: {{- toYaml .Values.master.nodeSelector | nindent 8 }} nodeSelector:
{{- toYaml .Values.master.nodeSelector | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.master.tolerations }} {{- if .Values.master.tolerations }}
tolerations: {{- toYaml . | nindent 8 }} tolerations:
{{- toYaml .Values.master.tolerations | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.image.pullSecret }} {{- if .Values.image.pullSecret }}
imagePullSecrets: imagePullSecrets:
@ -65,141 +59,27 @@ spec:
args: args:
- "master-server" - "master-server"
ports: ports:
- containerPort: {{ .Values.master.configmap.MASTER_LISTEN_PORT }} - containerPort: 5678
name: "master-port" name: "master-port"
env: env:
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: DOLPHINSCHEDULER_OPTS {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
valueFrom: {{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
configMapKeyRef: {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
key: DOLPHINSCHEDULER_OPTS envFrom:
name: {{ include "dolphinscheduler.fullname" . }}-master - configMapRef:
- name: MASTER_EXEC_THREADS name: {{ include "dolphinscheduler.fullname" . }}-common
valueFrom: - configMapRef:
configMapKeyRef: name: {{ include "dolphinscheduler.fullname" . }}-master
name: {{ include "dolphinscheduler.fullname" . }}-master
key: MASTER_EXEC_THREADS
- name: MASTER_EXEC_TASK_NUM
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: MASTER_EXEC_TASK_NUM
- name: MASTER_HEARTBEAT_INTERVAL
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: MASTER_HEARTBEAT_INTERVAL
- name: MASTER_TASK_COMMIT_RETRYTIMES
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: MASTER_TASK_COMMIT_RETRYTIMES
- name: MASTER_TASK_COMMIT_INTERVAL
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: MASTER_TASK_COMMIT_INTERVAL
- name: MASTER_MAX_CPULOAD_AVG
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: MASTER_MAX_CPULOAD_AVG
- name: MASTER_RESERVED_MEMORY
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: MASTER_RESERVED_MEMORY
- name: MASTER_LISTEN_PORT
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: MASTER_LISTEN_PORT
- name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-common
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
- name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
name: {{ template "dolphinscheduler.postgresql.fullname" . }}
key: postgresql-password
{{- else }}
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
key: database-password
{{- end }}
- name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: ZOOKEEPER_ROOT
{{- if .Values.zookeeper.enabled }}
value: {{ .Values.zookeeper.zookeeperRoot }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperRoot }}
{{- end }}
{{- if .Values.master.resources }} {{- if .Values.master.resources }}
resources: resources:
limits: {{- toYaml .Values.master.resources | nindent 12 }}
memory: {{ .Values.master.resources.limits.memory | quote }}
cpu: {{ .Values.master.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.master.resources.requests.memory | quote }}
cpu: {{ .Values.master.resources.requests.cpu | quote }}
{{- end }} {{- end }}
{{- if .Values.master.livenessProbe.enabled }} {{- if .Values.master.livenessProbe.enabled }}
livenessProbe: livenessProbe:
exec: exec:
command: command: ["bash", "/root/checkpoint.sh", "MasterServer"]
- sh
- /root/checkpoint.sh
- MasterServer
initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }} initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }} periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.master.livenessProbe.timeoutSeconds }} timeoutSeconds: {{ .Values.master.livenessProbe.timeoutSeconds }}
@ -209,10 +89,7 @@ spec:
{{- if .Values.master.readinessProbe.enabled }} {{- if .Values.master.readinessProbe.enabled }}
readinessProbe: readinessProbe:
exec: exec:
command: command: ["bash", "/root/checkpoint.sh", "MasterServer"]
- sh
- /root/checkpoint.sh
- MasterServer
initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }} initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }} periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.master.readinessProbe.timeoutSeconds }} timeoutSeconds: {{ .Values.master.readinessProbe.timeoutSeconds }}
@ -222,6 +99,7 @@ spec:
volumeMounts: volumeMounts:
- mountPath: "/opt/dolphinscheduler/logs" - mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-master name: {{ include "dolphinscheduler.fullname" . }}-master
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
volumes: volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-master - name: {{ include "dolphinscheduler.fullname" . }}-master
{{- if .Values.master.persistentVolumeClaim.enabled }} {{- if .Values.master.persistentVolumeClaim.enabled }}
@ -230,19 +108,19 @@ spec:
{{- else }} {{- else }}
emptyDir: {} emptyDir: {}
{{- end }} {{- end }}
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
{{- if .Values.master.persistentVolumeClaim.enabled }} {{- if .Values.master.persistentVolumeClaim.enabled }}
volumeClaimTemplates: volumeClaimTemplates:
- metadata: - metadata:
name: {{ include "dolphinscheduler.fullname" . }}-master name: {{ include "dolphinscheduler.fullname" . }}-master
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 10 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
accessModes: accessModes:
{{- range .Values.master.persistentVolumeClaim.accessModes }} {{- range .Values.master.persistentVolumeClaim.accessModes }}
- {{ . | quote }} - {{ . | quote }}
{{- end }} {{- end }}
storageClassName: {{ .Values.master.persistentVolumeClaim.storageClassName | quote }} storageClassName: {{ .Values.master.persistentVolumeClaim.storageClassName | quote }}
resources: resources:
requests: requests:

229
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@ -19,40 +19,34 @@ kind: StatefulSet
metadata: metadata:
name: {{ include "dolphinscheduler.fullname" . }}-worker name: {{ include "dolphinscheduler.fullname" . }}-worker
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker {{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: worker
spec: spec:
podManagementPolicy: {{ .Values.worker.podManagementPolicy }} podManagementPolicy: {{ .Values.worker.podManagementPolicy }}
replicas: {{ .Values.worker.replicas }} replicas: {{ .Values.worker.replicas }}
selector: selector:
matchLabels: matchLabels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker {{- include "dolphinscheduler.worker.labels" . | nindent 6 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: worker
serviceName: {{ template "dolphinscheduler.fullname" . }}-worker-headless serviceName: {{ template "dolphinscheduler.fullname" . }}-worker-headless
template: template:
metadata: metadata:
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker {{- include "dolphinscheduler.worker.labels" . | nindent 8 }}
app.kubernetes.io/instance: {{ .Release.Name }} {{- if .Values.worker.annotations }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: worker
{{- if .Values.alert.annotations }}
annotations: annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }} {{- toYaml .Values.worker.annotations | nindent 8 }}
{{- end }} {{- end }}
spec: spec:
{{- if .Values.worker.affinity }} {{- if .Values.worker.affinity }}
affinity: {{- toYaml .Values.worker.affinity | nindent 8 }} affinity:
{{- toYaml .Values.worker.affinity | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.worker.nodeSelector }} {{- if .Values.worker.nodeSelector }}
nodeSelector: {{- toYaml .Values.worker.nodeSelector | nindent 8 }} nodeSelector:
{{- toYaml .Values.worker.nodeSelector | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.worker.tolerations }} {{- if .Values.worker.tolerations }}
tolerations: {{- toYaml . | nindent 8 }} tolerations:
{{- toYaml .Values.worker.tolerations | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.image.pullSecret }} {{- if .Values.image.pullSecret }}
imagePullSecrets: imagePullSecrets:
@ -65,172 +59,33 @@ spec:
args: args:
- "worker-server" - "worker-server"
ports: ports:
- containerPort: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }} - containerPort: 1234
name: "worker-port" name: "worker-port"
- containerPort: 50051 - containerPort: 50051
name: "logger-port" name: "logger-port"
env: env:
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: DOLPHINSCHEDULER_OPTS
valueFrom:
configMapKeyRef:
key: DOLPHINSCHEDULER_OPTS
name: {{ include "dolphinscheduler.fullname" . }}-worker
- name: WORKER_EXEC_THREADS
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_EXEC_THREADS
- name: WORKER_HEARTBEAT_INTERVAL
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_HEARTBEAT_INTERVAL
- name: WORKER_MAX_CPULOAD_AVG
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_MAX_CPULOAD_AVG
- name: WORKER_RESERVED_MEMORY
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_RESERVED_MEMORY
- name: WORKER_LISTEN_PORT
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_LISTEN_PORT
- name: WORKER_GROUPS
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_GROUPS
- name: WORKER_WEUGHT
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_HOST_WEIGHT
- name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-common
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
- name: ALERT_LISTEN_HOST - name: ALERT_LISTEN_HOST
value: {{ include "dolphinscheduler.fullname" . }}-alert value: {{ include "dolphinscheduler.fullname" . }}-alert
- name: DATABASE_TYPE {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }}
{{- if .Values.postgresql.enabled }} {{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }}
value: "postgresql" {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }}
{{- else }} envFrom:
value: {{ .Values.externalDatabase.type | quote }} - configMapRef:
{{- end }} name: {{ include "dolphinscheduler.fullname" . }}-common
- name: DATABASE_DRIVER - configMapRef:
{{- if .Values.postgresql.enabled }} name: {{ include "dolphinscheduler.fullname" . }}-worker
value: "org.postgresql.Driver" - configMapRef:
{{- else }} name: {{ include "dolphinscheduler.fullname" . }}-alert
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
name: {{ template "dolphinscheduler.postgresql.fullname" . }}
key: postgresql-password
{{- else }}
name: {{ include "dolphinscheduler.fullname" . }}-externaldb
key: database-password
{{- end }}
- name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: ZOOKEEPER_ROOT
{{- if .Values.zookeeper.enabled }}
value: {{ .Values.zookeeper.zookeeperRoot }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperRoot }}
{{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }}
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
secretKeyRef:
key: fs-s3a-secret-key
name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
{{- end }}
{{- if .Values.worker.resources }} {{- if .Values.worker.resources }}
resources: resources:
limits: {{- toYaml .Values.worker.resources | nindent 12 }}
memory: {{ .Values.worker.resources.limits.memory | quote }}
cpu: {{ .Values.worker.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.worker.resources.requests.memory | quote }}
cpu: {{ .Values.worker.resources.requests.cpu | quote }}
{{- end }} {{- end }}
{{- if .Values.worker.livenessProbe.enabled }} {{- if .Values.worker.livenessProbe.enabled }}
livenessProbe: livenessProbe:
exec: exec:
command: command: ["bash", "/root/checkpoint.sh", "WorkerServer"]
- sh
- /root/checkpoint.sh
- WorkerServer
initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }} initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }} periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }} timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }}
@ -240,10 +95,7 @@ spec:
{{- if .Values.worker.readinessProbe.enabled }} {{- if .Values.worker.readinessProbe.enabled }}
readinessProbe: readinessProbe:
exec: exec:
command: command: ["bash", "/root/checkpoint.sh", "WorkerServer"]
- sh
- /root/checkpoint.sh
- WorkerServer
initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }}
@ -251,17 +103,12 @@ spec:
failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }}
{{- end }} {{- end }}
volumeMounts: volumeMounts:
- mountPath: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }} - mountPath: {{ default "/tmp/dolphinscheduler" .Values.common.configmap.DATA_BASEDIR_PATH | quote }}
name: {{ include "dolphinscheduler.fullname" . }}-worker-data name: {{ include "dolphinscheduler.fullname" . }}-worker-data
- mountPath: "/opt/dolphinscheduler/logs" - mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-worker-logs name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
- mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh" {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
subPath: "dolphinscheduler_env.sh" {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
name: {{ include "dolphinscheduler.fullname" . }}-common-env
{{- if .Values.common.fsFileResourcePersistence.enabled }}
- mountPath: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
name: {{ include "dolphinscheduler.fullname" . }}-fs-file
{{- end }}
volumes: volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-worker-data - name: {{ include "dolphinscheduler.fullname" . }}-worker-data
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
@ -277,18 +124,8 @@ spec:
{{- else }} {{- else }}
emptyDir: {} emptyDir: {}
{{- end }} {{- end }}
- name: {{ include "dolphinscheduler.fullname" . }}-common-env {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
configMap: {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
defaultMode: 0777
name: {{ include "dolphinscheduler.fullname" . }}-common
items:
- key: DOLPHINSCHEDULER_ENV
path: dolphinscheduler_env.sh
{{- if .Values.common.fsFileResourcePersistence.enabled }}
- name: {{ include "dolphinscheduler.fullname" . }}-fs-file
persistentVolumeClaim:
claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file
{{- end }}
{{- if .Values.worker.persistentVolumeClaim.enabled }} {{- if .Values.worker.persistentVolumeClaim.enabled }}
volumeClaimTemplates: volumeClaimTemplates:
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
@ -296,8 +133,7 @@ spec:
name: {{ include "dolphinscheduler.fullname" . }}-worker-data name: {{ include "dolphinscheduler.fullname" . }}-worker-data
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-data app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-data
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 10 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
accessModes: accessModes:
{{- range .Values.worker.persistentVolumeClaim.dataPersistentVolume.accessModes }} {{- range .Values.worker.persistentVolumeClaim.dataPersistentVolume.accessModes }}
@ -313,8 +149,7 @@ spec:
name: {{ include "dolphinscheduler.fullname" . }}-worker-logs name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-logs app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 10 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
accessModes: accessModes:
{{- range .Values.worker.persistentVolumeClaim.logsPersistentVolume.accessModes }} {{- range .Values.worker.persistentVolumeClaim.logsPersistentVolume.accessModes }}

8
docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml

@ -20,8 +20,7 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-alert name: {{ include "dolphinscheduler.fullname" . }}-alert
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
ports: ports:
- port: 50052 - port: 50052
@ -29,7 +28,4 @@ spec:
protocol: TCP protocol: TCP
name: alert-port name: alert-port
selector: selector:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert {{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: alert

31
docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml

@ -20,16 +20,35 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-api name: {{ include "dolphinscheduler.fullname" . }}-api
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }} {{- if and (eq .Values.api.service.type "LoadBalancer") .Values.api.service.annotations }}
annotations:
{{- range $key, $value := .Values.api.service.annotations }}
{{ $key }}: {{ $value | quote }}
{{- end }}
{{- end }}
spec: spec:
type: {{ .Values.api.service.type }}
{{- if and (eq .Values.api.service.type "ClusterIP") .Values.api.service.clusterIP }}
clusterIP: {{ .Values.api.service.clusterIP }}
{{- end }}
ports: ports:
- port: 12345 - port: 12345
targetPort: api-port targetPort: api-port
{{- if and (eq .Values.api.service.type "NodePort") .Values.api.service.nodePort }}
nodePort: {{ .Values.api.service.nodePort }}
{{- end }}
protocol: TCP protocol: TCP
name: api-port name: api-port
{{- if .Values.api.service.externalIPs }}
externalIPs:
{{- toYaml .Values.api.service.externalIPs | nindent 4 }}
{{- end }}
{{- if and (eq .Values.api.service.type "ExternalName") .Values.api.service.externalName }}
externalName: {{ .Values.api.service.externalName }}
{{- end }}
{{- if and (eq .Values.api.service.type "LoadBalancer") .Values.api.service.loadBalancerIP }}
loadBalancerIP: {{ .Values.api.service.loadBalancerIP }}
{{- end }}
selector: selector:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api {{- include "dolphinscheduler.api.labels" . | nindent 4 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: api

10
docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml

@ -20,17 +20,13 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-master-headless name: {{ include "dolphinscheduler.fullname" . }}-master-headless
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
clusterIP: "None" clusterIP: "None"
ports: ports:
- port: {{ .Values.master.configmap.MASTER_LISTEN_PORT }} - port: 5678
targetPort: master-port targetPort: master-port
protocol: TCP protocol: TCP
name: master-port name: master-port
selector: selector:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master {{- include "dolphinscheduler.master.labels" . | nindent 4 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: master

10
docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml

@ -20,12 +20,11 @@ metadata:
name: {{ include "dolphinscheduler.fullname" . }}-worker-headless name: {{ include "dolphinscheduler.fullname" . }}-worker-headless
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless
app.kubernetes.io/instance: {{ .Release.Name }} {{- include "dolphinscheduler.common.labels" . | nindent 4 }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
clusterIP: "None" clusterIP: "None"
ports: ports:
- port: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }} - port: 1234
targetPort: worker-port targetPort: worker-port
protocol: TCP protocol: TCP
name: worker-port name: worker-port
@ -34,7 +33,4 @@ spec:
protocol: TCP protocol: TCP
name: logger-port name: logger-port
selector: selector:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker {{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: worker

99
docker/kubernetes/dolphinscheduler/values.yaml

@ -23,7 +23,7 @@ timezone: "Asia/Shanghai"
image: image:
repository: "apache/dolphinscheduler" repository: "apache/dolphinscheduler"
tag: "latest" tag: "1.3.6-SNAPSHOT"
pullPolicy: "IfNotPresent" pullPolicy: "IfNotPresent"
pullSecret: "" pullSecret: ""
@ -54,8 +54,6 @@ externalDatabase:
zookeeper: zookeeper:
enabled: true enabled: true
fourlwCommandsWhitelist: "srvr,ruok,wchs,cons" fourlwCommandsWhitelist: "srvr,ruok,wchs,cons"
service:
port: "2181"
persistence: persistence:
enabled: false enabled: false
size: "20Gi" size: "20Gi"
@ -69,26 +67,48 @@ externalZookeeper:
zookeeperRoot: "/dolphinscheduler" zookeeperRoot: "/dolphinscheduler"
common: common:
## ConfigMap ## Configmap
configmap: configmap:
DOLPHINSCHEDULER_ENV: DOLPHINSCHEDULER_OPTS: ""
- "export HADOOP_HOME=/opt/soft/hadoop" DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
- "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop"
- "export SPARK_HOME1=/opt/soft/spark1"
- "export SPARK_HOME2=/opt/soft/spark2"
#- "export PYTHON_HOME=/opt/soft/python"
- "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk"
- "export HIVE_HOME=/opt/soft/hive"
- "export FLINK_HOME=/opt/soft/flink"
- "export DATAX_HOME=/opt/soft/datax"
- "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
RESOURCE_STORAGE_TYPE: "HDFS" RESOURCE_STORAGE_TYPE: "HDFS"
RESOURCE_UPLOAD_PATH: "/dolphinscheduler" RESOURCE_UPLOAD_PATH: "/dolphinscheduler"
FS_DEFAULT_FS: "file:///" FS_DEFAULT_FS: "file:///"
FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com" FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com"
FS_S3A_ACCESS_KEY: "xxxxxxx" FS_S3A_ACCESS_KEY: "xxxxxxx"
FS_S3A_SECRET_KEY: "xxxxxxx" FS_S3A_SECRET_KEY: "xxxxxxx"
HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE: "false"
JAVA_SECURITY_KRB5_CONF_PATH: "/opt/krb5.conf"
LOGIN_USER_KEYTAB_USERNAME: "hdfs@HADOOP.COM"
LOGIN_USER_KEYTAB_PATH: "/opt/hdfs.keytab"
KERBEROS_EXPIRE_TIME: "2"
HDFS_ROOT_USER: "hdfs"
RESOURCE_MANAGER_HTTPADDRESS_PORT: "8088"
YARN_RESOURCEMANAGER_HA_RM_IDS: ""
YARN_APPLICATION_STATUS_ADDRESS: "http://ds1:%s/ws/v1/cluster/apps/%s"
YARN_JOB_HISTORY_STATUS_ADDRESS: "http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"
DATASOURCE_ENCRYPTION_ENABLE: "false"
DATASOURCE_ENCRYPTION_SALT: "!@#$%^&*"
SUDO_ENABLE: "true"
# dolphinscheduler env
HADOOP_HOME: "/opt/soft/hadoop"
HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop"
SPARK_HOME1: "/opt/soft/spark1"
SPARK_HOME2: "/opt/soft/spark2"
PYTHON_HOME: "/usr/bin/python"
JAVA_HOME: "/usr/local/openjdk-8"
HIVE_HOME: "/opt/soft/hive"
FLINK_HOME: "/opt/soft/flink"
DATAX_HOME: "/opt/soft/datax"
## Shared storage persistence mounted into api, master and worker, such as Hadoop, Spark, Flink and DataX binary package
sharedStoragePersistence:
enabled: false
mountPath: "/opt/soft"
accessModes:
- "ReadWriteMany"
## storageClassName must support the access mode: ReadWriteMany
storageClassName: "-"
storage: "20Gi"
## If RESOURCE_STORAGE_TYPE is HDFS and FS_DEFAULT_FS is file:///, fsFileResourcePersistence should be enabled for resource storage ## If RESOURCE_STORAGE_TYPE is HDFS and FS_DEFAULT_FS is file:///, fsFileResourcePersistence should be enabled for resource storage
fsFileResourcePersistence: fsFileResourcePersistence:
enabled: false enabled: false
@ -126,17 +146,18 @@ master:
# requests: # requests:
# memory: "2Gi" # memory: "2Gi"
# cpu: "500m" # cpu: "500m"
## ConfigMap ## Configmap
configmap: configmap:
DOLPHINSCHEDULER_OPTS: "" MASTER_SERVER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
MASTER_EXEC_THREADS: "100" MASTER_EXEC_THREADS: "100"
MASTER_EXEC_TASK_NUM: "20" MASTER_EXEC_TASK_NUM: "20"
MASTER_DISPATCH_TASK_NUM: "3"
MASTER_HOST_SELECTOR: "LowerWeight"
MASTER_HEARTBEAT_INTERVAL: "10" MASTER_HEARTBEAT_INTERVAL: "10"
MASTER_TASK_COMMIT_RETRYTIMES: "5" MASTER_TASK_COMMIT_RETRYTIMES: "5"
MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100" MASTER_MAX_CPULOAD_AVG: "-1"
MASTER_RESERVED_MEMORY: "0.1" MASTER_RESERVED_MEMORY: "0.3"
MASTER_LISTEN_PORT: "5678"
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe: livenessProbe:
@ -194,16 +215,16 @@ worker:
# requests: # requests:
# memory: "2Gi" # memory: "2Gi"
# cpu: "500m" # cpu: "500m"
## ConfigMap ## Configmap
configmap: configmap:
DOLPHINSCHEDULER_OPTS: "" LOGGER_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
WORKER_SERVER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
WORKER_EXEC_THREADS: "100" WORKER_EXEC_THREADS: "100"
WORKER_HEARTBEAT_INTERVAL: "10" WORKER_HEARTBEAT_INTERVAL: "10"
WORKER_MAX_CPULOAD_AVG: "100"
WORKER_RESERVED_MEMORY: "0.1"
WORKER_LISTEN_PORT: "1234"
WORKER_GROUPS: "default"
WORKER_HOST_WEIGHT: "100" WORKER_HOST_WEIGHT: "100"
WORKER_MAX_CPULOAD_AVG: "-1"
WORKER_RESERVED_MEMORY: "0.3"
WORKER_GROUPS: "default"
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe: livenessProbe:
@ -275,9 +296,9 @@ alert:
# requests: # requests:
# memory: "1Gi" # memory: "1Gi"
# cpu: "500m" # cpu: "500m"
## ConfigMap ## Configmap
configmap: configmap:
DOLPHINSCHEDULER_OPTS: "" ALERT_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
ALERT_PLUGIN_DIR: "lib/plugin/alert" ALERT_PLUGIN_DIR: "lib/plugin/alert"
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
@ -338,9 +359,9 @@ api:
# requests: # requests:
# memory: "1Gi" # memory: "1Gi"
# cpu: "500m" # cpu: "500m"
## ConfigMap ## Configmap
configmap: configmap:
DOLPHINSCHEDULER_OPTS: "" API_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m"
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe: livenessProbe:
@ -367,6 +388,22 @@ api:
- "ReadWriteOnce" - "ReadWriteOnce"
storageClassName: "-" storageClassName: "-"
storage: "20Gi" storage: "20Gi"
service:
## type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer
type: "ClusterIP"
## clusterIP is the IP address of the service and is usually assigned randomly by the master
clusterIP: ""
## nodePort is the port on each node on which this service is exposed when type=NodePort
nodePort: ""
## externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service
externalIPs: []
## externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service, requires Type to be ExternalName
externalName: ""
## loadBalancerIP when service.type is LoadBalancer. LoadBalancer will get created with the IP specified in this field
loadBalancerIP: ""
## annotations may need to be set when service.type is LoadBalancer
## service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:EXAMPLE_CERT
annotations: {}
ingress: ingress:
enabled: false enabled: false

2
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java

@ -31,7 +31,7 @@ public class Constants {
public static final String ALERT_PROPERTIES_PATH = "/alert.properties"; public static final String ALERT_PROPERTIES_PATH = "/alert.properties";
/** default alert plugin dir **/ /** default alert plugin dir **/
public static final String ALERT_PLUGIN_PATH = "./lib/plugin/alert"; public static final String ALERT_PLUGIN_PATH = "lib/plugin/alert";
public static final int ALERT_SCAN_INTERVAL = 5000; public static final int ALERT_SCAN_INTERVAL = 5000;

2
dolphinscheduler-alert/src/main/resources/alert.properties

@ -21,7 +21,7 @@
#eg : Alert Server Listener port #eg : Alert Server Listener port
#alert.plugin.dir config the Alert Plugin dir . AlertServer while find and load the Alert Plugin Jar from this dir when deploy and start AlertServer on the server . #alert.plugin.dir config the Alert Plugin dir . AlertServer while find and load the Alert Plugin Jar from this dir when deploy and start AlertServer on the server .
#alert.plugin.dir=./lib/plugin/alert #alert.plugin.dir=lib/plugin/alert
#maven.local.repository=/Users/gaojun/Documents/jianguoyun/localRepository #maven.local.repository=/Users/gaojun/Documents/jianguoyun/localRepository

2
dolphinscheduler-api/src/main/resources/application-api.properties

@ -41,7 +41,7 @@ server.jetty.max-http-form-post-size=5000000
# messages encoding # messages encoding
spring.messages.encoding=UTF-8 spring.messages.encoding=UTF-8
# i18n classpath folder , file prefix messages, if have many files, use "," seperator # i18n classpath folder, file prefix messages. if have many files, use "," seperator
spring.messages.basename=i18n/messages spring.messages.basename=i18n/messages
# Authentication types (supported types: PASSWORD) # Authentication types (supported types: PASSWORD)

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -93,11 +93,19 @@ public final class Constants {
public static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key"; public static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key";
/**
* hadoop configuration
*/
public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE";
public static final String HADOOP_RM_STATE_STANDBY = "STANDBY";
public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port";
/** /**
* yarn.resourcemanager.ha.rm.ids * yarn.resourcemanager.ha.rm.ids
*/ */
public static final String YARN_RESOURCEMANAGER_HA_RM_IDS = "yarn.resourcemanager.ha.rm.ids"; public static final String YARN_RESOURCEMANAGER_HA_RM_IDS = "yarn.resourcemanager.ha.rm.ids";
public static final String YARN_RESOURCEMANAGER_HA_XX = "xx";
/** /**
@ -153,7 +161,6 @@ public final class Constants {
* development.state * development.state
*/ */
public static final String DEVELOPMENT_STATE = "development.state"; public static final String DEVELOPMENT_STATE = "development.state";
public static final String DEVELOPMENT_STATE_DEFAULT_VALUE = "true";
/** /**
* sudo enable * sudo enable
@ -266,15 +273,6 @@ public final class Constants {
public static final String AT_SIGN = "@"; public static final String AT_SIGN = "@";
public static final String WORKER_MAX_CPULOAD_AVG = "worker.max.cpuload.avg";
public static final String WORKER_RESERVED_MEMORY = "worker.reserved.memory";
public static final String MASTER_MAX_CPULOAD_AVG = "master.max.cpuload.avg";
public static final String MASTER_RESERVED_MEMORY = "master.reserved.memory";
/** /**
* date format of yyyy-MM-dd HH:mm:ss * date format of yyyy-MM-dd HH:mm:ss
*/ */
@ -473,15 +471,6 @@ public final class Constants {
*/ */
public static final String CMDPARAM_COMPLEMENT_DATA_END_DATE = "complementEndDate"; public static final String CMDPARAM_COMPLEMENT_DATA_END_DATE = "complementEndDate";
/**
* hadoop configuration
*/
public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE";
public static final String HADOOP_RM_STATE_STANDBY = "STANDBY";
public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port";
/** /**
* data source config * data source config
@ -834,11 +823,6 @@ public final class Constants {
*/ */
public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username"; public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username";
/**
* default worker group id
*/
public static final int DEFAULT_WORKER_ID = -1;
/** /**
* loginUserFromKeytab path * loginUserFromKeytab path
*/ */

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java

@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory;
*/ */
public class NetUtils { public class NetUtils {
private static final Pattern STS_PATTERN = Pattern.compile("-\\d+$"); // StatefulSet pattern
private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3,5}$"); private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3,5}$");
private static final String NETWORK_PRIORITY_DEFAULT = "default"; private static final String NETWORK_PRIORITY_DEFAULT = "default";
private static final String NETWORK_PRIORITY_INNER = "inner"; private static final String NETWORK_PRIORITY_INNER = "inner";
@ -81,13 +80,8 @@ public class NetUtils {
if (inetAddress != null) { if (inetAddress != null) {
if (Constants.KUBERNETES_MODE) { if (Constants.KUBERNETES_MODE) {
String canonicalHost = inetAddress.getCanonicalHostName(); String canonicalHost = inetAddress.getCanonicalHostName();
if (!canonicalHost.contains(".") || IP_PATTERN.matcher(canonicalHost).matches()) { String[] items = canonicalHost.split("\\.");
String host = inetAddress.getHostName(); if (items.length == 6 && "svc".equals(items[3])) {
if (STS_PATTERN.matcher(host).find()) {
return String.format("%s.%s", host, host.replaceFirst("\\d+$", "headless"));
}
} else if (canonicalHost.contains(".")) {
String[] items = canonicalHost.split("\\.");
return String.format("%s.%s", items[0], items[1]); return String.format("%s.%s", items[0], items[1]);
} }
return canonicalHost; return canonicalHost;

56
dolphinscheduler-common/src/main/resources/common.properties

@ -15,16 +15,16 @@
# limitations under the License. # limitations under the License.
# #
# resource storage type : HDFS, S3, NONE # user data local directory path, please make sure the directory exists and have read write permissions
data.basedir.path=/tmp/dolphinscheduler
# resource storage type: HDFS, S3, NONE
resource.storage.type=NONE resource.storage.type=NONE
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.upload.path=/dolphinscheduler resource.upload.path=/dolphinscheduler
# user data local directory path, please make sure the directory exists and have read write permissions # whether to startup kerberos
#data.basedir.path=/tmp/dolphinscheduler
# whether kerberos starts
hadoop.security.authentication.startup.state=false hadoop.security.authentication.startup.state=false
# java.security.krb5.conf path # java.security.krb5.conf path
@ -36,13 +36,16 @@ login.user.keytab.username=hdfs-mycluster@ESZ.COM
# login user from keytab path # login user from keytab path
login.user.keytab.path=/opt/hdfs.headless.keytab login.user.keytab.path=/opt/hdfs.headless.keytab
#resource.view.suffixs # kerberos expire time, the unit is hour
kerberos.expire.time=2
# resource view suffixs
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
# if resource.storage.type=HDFS, the user need to have permission to create directories under the HDFS root path # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
hdfs.root.user=hdfs hdfs.root.user=hdfs
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
fs.defaultFS=hdfs://mycluster:8020 fs.defaultFS=hdfs://mycluster:8020
# if resource.storage.type=S3, s3 endpoint # if resource.storage.type=S3, s3 endpoint
@ -54,31 +57,32 @@ fs.s3a.access.key=A3DXS30FO22544RE
# if resource.storage.type=S3, s3 secret key # if resource.storage.type=S3, s3 secret key
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty # resourcemanager port, the default value is 8088 if not specified
resource.manager.httpaddress.port=8088
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname. # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
yarn.application.status.address=http://10.172.33.1:%s/ws/v1/cluster/apps/%s yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
# if custom you resourcemanager port ,you need to replace 8088 else default value. # job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
resource.manager.httpaddress.port=8088
# job history status url when application number threshold is reached(default 10000,maybe it was set to 1000)
yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
# system env path, If you want to set your own path, you need to set this env file to an absolute path # datasource encryption enable
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh datasource.encryption.enable=false
development.state=false
# kerberos tgt expire time, unit is hours
kerberos.expire.time=2
# datasource encryption salt # datasource encryption salt
datasource.encryption.enable=false
datasource.encryption.salt=!@#$%^&* datasource.encryption.salt=!@#$%^&*
# Network IP gets priority, default inner outer # use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
sudo.enable=true
# network IP gets priority, default: inner outer
#dolphin.scheduler.network.priority.strategy=default #dolphin.scheduler.network.priority.strategy=default
# use sudo or not, if set true ,executing user is tenant user and deploy user need sudo permissions ; if set false, executing user is the deploy user, don't need sudo permissions. # system env path
sudo.enable=true #dolphinscheduler.env.path=env/dolphinscheduler_env.sh
# development state
development.state=false

15
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java

@ -51,11 +51,24 @@ public class NetUtilsTest {
assertEquals("172.17.0.15", NetUtils.getHost(address)); assertEquals("172.17.0.15", NetUtils.getHost(address));
CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true); CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true);
assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address)); assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address));
address = mock(InetAddress.class);
when(address.getCanonicalHostName()).thenReturn("busybox-1.default-subdomain.my-namespace.svc.cluster-domain.example");
when(address.getHostName()).thenReturn("busybox-1");
CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true);
assertEquals("busybox-1.default-subdomain", NetUtils.getHost(address));
address = mock(InetAddress.class);
when(address.getCanonicalHostName()).thenReturn("dolphinscheduler.cluster-domain.example");
when(address.getHostName()).thenReturn("dolphinscheduler");
CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true);
assertEquals("dolphinscheduler.cluster-domain.example", NetUtils.getHost(address));
address = mock(InetAddress.class); address = mock(InetAddress.class);
when(address.getCanonicalHostName()).thenReturn("dolphinscheduler-worker-0"); when(address.getCanonicalHostName()).thenReturn("dolphinscheduler-worker-0");
when(address.getHostName()).thenReturn("dolphinscheduler-worker-0"); when(address.getHostName()).thenReturn("dolphinscheduler-worker-0");
CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true); CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true);
assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address)); assertEquals("dolphinscheduler-worker-0", NetUtils.getHost(address));
} }
@Test @Test

8
dolphinscheduler-dao/src/main/resources/datasource.properties

@ -18,14 +18,14 @@
# postgresql # postgresql
spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.driver-class-name=org.postgresql.Driver
spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
spring.datasource.username=test spring.datasource.username=root
spring.datasource.password=test spring.datasource.password=root
# mysql # mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 #spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
#spring.datasource.username=xxxx #spring.datasource.username=ds_user
#spring.datasource.password=xxxx #spring.datasource.password=dolphinscheduler
# connection configuration # connection configuration
#spring.datasource.initialSize=5 #spring.datasource.initialSize=5

22
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java

@ -14,9 +14,11 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.server.master.config; package org.apache.dolphinscheduler.server.master.config;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@ -25,21 +27,27 @@ import org.springframework.stereotype.Component;
@PropertySource(value = "master.properties") @PropertySource(value = "master.properties")
public class MasterConfig { public class MasterConfig {
@Value("${master.listen.port:5678}")
private int listenPort;
@Value("${master.exec.threads:100}") @Value("${master.exec.threads:100}")
private int masterExecThreads; private int masterExecThreads;
@Value("${master.exec.task.num:20}") @Value("${master.exec.task.num:20}")
private int masterExecTaskNum; private int masterExecTaskNum;
@Value("${master.dispatch.task.num:3}")
private int masterDispatchTaskNumber;
@Value("${master.host.selector:LowerWeight}")
private String hostSelector;
@Value("${master.heartbeat.interval:10}") @Value("${master.heartbeat.interval:10}")
private int masterHeartbeatInterval; private int masterHeartbeatInterval;
@Value("${master.task.commit.retryTimes:5}") @Value("${master.task.commit.retryTimes:5}")
private int masterTaskCommitRetryTimes; private int masterTaskCommitRetryTimes;
@Value("${master.dispatch.task.num:3}")
private int masterDispatchTaskNumber;
@Value("${master.task.commit.interval:1000}") @Value("${master.task.commit.interval:1000}")
private int masterTaskCommitInterval; private int masterTaskCommitInterval;
@ -49,12 +57,6 @@ public class MasterConfig {
@Value("${master.reserved.memory:0.3}") @Value("${master.reserved.memory:0.3}")
private double masterReservedMemory; private double masterReservedMemory;
@Value("${master.host.selector:lowerWeight}")
private String hostSelector;
@Value("${master.listen.port:5678}")
private int listenPort;
public int getListenPort() { public int getListenPort() {
return listenPort; return listenPort;
} }
@ -112,7 +114,7 @@ public class MasterConfig {
} }
public double getMasterMaxCpuloadAvg() { public double getMasterMaxCpuloadAvg() {
if (masterMaxCpuloadAvg == -1){ if (masterMaxCpuloadAvg == -1) {
return Constants.DEFAULT_MASTER_CPU_LOAD; return Constants.DEFAULT_MASTER_CPU_LOAD;
} }
return masterMaxCpuloadAvg; return masterMaxCpuloadAvg;

6
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java

@ -29,6 +29,9 @@ import org.springframework.stereotype.Component;
@PropertySource(value = "worker.properties") @PropertySource(value = "worker.properties")
public class WorkerConfig { public class WorkerConfig {
@Value("${worker.listen.port:1234}")
private int listenPort;
@Value("${worker.exec.threads:100}") @Value("${worker.exec.threads:100}")
private int workerExecThreads; private int workerExecThreads;
@ -47,9 +50,6 @@ public class WorkerConfig {
@Value("${worker.reserved.memory:0.3}") @Value("${worker.reserved.memory:0.3}")
private double workerReservedMemory; private double workerReservedMemory;
@Value("${worker.listen.port:1234}")
private int listenPort;
@Value("#{'${worker.groups:default}'.split(',')}") @Value("#{'${worker.groups:default}'.split(',')}")
private Set<String> workerGroups; private Set<String> workerGroups;

64
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java

@ -14,23 +14,29 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.server.worker.task; package org.apache.dolphinscheduler.server.worker.task;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*; import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.Collections; import java.util.Collections;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* python command executor * python command executor
@ -46,7 +52,7 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
* python * python
*/ */
public static final String PYTHON = "python"; public static final String PYTHON = "python";
private static final Pattern PYTHON_PATH_PATTERN = Pattern.compile("/bin/python[\\d.]*$");
/** /**
* constructor * constructor
@ -115,54 +121,64 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
@Override @Override
protected String commandInterpreter() { protected String commandInterpreter() {
String pythonHome = getPythonHome(taskExecutionContext.getEnvFile()); String pythonHome = getPythonHome(taskExecutionContext.getEnvFile());
if (StringUtils.isEmpty(pythonHome)){ return getPythonCommand(pythonHome);
}
/**
* get python command
*
* @param pythonHome python home
* @return python command
*/
public static String getPythonCommand(String pythonHome) {
if (StringUtils.isEmpty(pythonHome)) {
return PYTHON; return PYTHON;
} }
File file = new File(pythonHome);
if (file.exists() && file.isFile()) {
return pythonHome;
}
if (PYTHON_PATH_PATTERN.matcher(pythonHome).find()) {
return pythonHome;
}
return pythonHome + "/bin/python"; return pythonHome + "/bin/python";
} }
/** /**
* get the absolute path of the Python are installed * get python home
* note :
*
* for example :
* your PYTHON_HOM is /opt/python3.7
* *
* @param envPath env path * @param envPath env path
* @return python home * @return python home
*/ */
private static String getPythonHome(String envPath){ public static String getPythonHome(String envPath) {
BufferedReader br = null; BufferedReader br = null;
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
try { try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(envPath))); br = new BufferedReader(new InputStreamReader(new FileInputStream(envPath)));
String line; String line;
while ((line = br.readLine()) != null){ while ((line = br.readLine()) != null) {
if (line.contains(Constants.PYTHON_HOME)){ if (line.contains(Constants.PYTHON_HOME)) {
sb.append(line); sb.append(line);
break; break;
} }
} }
String result = sb.toString(); String result = sb.toString();
if (org.apache.commons.lang.StringUtils.isEmpty(result)){ if (StringUtils.isEmpty(result)) {
return null; return null;
} }
String[] arrs = result.split(Constants.EQUAL_SIGN); String[] arrs = result.split(Constants.EQUAL_SIGN);
if (arrs.length == 2){ if (arrs.length == 2) {
return arrs[1]; return arrs[1];
} }
} catch (IOException e) {
}catch (IOException e){ logger.error("read file failure", e);
logger.error("read file failure",e); } finally {
}finally {
try { try {
if (br != null){ if (br != null) {
br.close(); br.close();
} }
} catch (IOException e) { } catch (IOException e) {
logger.error(e.getMessage(),e); logger.error(e.getMessage(), e);
} }
} }
return null; return null;

11
dolphinscheduler-server/src/main/resources/config/install_config.conf

@ -53,7 +53,7 @@ deployUser="dolphinscheduler"
# Note: find and load the Alert Plugin Jar from this dir. # Note: find and load the Alert Plugin Jar from this dir.
alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert" alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert"
# resource storage type:HDFS,S3,NONE # resource storage type: HDFS, S3, NONE
resourceStorageType="NONE" resourceStorageType="NONE"
# if resourceStorageType is HDFS,defaultFS write namenode address,HA you need to put core-site.xml and hdfs-site.xml in the conf directory. # if resourceStorageType is HDFS,defaultFS write namenode address,HA you need to put core-site.xml and hdfs-site.xml in the conf directory.
@ -66,13 +66,16 @@ s3Endpoint="http://192.168.xx.xx:9010"
s3AccessKey="xxxxxxxxxx" s3AccessKey="xxxxxxxxxx"
s3SecretKey="xxxxxxxxxx" s3SecretKey="xxxxxxxxxx"
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty # resourcemanager port, the default value is 8088 if not specified
resourceManagerHttpAddressPort=8088
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
yarnHaIps="192.168.xx.xx,192.168.xx.xx" yarnHaIps="192.168.xx.xx,192.168.xx.xx"
# if resourcemanager HA enable or not use resourcemanager, please skip this value setting; If resourcemanager is single, you only need to replace yarnIp1 to actual resourcemanager hostname. # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
singleYarnIp="yarnIp1" singleYarnIp="yarnIp1"
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。/dolphinscheduler is recommended # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resourceUploadPath="/dolphinscheduler" resourceUploadPath="/dolphinscheduler"
# who have permissions to create directory under HDFS/S3 root path # who have permissions to create directory under HDFS/S3 root path

23
dolphinscheduler-server/src/main/resources/master.properties

@ -15,29 +15,32 @@
# limitations under the License. # limitations under the License.
# #
# master execute thread number # master listen port
#master.listen.port=5678
# master execute thread number to limit process instances in parallel
#master.exec.threads=100 #master.exec.threads=100
# master execute task number in parallel # master execute task number in parallel per process instance
#master.exec.task.num=20 #master.exec.task.num=20
# master dispatch task number # master dispatch task number per batch
#master.dispatch.task.num=3 #master.dispatch.task.num=3
# master heartbeat interval # master host selector to select a suitable worker, default value: LowerWeight. Optional values include Random, RoundRobin, LowerWeight
#master.host.selector=LowerWeight
# master heartbeat interval, the unit is second
#master.heartbeat.interval=10 #master.heartbeat.interval=10
# master commit task retry times # master commit task retry times
#master.task.commit.retryTimes=5 #master.task.commit.retryTimes=5
# master commit task interval # master commit task interval, the unit is millisecond
#master.task.commit.interval=1000 #master.task.commit.interval=1000
# only less than cpu avg load, master server can work. default value -1 : the number of cpu cores * 2 # master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
#master.max.cpuload.avg=-1 #master.max.cpuload.avg=-1
# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. # master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
#master.reserved.memory=0.3 #master.reserved.memory=0.3
# master listen port
#master.listen.port=5678

20
dolphinscheduler-server/src/main/resources/worker.properties

@ -15,10 +15,13 @@
# limitations under the License. # limitations under the License.
# #
# worker execute thread number # worker listener port
#worker.listen.port=1234
# worker execute thread number to limit task instances in parallel
#worker.exec.threads=100 #worker.exec.threads=100
# worker heartbeat interval # worker heartbeat interval, the unit is second
#worker.heartbeat.interval=10 #worker.heartbeat.interval=10
# worker host weight to dispatch tasks, default value 100 # worker host weight to dispatch tasks, default value 100
@ -27,17 +30,14 @@
# worker tenant auto create # worker tenant auto create
#worker.tenant.auto.create=false #worker.tenant.auto.create=false
# only less than cpu avg load, worker server can work. default value -1: the number of cpu cores * 2 # worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2
#worker.max.cpuload.avg=-1 #worker.max.cpuload.avg=-1
# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G. # worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G
#worker.reserved.memory=0.3 #worker.reserved.memory=0.3
# worker listener port # default worker groups separated by comma, like 'worker.groups=default,test'
#worker.listen.port=1234
# default worker groups, if this worker belongs different groups, you can config the following like that 'worker.groups=default,test'
#worker.groups=default #worker.groups=default
# alert server listener host # alert server listen host
alert.listen.host=localhost #alert.listen.host=localhost

82
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java

@ -1,82 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.worker;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class EnvFileTest {
private static final Logger logger = LoggerFactory.getLogger(EnvFileTest.class);
@Test
public void test() {
String path = System.getProperty("user.dir")+"/script/env/dolphinscheduler_env.sh";
String pythonHome = getPythonHome(path);
logger.info(pythonHome);
}
/**
* get python home
* @param path
* @return
*/
private static String getPythonHome(String path){
BufferedReader br = null;
String line = null;
StringBuilder sb = new StringBuilder();
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(path)));
while ((line = br.readLine()) != null){
if (line.contains(Constants.PYTHON_HOME)) {
sb.append(line);
break;
}
}
String result = sb.toString();
if (StringUtils.isEmpty(result)){
return null;
}
String[] arrs = result.split("=");
if (arrs.length == 2){
return arrs[1];
}
}catch (IOException e){
logger.error("read file failed",e);
}finally {
try {
if (br != null){
br.close();
}
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
}
return null;
}
}

58
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.worker.task;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PythonCommandExecutorTest {
private static final Logger logger = LoggerFactory.getLogger(PythonCommandExecutorTest.class);
@Test
public void testGetPythonHome() {
String path = System.getProperty("user.dir") + "/script/env/dolphinscheduler_env.sh";
if (path.contains("dolphinscheduler-server/")) {
path = path.replace("dolphinscheduler-server/", "");
}
String pythonHome = PythonCommandExecutor.getPythonHome(path);
logger.info(pythonHome);
Assert.assertNotNull(pythonHome);
}
@Test
public void testGetPythonCommand() {
String pythonCommand = PythonCommandExecutor.getPythonCommand(null);
Assert.assertEquals(PythonCommandExecutor.PYTHON, pythonCommand);
pythonCommand = PythonCommandExecutor.getPythonCommand("");
Assert.assertEquals(PythonCommandExecutor.PYTHON, pythonCommand);
pythonCommand = PythonCommandExecutor.getPythonCommand("/usr/bin/python");
Assert.assertEquals("/usr/bin/python", pythonCommand);
pythonCommand = PythonCommandExecutor.getPythonCommand("/usr/local/bin/python2");
Assert.assertEquals("/usr/local/bin/python2", pythonCommand);
pythonCommand = PythonCommandExecutor.getPythonCommand("/opt/python/bin/python3.8");
Assert.assertEquals("/opt/python/bin/python3.8", pythonCommand);
pythonCommand = PythonCommandExecutor.getPythonCommand("/opt/soft/python");
Assert.assertEquals("/opt/soft/python/bin/python", pythonCommand);
pythonCommand = PythonCommandExecutor.getPythonCommand("/opt/soft/python-3.8");
Assert.assertEquals("/opt/soft/python-3.8/bin/python", pythonCommand);
}
}

8
install.sh

@ -43,8 +43,10 @@ sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.propert
sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties
sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties
sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties
sed -i ${txt} "s#resource.manager.httpaddress.port.*#resource.manager.httpaddress.port=${resourceManagerHttpAddressPort}#g" conf/common.properties
sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties
sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:%s/ws/v1/cluster/apps/%s#g" conf/common.properties
sed -i ${txt} "s#yarn.job.history.status.address.*#yarn.job.history.status.address=http://${singleYarnIp}:19888/ws/v1/history/mapreduce/jobs/%s#g" conf/common.properties
sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties
sed -i ${txt} "s#resource.upload.path.*#resource.upload.path=${resourceUploadPath}#g" conf/common.properties sed -i ${txt} "s#resource.upload.path.*#resource.upload.path=${resourceUploadPath}#g" conf/common.properties
sed -i ${txt} "s#resource.storage.type.*#resource.storage.type=${resourceStorageType}#g" conf/common.properties sed -i ${txt} "s#resource.storage.type.*#resource.storage.type=${resourceStorageType}#g" conf/common.properties
@ -56,9 +58,9 @@ sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeep
sed -i ${txt} "s#\#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties sed -i ${txt} "s#\#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties
sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties
sed -i ${txt} "s#alert.plugin.dir.*#alert.plugin.dir=${alertPluginDir}#g" conf/alert.properties sed -i ${txt} "s#\#alert.plugin.dir.*#alert.plugin.dir=${alertPluginDir}#g" conf/alert.properties
sed -i ${txt} "s#alert.listen.host.*#alert.listen.host=${alertServer}#g" conf/worker.properties sed -i ${txt} "s#\#alert.listen.host.*#alert.listen.host=${alertServer}#g" conf/worker.properties
# 2.create directory # 2.create directory
echo "2.create directory" echo "2.create directory"

3
pom.xml

@ -970,7 +970,6 @@
<include>**/server/worker/shell/ShellCommandExecutorTest.java</include> <include>**/server/worker/shell/ShellCommandExecutorTest.java</include>
<include>**/server/worker/sql/SqlExecutorTest.java</include> <include>**/server/worker/sql/SqlExecutorTest.java</include>
<include>**/server/worker/task/spark/SparkTaskTest.java</include> <include>**/server/worker/task/spark/SparkTaskTest.java</include>
<include>**/server/worker/task/EnvFileTest.java</include>
<include>**/server/worker/task/spark/SparkTaskTest.java</include> <include>**/server/worker/task/spark/SparkTaskTest.java</include>
<include>**/server/worker/task/datax/DataxTaskTest.java</include> <include>**/server/worker/task/datax/DataxTaskTest.java</include>
<!--<include>**/server/worker/task/http/HttpTaskTest.java</include>--> <!--<include>**/server/worker/task/http/HttpTaskTest.java</include>-->
@ -979,9 +978,9 @@
<include>**/server/worker/task/shell/ShellTaskTest.java</include> <include>**/server/worker/task/shell/ShellTaskTest.java</include>
<include>**/server/worker/task/TaskManagerTest.java</include> <include>**/server/worker/task/TaskManagerTest.java</include>
<include>**/server/worker/task/AbstractCommandExecutorTest.java</include> <include>**/server/worker/task/AbstractCommandExecutorTest.java</include>
<include>**/server/worker/task/PythonCommandExecutorTest.java</include>
<include>**/server/worker/task/ShellTaskReturnTest.java</include> <include>**/server/worker/task/ShellTaskReturnTest.java</include>
<include>**/server/worker/task/sql/SqlTaskTest.java</include> <include>**/server/worker/task/sql/SqlTaskTest.java</include>
<include>**/server/worker/EnvFileTest.java</include>
<include>**/server/worker/runner/TaskExecuteThreadTest.java</include> <include>**/server/worker/runner/TaskExecuteThreadTest.java</include>
<include>**/server/worker/runner/WorkerManagerThreadTest.java</include> <include>**/server/worker/runner/WorkerManagerThreadTest.java</include>
<include>**/service/quartz/cron/CronUtilsTest.java</include> <include>**/service/quartz/cron/CronUtilsTest.java</include>

2
script/create-dolphinscheduler.sh

@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export STOP_TIMEOUT=5 export STOP_TIMEOUT=5
CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler

40
script/dolphinscheduler-daemon.sh

@ -29,10 +29,11 @@ shift
command=$1 command=$1
shift shift
echo "Begin $startStop $command......"
BIN_DIR=`dirname $0` BIN_DIR=`dirname $0`
BIN_DIR=`cd "$BIN_DIR"; pwd` BIN_DIR=`cd "$BIN_DIR"; pwd`
DOLPHINSCHEDULER_HOME=$BIN_DIR/.. DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
source /etc/profile source /etc/profile
@ -56,51 +57,47 @@ pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid
cd $DOLPHINSCHEDULER_HOME cd $DOLPHINSCHEDULER_HOME
export DOLPHINSCHEDULER_OPTS="-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm $DOLPHINSCHEDULER_OPTS"
if [ "$command" = "api-server" ]; then if [ "$command" = "api-server" ]; then
HEAP_INITIAL_SIZE=1g
HEAP_MAX_SIZE=1g
HEAP_NEW_GENERATION_SIZE=512m
LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api" LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api"
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $API_SERVER_OPTS"
elif [ "$command" = "master-server" ]; then elif [ "$command" = "master-server" ]; then
HEAP_INITIAL_SIZE=4g
HEAP_MAX_SIZE=4g
HEAP_NEW_GENERATION_SIZE=2g
LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false" LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.master.MasterServer CLASS=org.apache.dolphinscheduler.server.master.MasterServer
HEAP_OPTS="-Xms4g -Xmx4g -Xmn2g"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $MASTER_SERVER_OPTS"
elif [ "$command" = "worker-server" ]; then elif [ "$command" = "worker-server" ]; then
HEAP_INITIAL_SIZE=2g
HEAP_MAX_SIZE=2g
HEAP_NEW_GENERATION_SIZE=1g
LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false" LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
HEAP_OPTS="-Xms2g -Xmx2g -Xmn1g"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $WORKER_SERVER_OPTS"
elif [ "$command" = "alert-server" ]; then elif [ "$command" = "alert-server" ]; then
HEAP_INITIAL_SIZE=1g
HEAP_MAX_SIZE=1g
HEAP_NEW_GENERATION_SIZE=512m
LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml" LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml"
CLASS=org.apache.dolphinscheduler.alert.AlertServer CLASS=org.apache.dolphinscheduler.alert.AlertServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $ALERT_SERVER_OPTS"
elif [ "$command" = "logger-server" ]; then elif [ "$command" = "logger-server" ]; then
HEAP_INITIAL_SIZE=1g
HEAP_MAX_SIZE=1g
HEAP_NEW_GENERATION_SIZE=512m
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $LOGGER_SERVER_OPTS"
elif [ "$command" = "zookeeper-server" ]; then elif [ "$command" = "zookeeper-server" ]; then
#note: this command just for getting a quick experience,not recommended for production. this operation will start a standalone zookeeper server #note: this command just for getting a quick experience,not recommended for production. this operation will start a standalone zookeeper server
LOG_FILE="-Dlogback.configurationFile=classpath:logback-zookeeper.xml" LOG_FILE="-Dlogback.configurationFile=classpath:logback-zookeeper.xml"
CLASS=org.apache.dolphinscheduler.service.zk.ZKServer CLASS=org.apache.dolphinscheduler.service.zk.ZKServer
else else
echo "Error: No command named \`$command' was found." echo "Error: No command named '$command' was found."
exit 1 exit 1
fi fi
export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION_SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS"
case $startStop in case $startStop in
(start) (start)
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
if [ "$DOCKER" = "true" ]; then if [ "$DOCKER" = "true" ]; then
echo "start in docker" echo start $command in docker
export DOLPHINSCHEDULER_OPTS="$DOLPHINSCHEDULER_OPTS -XX:-UseContainerSupport"
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
$JAVA_HOME/bin/java $exec_command $JAVA_HOME/bin/java $exec_command
else else
[ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR" [ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR"
@ -113,6 +110,7 @@ case $startStop in
fi fi
echo starting $command, logging to $log echo starting $command, logging to $log
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &"
nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &
echo $! > $pid echo $! > $pid

2
script/upgrade-dolphinscheduler.sh

@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export STOP_TIMEOUT=5 export STOP_TIMEOUT=5
CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler

Loading…
Cancel
Save