From 1e0ff5fb900f5919cf731067d882874433405282 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Wed, 3 Feb 2021 11:21:39 +0800 Subject: [PATCH] [Improvement][Docker] update outdated readme and fix PSQLException (#4652) --- docker/build/Dockerfile | 12 ++-- docker/build/README.md | 45 ++++++-------- docker/build/README_zh_CN.md | 45 ++++++-------- .../dolphinscheduler/common.properties.tpl | 4 +- .../logback/logback-alert.xml | 2 +- docker/build/startup-init-conf.sh | 4 +- docker/build/startup.sh | 28 +++++---- docker/docker-swarm/docker-compose.yml | 59 ++++++++----------- docker/docker-swarm/docker-stack.yml | 29 +++++---- 9 files changed, 100 insertions(+), 128 deletions(-) diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 8a837bf8ae..cb15763678 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -39,12 +39,12 @@ RUN ln -s /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bi ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler # 3. add configuration and modify permissions and set soft links -ADD ./checkpoint.sh /root/checkpoint.sh -ADD ./startup-init-conf.sh /root/startup-init-conf.sh -ADD ./startup.sh /root/startup.sh -ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ -ADD ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ -ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ +COPY ./checkpoint.sh /root/checkpoint.sh +COPY ./startup-init-conf.sh /root/startup-init-conf.sh +COPY ./startup.sh /root/startup.sh +COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ +COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ +COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ RUN dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ diff --git a/docker/build/README.md b/docker/build/README.md index 8345a676e8..dc0d5126d7 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -13,39 +13,31 @@ Official Website: https://dolphinscheduler.apache.org ## How to use this docker image -#### You can start a dolphinscheduler instance +#### You can start a dolphinscheduler by docker-compose (recommended) ``` -$ docker run -dit --name dolphinscheduler \ --e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ --p 12345:12345 \ -dolphinscheduler all +$ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d ``` -The default postgres user `root`, postgres password `root` and database `dolphinscheduler` are created in the `startup.sh`. +The default **postgres** user `root`, postgres password `root` and database `dolphinscheduler` are created in the `docker-compose.yml`. + +The default **zookeeper** is created in the `docker-compose.yml`. -The default zookeeper is created in the `startup.sh`. +Access the Web UI：http://192.168.xx.xx:12345 #### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`** -You can specify **existing postgres service**. Example: +You can specify **existing postgres and zookeeper service**. Example: ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -d --name dolphinscheduler \ +-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler all ``` -You can specify **existing zookeeper service**. Example: - -``` -$ docker run -dit --name dolphinscheduler \ --e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ --p 12345:12345 \ -dolphinscheduler all -``` +Access the Web UI：http://192.168.xx.xx:12345 #### Or start a standalone dolphinscheduler server @@ -54,8 +46,8 @@ You can start a standalone dolphinscheduler server. * Start a **master server**, For example: ``` -$ docker run -dit --name dolphinscheduler \ --e ZOOKEEPER_QUORUM="l92.168.x.x:2181" +$ docker run -d --name dolphinscheduler-master \ +-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler master-server @@ -64,17 +56,16 @@ dolphinscheduler master-server * Start a **worker server**, For example: ``` -$ docker run -dit --name dolphinscheduler \ --e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ +$ docker run -d --name dolphinscheduler-worker \ +-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" \ dolphinscheduler worker-server ``` * Start a **api server**, For example: ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -d --name dolphinscheduler-api \ +-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 12345:12345 \ @@ -84,7 +75,7 @@ dolphinscheduler api-server * Start a **alert server**, For example: ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -d --name dolphinscheduler-alert \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler alert-server @@ -106,7 +97,7 @@ $ sh ./docker/build/hooks/build In Windows, Example: ```bat -c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat +C:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat ``` Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script files if you don't understand diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 27b7465022..10a13061f4 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -13,39 +13,31 @@ Official Website: https://dolphinscheduler.apache.org ## 如何使用docker镜像 -#### 你可以运行一个dolphinscheduler实例 +#### **以 docker-compose 的方式启动dolphinscheduler(推荐)** ``` -$ docker run -dit --name dolphinscheduler \ --e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ --p 12345:12345 \ -dolphinscheduler all +$ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d ``` -在`startup.sh`脚本中，默认的创建`Postgres`的用户、密码和数据库，默认值分别为：`root`、`root`、`dolphinscheduler`。 +在`docker-compose.yml`文件中，默认的创建`Postgres`的用户、密码和数据库，默认值分别为：`root`、`root`、`dolphinscheduler`。 + +同时，默认的`Zookeeper`也会在`docker-compose.yml`文件中被创建。 -同时，默认的`Zookeeper`也会在`startup.sh`脚本中被创建。 +访问前端界面：http://192.168.xx.xx:12345 #### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务 -你可以指定一个已经存在的 **`Postgres`** 服务. 如下: +你可以指定已经存在的 **`Postgres`** 和 **`Zookeeper`** 服务. 如下: ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -d --name dolphinscheduler \ +-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler all ``` -你也可以指定一个已经存在的 **Zookeeper** 服务. 如下: - -``` -$ docker run -dit --name dolphinscheduler \ --e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ --p 12345:12345 \ -dolphinscheduler all -``` +访问前端界面：http://192.168.xx.xx:12345 #### 或者运行dolphinscheduler中的部分服务 @@ -54,8 +46,8 @@ dolphinscheduler all * 启动一个 **master server**, 如下: ``` -$ docker run -dit --name dolphinscheduler \ --e ZOOKEEPER_QUORUM="l92.168.x.x:2181" +$ docker run -d --name dolphinscheduler-master \ +-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler master-server @@ -64,17 +56,16 @@ dolphinscheduler master-server * 启动一个 **worker server**, 如下: ``` -$ docker run -dit --name dolphinscheduler \ --e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ +$ docker run -d --name dolphinscheduler-worker \ +-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" \ dolphinscheduler worker-server ``` * 启动一个 **api server**, 如下: ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -d --name dolphinscheduler-api \ +-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 12345:12345 \ @@ -84,7 +75,7 @@ dolphinscheduler api-server * 启动一个 **alert server**, 如下: ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -d --name dolphinscheduler-alert \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler alert-server @@ -106,7 +97,7 @@ $ sh ./docker/build/hooks/build Windows系统, 如下: ```bat -c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat +C:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat ``` 如果你不理解这些脚本 `./docker/build/hooks/build` `./docker/build/hooks/build.bat`，请阅读里面的内容。 diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl index 0533c7e710..ff74598fd4 100644 --- a/docker/build/conf/dolphinscheduler/common.properties.tpl +++ b/docker/build/conf/dolphinscheduler/common.properties.tpl @@ -25,7 +25,7 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} # resource upload startup type : HDFS,S3,NONE -resource.storage.type=${DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE} +resource.storage.type=${RESOURCE_STORAGE_TYPE} #============================================================================ # HDFS @@ -58,7 +58,7 @@ kerberos.expire.time=7 # S3 #============================================================================ # if resource.storage.type=S3，the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir -fs.defaultFS=${DOLPHINSCHEDULER_FS_DEFAULTFS} +fs.defaultFS=${FS_DEFAULT_FS} # if resource.storage.type=S3，s3 endpoint fs.s3a.endpoint=${FS_S3A_ENDPOINT} diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml index 25864fde20..eec78385db 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml @@ -37,7 +37,7 @@ - + \ No newline at end of file diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 5b1fe06d3b..200f17d2d5 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -38,9 +38,9 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} #============================================================================ export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} -export DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE=${DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE:-"HDFS"} +export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"} export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"} -export DOLPHINSCHEDULER_FS_DEFAULTFS=${DOLPHINSCHEDULER_FS_DEFAULTFS:-"file:///data/dolphinscheduler"} +export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"} export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"} export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"} export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"} diff --git a/docker/build/startup.sh b/docker/build/startup.sh index 6d0689f88e..eb6a96a6a8 100755 --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -22,8 +22,8 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs -# start database -initDatabase() { +# wait database +waitDatabase() { echo "test ${DATABASE_TYPE} service" while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) @@ -49,13 +49,16 @@ initDatabase() { exit 1 fi fi +} +# init database +initDatabase() { echo "import sql data" ${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh } -# start zk -initZK() { +# wait zk +waitZK() { echo "connect remote zookeeper" echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do while ! nc -z ${line%:*} ${line#*:}; do @@ -122,7 +125,8 @@ source /root/startup-init-conf.sh case "$1" in (all) - initZK + waitZK + waitDatabase initDatabase initMasterServer initWorkerServer @@ -132,26 +136,26 @@ case "$1" in LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log ;; (master-server) - initZK - initDatabase + waitZK + waitDatabase initMasterServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log ;; (worker-server) - initZK - initDatabase + waitZK initWorkerServer initLoggerServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log ;; (api-server) - initZK + waitZK + waitDatabase initDatabase initApiServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log ;; (alert-server) - initDatabase + waitDatabase initAlertServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log ;; @@ -166,7 +170,7 @@ case "$1" in esac # init directories and log files -mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null >> ${LOGFILE} +mkdir -p ${DOLPHINSCHEDULER_LOGS} && cat /dev/null >> ${LOGFILE} echo "tail begin" exec bash -c "tail -n 1 -f ${LOGFILE}" diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index a96fb14e22..721d18b8cb 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -28,10 +28,10 @@ services: POSTGRESQL_USERNAME: root POSTGRESQL_PASSWORD: root POSTGRESQL_DATABASE: dolphinscheduler - POSTGRESQL_REPLICATION_PASSWORD: dolphinscheduler@123 volumes: - dolphinscheduler-postgresql:/bitnami/postgresql - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d + restart: unless-stopped networks: - dolphinscheduler @@ -46,13 +46,14 @@ services: ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-api: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-api - command: ["api-server"] + command: api-server ports: - 12345:12345 environment: @@ -63,9 +64,9 @@ services: DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 - ZOOKEEPER_ROOT: /dolphinscheduler - DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE: "HDFS" - DOLPHINSCHEDULER_FS_DEFAULTFS: "file:///data/dolphinscheduler" + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] interval: 30s @@ -77,14 +78,15 @@ services: - dolphinscheduler-zookeeper volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-resource-storage-local:/dolphinscheduler + - dolphinscheduler-resource-local:/dolphinscheduler + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-alert: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-alert - command: ["alert-server"] + command: alert-server environment: TZ: Asia/Shanghai XLS_FILE_PATH: "/tmp/xls" @@ -116,13 +118,14 @@ services: - dolphinscheduler-postgresql volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-master: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-master - command: ["master-server"] + command: master-server ports: - 5678:5678 environment: @@ -140,7 +143,6 @@ services: DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 - ZOOKEEPER_ROOT: /dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] interval: 30s @@ -152,13 +154,14 @@ services: - dolphinscheduler-zookeeper volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-worker: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-worker - command: ["worker-server"] + command: worker-server ports: - 1234:1234 - 50051:50051 @@ -170,16 +173,11 @@ services: WORKER_MAX_CPULOAD_AVG: "100" WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" - DATABASE_HOST: dolphinscheduler-postgresql - DATABASE_PORT: 5432 - DATABASE_USERNAME: root - DATABASE_PASSWORD: root - DATABASE_DATABASE: dolphinscheduler + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 - ZOOKEEPER_ROOT: /dolphinscheduler - DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE: "HDFS" - DOLPHINSCHEDULER_FS_DEFAULTFS: "file:///data/dolphinscheduler" + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] interval: 30s @@ -190,18 +188,11 @@ services: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - - type: bind - source: ./dolphinscheduler_env.sh - target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh - - type: volume - source: dolphinscheduler-worker-data - target: /tmp/dolphinscheduler - - type: volume - source: dolphinscheduler-logs - target: /opt/dolphinscheduler/logs - - type: volume - source: dolphinscheduler-resource-storage-local - target: /dolphinscheduler + - ./dolphinscheduler_env.sh:/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-resource-local:/dolphinscheduler + restart: unless-stopped networks: - dolphinscheduler @@ -215,8 +206,4 @@ volumes: dolphinscheduler-zookeeper: dolphinscheduler-worker-data: dolphinscheduler-logs: - dolphinscheduler-resource-storage-local: - -configs: - dolphinscheduler-worker-task-env: - file: ./dolphinscheduler_env.sh \ No newline at end of file + dolphinscheduler-resource-local: \ No newline at end of file diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index 06cb781121..6ef73abf2f 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -27,7 +27,6 @@ services: POSTGRESQL_USERNAME: root POSTGRESQL_PASSWORD: root POSTGRESQL_DATABASE: dolphinscheduler - POSTGRESQL_REPLICATION_PASSWORD: dolphinscheduler@123 volumes: - dolphinscheduler-postgresql:/bitnami/postgresql networks: @@ -54,7 +53,7 @@ services: dolphinscheduler-api: image: apache/dolphinscheduler:latest - command: ["api-server"] + command: api-server ports: - 12345:12345 environment: @@ -65,7 +64,9 @@ services: DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 - ZOOKEEPER_ROOT: /dolphinscheduler + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] interval: 30s @@ -82,7 +83,7 @@ services: dolphinscheduler-alert: image: apache/dolphinscheduler:latest - command: ["alert-server"] + command: alert-server environment: TZ: Asia/Shanghai XLS_FILE_PATH: "/tmp/xls" @@ -120,7 +121,7 @@ services: dolphinscheduler-master: image: apache/dolphinscheduler:latest - command: ["master-server"] + command: master-server ports: - 5678:5678 environment: @@ -154,7 +155,7 @@ services: dolphinscheduler-worker: image: apache/dolphinscheduler:latest - command: ["worker-server"] + command: worker-server ports: - 1234:1234 - 50051:50051 @@ -166,25 +167,23 @@ services: WORKER_MAX_CPULOAD_AVG: "100" WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" - DATABASE_HOST: dolphinscheduler-postgresql - DATABASE_PORT: 5432 - DATABASE_USERNAME: root - DATABASE_PASSWORD: root - DATABASE_DATABASE: dolphinscheduler + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] interval: 30s timeout: 5s retries: 3 start_period: 30s - volumes: - - dolphinscheduler-worker-data:/tmp/dolphinscheduler - - dolphinscheduler-logs:/opt/dolphinscheduler/logs configs: - source: dolphinscheduler-worker-task-env target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + volumes: + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs networks: - dolphinscheduler deploy: