diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl index 14ea58ba6e..0533c7e710 100644 --- a/docker/build/conf/dolphinscheduler/common.properties.tpl +++ b/docker/build/conf/dolphinscheduler/common.properties.tpl @@ -31,7 +31,7 @@ resource.storage.type=${DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE} # HDFS #============================================================================ # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended -#resource.upload.path=/dolphinscheduler +resource.upload.path=${RESOURCE_UPLOAD_PATH} # whether kerberos starts #hadoop.security.authentication.startup.state=false @@ -61,13 +61,13 @@ kerberos.expire.time=7 fs.defaultFS=${DOLPHINSCHEDULER_FS_DEFAULTFS} # if resource.storage.type=S3，s3 endpoint -#fs.s3a.endpoint=http://192.168.199.91:9010 +fs.s3a.endpoint=${FS_S3A_ENDPOINT} # if resource.storage.type=S3，s3 access key -#fs.s3a.access.key=A3DXS30FO22544RE +fs.s3a.access.key=${FS_S3A_ACCESS_KEY} # if resource.storage.type=S3，s3 secret key -#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK +fs.s3a.secret.key=${FS_S3A_SECRET_KEY} # if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx diff --git a/docker/build/conf/dolphinscheduler/datasource.properties.tpl b/docker/build/conf/dolphinscheduler/datasource.properties.tpl index aefb9e3b0b..f7c5ee6881 100644 --- a/docker/build/conf/dolphinscheduler/datasource.properties.tpl +++ b/docker/build/conf/dolphinscheduler/datasource.properties.tpl @@ -15,16 +15,11 @@ # limitations under the License. # - -# mysql -#spring.datasource.driver-class-name=com.mysql.jdbc.Driver -#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 - -# postgre -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 -spring.datasource.username=${POSTGRESQL_USERNAME} -spring.datasource.password=${POSTGRESQL_PASSWORD} +# db +spring.datasource.driver-class-name=${DATABASE_DRIVER} +spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS} +spring.datasource.username=${DATABASE_USERNAME} +spring.datasource.password=${DATABASE_PASSWORD} ## base spring data source configuration todo need to remove #spring.datasource.type=com.alibaba.druid.pool.DruidDataSource diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl index 362a4e85ad..51540aa345 100644 --- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl +++ b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl @@ -19,7 +19,7 @@ zookeeper.quorum=${ZOOKEEPER_QUORUM} # dolphinscheduler root directory -#zookeeper.dolphinscheduler.root=/dolphinscheduler +zookeeper.dolphinscheduler.root=${ZOOKEEPER_ROOT} # dolphinscheduler failover directory #zookeeper.session.timeout=60000 diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index da6eb21b7d..face6be4b7 100644 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -24,22 +24,32 @@ echo "init env variables" #============================================================================ # Database Source #============================================================================ -export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"} -export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"} -export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"} -export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"} -export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"} +export DATABASE_HOST=${DATABASE_HOST:-"127.0.0.1"} +export DATABASE_PORT=${DATABASE_PORT:-"5432"} +export DATABASE_USERNAME=${DATABASE_USERNAME:-"root"} +export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"} +export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"} +export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"} +export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"} +export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} #============================================================================ # System #============================================================================ export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} +export RESOURCE_STORAGE_TYPE=${DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE:-"HDFS"} +export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"} +export FS_DEFAULT_FS=${DOLPHINSCHEDULER_FS_DEFAULT_FS:-"file:///data/dolphinscheduler"} +export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"} +export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"} +export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"} #============================================================================ # Zookeeper #============================================================================ export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"} +export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"} #============================================================================ # Master Server diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 2034726868..16af4a3273 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -57,11 +57,11 @@ services: - 12345:12345 environment: TZ: Asia/Shanghai - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE: "HDFS" DOLPHINSCHEDULER_FS_DEFAULTFS: "file:///data/dolphinscheduler" @@ -123,11 +123,11 @@ services: ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_USERS: "" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] interval: 30s @@ -156,11 +156,11 @@ services: MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] @@ -192,11 +192,11 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE: "HDFS" DOLPHINSCHEDULER_FS_DEFAULTFS: "file:///data/dolphinscheduler" diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index 5c3873fb07..06dabce6c3 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -58,11 +58,11 @@ services: - 12345:12345 environment: TZ: Asia/Shanghai - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] @@ -120,11 +120,11 @@ services: ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_USERS: "" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] interval: 30 @@ -153,11 +153,11 @@ services: MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] @@ -188,11 +188,11 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]