diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index acd2cb99e5..c48b51e377 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -90,6 +90,6 @@ RUN chmod +x /root/checkpoint.sh && \ RUN rm -rf /var/cache/apk/* #9. expose port -EXPOSE 2181 2888 3888 5432 12345 50051 8888 +EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888 ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] \ No newline at end of file diff --git a/dockerfile/README.md b/dockerfile/README.md index 60af7fad8f..b407f57d3b 100644 --- a/dockerfile/README.md +++ b/dockerfile/README.md @@ -162,18 +162,6 @@ This environment variable sets the runtime environment for task. The default val User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler` -**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`** - -Directory path for user data download. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/download` - -**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`** - -Process execute directory. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/exec` - -**`TASK_QUEUE`** - -This environment variable sets the task queue for `master-server` and `worker-serverr`. The default value is `zookeeper`. - **`ZOOKEEPER_QUORUM`** This environment variable sets zookeeper quorum for `master-server` and `worker-serverr`. The default value is `127.0.0.1:2181`. @@ -208,6 +196,10 @@ This environment variable sets max cpu load avg for `master-server`. The default This environment variable sets reserved memory for `master-server`. The default value is `0.1`. +**`MASTER_LISTEN_PORT`** + +This environment variable sets port for `master-server`. The default value is `5678`. + **`WORKER_EXEC_THREADS`** This environment variable sets exec thread num for `worker-server`. The default value is `100`. @@ -228,6 +220,14 @@ This environment variable sets max cpu load avg for `worker-server`. The default This environment variable sets reserved memory for `worker-server`. The default value is `0.1`. +**`WORKER_LISTEN_PORT`** + +This environment variable sets port for `worker-server`. The default value is `1234`. + +**`WORKER_GROUP`** + +This environment variable sets group for `worker-server`. The default value is `default`. + **`XLS_FILE_PATH`** This environment variable sets xls file path for `alert-server`. The default value is `/tmp/xls`. diff --git a/dockerfile/README_zh_CN.md b/dockerfile/README_zh_CN.md index 900c8b50d9..187261581d 100644 --- a/dockerfile/README_zh_CN.md +++ b/dockerfile/README_zh_CN.md @@ -162,18 +162,6 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler`。 -**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`** - -用户数据下载目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler/download`。 - -**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`** - -任务执行目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler/exec`。 - -**`TASK_QUEUE`** - -配置`master-server`和`worker-serverr`的`Zookeeper`任务队列名, 默认值 `zookeeper`。 - **`ZOOKEEPER_QUORUM`** 配置`master-server`和`worker-serverr`的`Zookeeper`地址, 默认值 `127.0.0.1:2181`。 @@ -208,6 +196,10 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 配置`master-server`的保留内存,默认值 `0.1`。 +**`MASTER_LISTEN_PORT`** + +配置`master-server`的端口,默认值 `5678`。 + **`WORKER_EXEC_THREADS`** 配置`worker-server`中的执行线程数量,默认值 `100`。 @@ -228,6 +220,14 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 配置`worker-server`的保留内存,默认值 `0.1`。 +**`WORKER_LISTEN_PORT`** + +配置`worker-server`的端口,默认值 `1234`。 + +**`WORKER_GROUP`** + +配置`worker-server`的分组,默认值 `default`。 + **`XLS_FILE_PATH`** 配置`alert-server`的`XLS`文件的存储路径,默认值 `/tmp/xls`。 diff --git a/dockerfile/conf/dolphinscheduler/application-api.properties.tpl b/dockerfile/conf/dolphinscheduler/application-api.properties.tpl index 424ea55f7d..88915923fa 100644 --- a/dockerfile/conf/dolphinscheduler/application-api.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/application-api.properties.tpl @@ -14,21 +14,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # + # server port server.port=12345 + # session config server.servlet.session.timeout=7200 + # servlet config server.servlet.context-path=/dolphinscheduler/ + # file size limit for upload spring.servlet.multipart.max-file-size=1024MB spring.servlet.multipart.max-request-size=1024MB + # post content server.jetty.max-http-post-size=5000000 + # i18n spring.messages.encoding=UTF-8 + #i18n classpath folder , file prefix messages, if have many files, use "," seperator spring.messages.basename=i18n/messages + # Authentication types (supported types: PASSWORD) security.authentication.type=PASSWORD diff --git a/dockerfile/conf/dolphinscheduler/application.properties.tpl b/dockerfile/conf/dolphinscheduler/application.properties.tpl deleted file mode 100644 index 6dd8a18e11..0000000000 --- a/dockerfile/conf/dolphinscheduler/application.properties.tpl +++ /dev/null @@ -1,115 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# base spring data source configuration -spring.datasource.type=com.alibaba.druid.pool.DruidDataSource -# postgre -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 -# mysql -#spring.datasource.driver-class-name=com.mysql.jdbc.Driver -#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 -spring.datasource.username=${POSTGRESQL_USERNAME} -spring.datasource.password=${POSTGRESQL_PASSWORD} -# connection configuration -spring.datasource.initialSize=5 -# min connection number -spring.datasource.minIdle=5 -# max connection number -spring.datasource.maxActive=50 -# max wait time for get a connection in milliseconds. if configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. -# If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. -spring.datasource.maxWait=60000 -# milliseconds for check to close free connections -spring.datasource.timeBetweenEvictionRunsMillis=60000 -# the Destroy thread detects the connection interval and closes the physical connection in milliseconds if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. -spring.datasource.timeBetweenConnectErrorMillis=60000 -# the longest time a connection remains idle without being evicted, in milliseconds -spring.datasource.minEvictableIdleTimeMillis=300000 -#the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. -spring.datasource.validationQuery=SELECT 1 -#check whether the connection is valid for timeout, in seconds -spring.datasource.validationQueryTimeout=3 -# when applying for a connection, if it is detected that the connection is idle longer than time Between Eviction Runs Millis, -# validation Query is performed to check whether the connection is valid -spring.datasource.testWhileIdle=true -#execute validation to check if the connection is valid when applying for a connection -spring.datasource.testOnBorrow=true -#execute validation to check if the connection is valid when the connection is returned -spring.datasource.testOnReturn=false -spring.datasource.defaultAutoCommit=true -spring.datasource.keepAlive=true -# open PSCache, specify count PSCache for every connection -spring.datasource.poolPreparedStatements=true -spring.datasource.maxPoolPreparedStatementPerConnectionSize=20 -spring.datasource.spring.datasource.filters=stat,wall,log4j -spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 - -#mybatis -mybatis-plus.mapper-locations=classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml -mybatis-plus.typeEnumsPackage=org.apache.dolphinscheduler.*.enums -#Entity scan, where multiple packages are separated by a comma or semicolon -mybatis-plus.typeAliasesPackage=org.apache.dolphinscheduler.dao.entity -#Primary key type AUTO:" database ID AUTO ", INPUT:" user INPUT ID", ID_WORKER:" global unique ID (numeric type unique ID)", UUID:" global unique ID UUID"; -mybatis-plus.global-config.db-config.id-type=AUTO -#Field policy IGNORED:" ignore judgment ",NOT_NULL:" not NULL judgment "),NOT_EMPTY:" not NULL judgment" -mybatis-plus.global-config.db-config.field-strategy=NOT_NULL -#The hump underline is converted -mybatis-plus.global-config.db-config.column-underline=true -mybatis-plus.global-config.db-config.logic-delete-value=-1 -mybatis-plus.global-config.db-config.logic-not-delete-value=0 -mybatis-plus.global-config.db-config.banner=false -#The original configuration -mybatis-plus.configuration.map-underscore-to-camel-case=true -mybatis-plus.configuration.cache-enabled=false -mybatis-plus.configuration.call-setters-on-nulls=true -mybatis-plus.configuration.jdbc-type-for-null=null - -# master settings -# master execute thread num -master.exec.threads=${MASTER_EXEC_THREADS} -# master execute task number in parallel -master.exec.task.num=${MASTER_EXEC_TASK_NUM} -# master heartbeat interval -master.heartbeat.interval=${MASTER_HEARTBEAT_INTERVAL} -# master commit task retry times -master.task.commit.retryTimes=${MASTER_TASK_COMMIT_RETRYTIMES} -# master commit task interval -master.task.commit.interval=${MASTER_TASK_COMMIT_INTERVAL} -# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 -master.max.cpuload.avg=${MASTER_MAX_CPULOAD_AVG} -# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. -master.reserved.memory=${MASTER_RESERVED_MEMORY} - -# worker settings -# worker execute thread num -worker.exec.threads=${WORKER_EXEC_THREADS} -# worker heartbeat interval -worker.heartbeat.interval=${WORKER_HEARTBEAT_INTERVAL} -# submit the number of tasks at a time -worker.fetch.task.num=${WORKER_FETCH_TASK_NUM} -# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 -worker.max.cpuload.avg=${WORKER_MAX_CPULOAD_AVG} -# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G. -worker.reserved.memory=${WORKER_RESERVED_MEMORY} - -# data quality analysis is not currently in use. please ignore the following configuration -# task record -task.record.flag=false -task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8 -task.record.datasource.username=xx -task.record.datasource.password=xx diff --git a/dockerfile/conf/dolphinscheduler/common.properties.tpl b/dockerfile/conf/dolphinscheduler/common.properties.tpl index 8134fc7a9b..f318ff8414 100644 --- a/dockerfile/conf/dolphinscheduler/common.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/common.properties.tpl @@ -15,70 +15,64 @@ # limitations under the License. # -#task queue implementation, default "zookeeper" -dolphinscheduler.queue.impl=${TASK_QUEUE} - -#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 -zookeeper.quorum=${ZOOKEEPER_QUORUM} -#dolphinscheduler root directory -zookeeper.dolphinscheduler.root=/dolphinscheduler -#dolphinscheduler failover directory -zookeeper.session.timeout=300 -zookeeper.connection.timeout=300 -zookeeper.retry.base.sleep=100 -zookeeper.retry.max.sleep=30000 -zookeeper.retry.maxtime=5 - #============================================================================ # System #============================================================================ # system env path. self configuration, please make sure the directory and file exists and have read write execute permissions dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} -#resource.view.suffixs -resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties -# is development state? default "false" -development.state=true + # user data directory path, self configuration, please make sure the directory exists and have read write permissions data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} -# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions -data.download.basedir.path=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH} -# process execute directory. self configuration, please make sure the directory exists and have read write permissions -process.exec.basepath=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH} # resource upload startup type : HDFS,S3,NONE -res.upload.startup.type=NONE +resource.storage.type=NONE #============================================================================ # HDFS #============================================================================ -# Users who have permission to create directories under the HDFS root path -hdfs.root.user=hdfs -# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended -data.store2hdfs.basepath=/dolphinscheduler +# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended +#resource.upload.path=/dolphinscheduler + # whether kerberos starts -hadoop.security.authentication.startup.state=false +#hadoop.security.authentication.startup.state=false + # java.security.krb5.conf path -java.security.krb5.conf.path=/opt/krb5.conf +#java.security.krb5.conf.path=/opt/krb5.conf + # loginUserFromKeytab user -login.user.keytab.username=hdfs-mycluster@ESZ.COM +#login.user.keytab.username=hdfs-mycluster@ESZ.COM + # loginUserFromKeytab path -login.user.keytab.path=/opt/hdfs.headless.keytab +#login.user.keytab.path=/opt/hdfs.headless.keytab + +#resource.view.suffixs +#resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties + +# if resource.storage.type=HDFS, the user need to have permission to create directories under the HDFS root path +hdfs.root.user=hdfs + +# kerberos expire time +kerberos.expire.time=7 #============================================================================ # S3 #============================================================================ -# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml -# to the conf directory,support s3,for example : s3a://dolphinscheduler +# if resource.storage.type=S3,the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir fs.defaultFS=hdfs://mycluster:8020 -# s3 need,s3 endpoint -fs.s3a.endpoint=http://192.168.199.91:9010 -# s3 need,s3 access key -fs.s3a.access.key=A3DXS30FO22544RE -# s3 need,s3 secret key -fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK -#resourcemanager ha note this need ips , this empty if single + +# if resource.storage.type=S3,s3 endpoint +#fs.s3a.endpoint=http://192.168.199.91:9010 + +# if resource.storage.type=S3,s3 access key +#fs.s3a.access.key=A3DXS30FO22544RE + +# if resource.storage.type=S3,s3 secret key +#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK + +# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx -# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine + +# If resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s diff --git a/dockerfile/conf/dolphinscheduler/datasource.properties.tpl b/dockerfile/conf/dolphinscheduler/datasource.properties.tpl new file mode 100644 index 0000000000..aefb9e3b0b --- /dev/null +++ b/dockerfile/conf/dolphinscheduler/datasource.properties.tpl @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +# mysql +#spring.datasource.driver-class-name=com.mysql.jdbc.Driver +#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 + +# postgre +spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 +spring.datasource.username=${POSTGRESQL_USERNAME} +spring.datasource.password=${POSTGRESQL_PASSWORD} + +## base spring data source configuration todo need to remove +#spring.datasource.type=com.alibaba.druid.pool.DruidDataSource + +# connection configuration +#spring.datasource.initialSize=5 +# min connection number +#spring.datasource.minIdle=5 +# max connection number +#spring.datasource.maxActive=50 + +# max wait time for get a connection in milliseconds. if configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. +# If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. +#spring.datasource.maxWait=60000 + +# milliseconds for check to close free connections +#spring.datasource.timeBetweenEvictionRunsMillis=60000 + +# the Destroy thread detects the connection interval and closes the physical connection in milliseconds if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. +#spring.datasource.timeBetweenConnectErrorMillis=60000 + +# the longest time a connection remains idle without being evicted, in milliseconds +#spring.datasource.minEvictableIdleTimeMillis=300000 + +#the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. +#spring.datasource.validationQuery=SELECT 1 + +#check whether the connection is valid for timeout, in seconds +#spring.datasource.validationQueryTimeout=3 + +# when applying for a connection, if it is detected that the connection is idle longer than time Between Eviction Runs Millis, +# validation Query is performed to check whether the connection is valid +#spring.datasource.testWhileIdle=true + +#execute validation to check if the connection is valid when applying for a connection +#spring.datasource.testOnBorrow=true +#execute validation to check if the connection is valid when the connection is returned +#spring.datasource.testOnReturn=false +#spring.datasource.defaultAutoCommit=true +#spring.datasource.keepAlive=true + +# open PSCache, specify count PSCache for every connection +#spring.datasource.poolPreparedStatements=true +#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20 \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/master.properties.tpl b/dockerfile/conf/dolphinscheduler/master.properties.tpl new file mode 100644 index 0000000000..17dd6f9d69 --- /dev/null +++ b/dockerfile/conf/dolphinscheduler/master.properties.tpl @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# master execute thread num +master.exec.threads=${MASTER_EXEC_THREADS} + +# master execute task number in parallel +master.exec.task.num=${MASTER_EXEC_TASK_NUM} + +# master heartbeat interval +master.heartbeat.interval=${MASTER_HEARTBEAT_INTERVAL} + +# master commit task retry times +master.task.commit.retryTimes=${MASTER_TASK_COMMIT_RETRYTIMES} + +# master commit task interval +master.task.commit.interval=${MASTER_TASK_COMMIT_INTERVAL} + +# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 +master.max.cpuload.avg=${MASTER_MAX_CPULOAD_AVG} + +# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. +master.reserved.memory=${MASTER_RESERVED_MEMORY} + +# master listen port +#master.listen.port=${MASTER_LISTEN_PORT} \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl index 7c7c92e425..25645795bb 100644 --- a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl @@ -19,39 +19,36 @@ # Configure Main Scheduler Properties #============================================================================ #org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate -org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate -# postgre -org.quartz.dataSource.myDs.driver = org.postgresql.Driver -org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 -org.quartz.dataSource.myDs.user = ${POSTGRESQL_USERNAME} -org.quartz.dataSource.myDs.password = ${POSTGRESQL_PASSWORD} -org.quartz.scheduler.instanceName = DolphinScheduler -org.quartz.scheduler.instanceId = AUTO -org.quartz.scheduler.makeSchedulerThreadDaemon = true -org.quartz.jobStore.useProperties = false +#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + +#org.quartz.scheduler.instanceName = DolphinScheduler +#org.quartz.scheduler.instanceId = AUTO +#org.quartz.scheduler.makeSchedulerThreadDaemon = true +#org.quartz.jobStore.useProperties = false #============================================================================ # Configure ThreadPool #============================================================================ -org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool -org.quartz.threadPool.makeThreadsDaemons = true -org.quartz.threadPool.threadCount = 25 -org.quartz.threadPool.threadPriority = 5 + +#org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool +#org.quartz.threadPool.makeThreadsDaemons = true +#org.quartz.threadPool.threadCount = 25 +#org.quartz.threadPool.threadPriority = 5 #============================================================================ # Configure JobStore #============================================================================ -org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX -org.quartz.jobStore.tablePrefix = QRTZ_ -org.quartz.jobStore.isClustered = true -org.quartz.jobStore.misfireThreshold = 60000 -org.quartz.jobStore.clusterCheckinInterval = 5000 -org.quartz.jobStore.acquireTriggersWithinLock=true -org.quartz.jobStore.dataSource = myDs + +#org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX + +#org.quartz.jobStore.tablePrefix = QRTZ_ +#org.quartz.jobStore.isClustered = true +#org.quartz.jobStore.misfireThreshold = 60000 +#org.quartz.jobStore.clusterCheckinInterval = 5000 +#org.quartz.jobStore.acquireTriggersWithinLock=true +#org.quartz.jobStore.dataSource = myDs #============================================================================ -# Configure Datasources +# Configure Datasources #============================================================================ -org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider -org.quartz.dataSource.myDs.maxConnections = 10 -org.quartz.dataSource.myDs.validationQuery = select 1 \ No newline at end of file +#org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/worker.properties.tpl b/dockerfile/conf/dolphinscheduler/worker.properties.tpl new file mode 100644 index 0000000000..d596be94bc --- /dev/null +++ b/dockerfile/conf/dolphinscheduler/worker.properties.tpl @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# worker execute thread num +worker.exec.threads=${WORKER_EXEC_THREADS} + +# worker heartbeat interval +worker.heartbeat.interval=${WORKER_HEARTBEAT_INTERVAL} + +# submit the number of tasks at a time +worker.fetch.task.num=${WORKER_FETCH_TASK_NUM} + +# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 +worker.max.cpuload.avg=${WORKER_MAX_CPULOAD_AVG} + +# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G. +worker.reserved.memory=${WORKER_RESERVED_MEMORY} + +# worker listener port +#worker.listen.port=${WORKER_LISTEN_PORT} + +# default worker group +#worker.group=${WORKER_GROUP} \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/zookeeper.properties.tpl b/dockerfile/conf/dolphinscheduler/zookeeper.properties.tpl new file mode 100644 index 0000000000..a0ef72dc8f --- /dev/null +++ b/dockerfile/conf/dolphinscheduler/zookeeper.properties.tpl @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 +zookeeper.quorum=${ZOOKEEPER_QUORUM} + +# dolphinscheduler root directory +#zookeeper.dolphinscheduler.root=/dolphinscheduler + +# dolphinscheduler failover directory +#zookeeper.session.timeout=300 +#zookeeper.connection.timeout=300 +#zookeeper.retry.base.sleep=100 +#zookeeper.retry.max.sleep=30000 +#zookeeper.retry.maxtime=5 \ No newline at end of file diff --git a/dockerfile/startup-init-conf.sh b/dockerfile/startup-init-conf.sh index d2b50fa3e7..da6eb21b7d 100644 --- a/dockerfile/startup-init-conf.sh +++ b/dockerfile/startup-init-conf.sh @@ -35,13 +35,10 @@ export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"} #============================================================================ export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} -export DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH:-"/tmp/dolphinscheduler/download"} -export DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH:-"/tmp/dolphinscheduler/exec"} #============================================================================ # Zookeeper #============================================================================ -export TASK_QUEUE=${TASK_QUEUE:-"zookeeper"} export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"} #============================================================================ @@ -54,6 +51,7 @@ export MASTER_TASK_COMMIT_RETRYTIMES=${MASTER_TASK_COMMIT_RETRYTIMES:-"5"} export MASTER_TASK_COMMIT_INTERVAL=${MASTER_TASK_COMMIT_INTERVAL:-"1000"} export MASTER_MAX_CPULOAD_AVG=${MASTER_MAX_CPULOAD_AVG:-"100"} export MASTER_RESERVED_MEMORY=${MASTER_RESERVED_MEMORY:-"0.1"} +export MASTER_LISTEN_PORT=${MASTER_LISTEN_PORT:-"5678"} #============================================================================ # Worker Server @@ -63,6 +61,8 @@ export WORKER_HEARTBEAT_INTERVAL=${WORKER_HEARTBEAT_INTERVAL:-"10"} export WORKER_FETCH_TASK_NUM=${WORKER_FETCH_TASK_NUM:-"3"} export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"100"} export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.1"} +export WORKER_LISTEN_PORT=${WORKER_LISTEN_PORT:-"1234"} +export WORKER_GROUP=${WORKER_GROUP:-"default"} #============================================================================ # Alert Server