分布式调度框架。
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

268 lines
9.1 KiB

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
spring:
application:
name: standalone-server
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
banner:
charset: UTF-8
cache:
# default enable cache, you can disable by `type: none`
type: none
cache-names:
- tenant
- user
- processDefinition
- processTaskRelation
- taskDefinition
caffeine:
spec: maximumSize=100,expireAfterWrite=300s,recordStats
sql:
init:
schema-locations: classpath:sql/dolphinscheduler_h2.sql
datasource:
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true
username: sa
password: ""
quartz:
job-store-type: jdbc
jdbc:
initialize-schema: never
properties:
org.quartz.threadPool:threadPriority: 5
org.quartz.jobStore.isClustered: true
org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true
org.quartz.scheduler.instanceName: DolphinScheduler
org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
org.quartz.jobStore.useProperties: false
org.quartz.threadPool.makeThreadsDaemons: true
org.quartz.threadPool.threadCount: 25
org.quartz.jobStore.misfireThreshold: 60000
org.quartz.scheduler.makeSchedulerThreadDaemon: true
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
org.quartz.jobStore.clusterCheckinInterval: 5000
servlet:
multipart:
max-file-size: 1024MB
max-request-size: 1024MB
messages:
basename: i18n/messages
3 years ago
jpa:
hibernate:
ddl-auto: none
registry:
type: zookeeper
zookeeper:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
max-retries: 5
session-timeout: 30s
connection-timeout: 9s
block-until-connected: 600ms
digest: ~
security:
authentication:
# Authentication types (supported types: PASSWORD,LDAP)
type: PASSWORD
# IF you set type `LDAP`, below config will be effective
ldap:
# ldap server config
urls: ldap://ldap.forumsys.com:389/
base-dn: dc=example,dc=com
username: cn=read-only-admin,dc=example,dc=com
password: password
user:
# admin userId when you use LDAP login
admin: read-only-admin
identity-attribute: uid
email-attribute: mail
# action when ldap user is not exist (supported types: CREATE,DENY)
not-exist-action: CREATE
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
traffic:
control:
global-switch: false
max-global-qps-rate: 300
tenant-switch: false
default-tenant-qps-rate: 10
#customize-tenant-qps-rate:
# eg.
#tenant1: 11
#tenant2: 20
master:
listen-port: 5678
# master fetch command num
fetch-command-num: 10
# master prepare execute thread number to limit handle commands in parallel
pre-exec-threads: 10
# master execute thread number to limit process instances in parallel
exec-threads: 10
# master dispatch task number per batch
dispatch-task-number: 3
# master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
host-selector: lower_weight
# master heartbeat interval
heartbeat-interval: 10s
# Master heart beat task error threshold, if the continuous error count exceed this count, the master will close.
heartbeat-error-threshold: 5
# master commit task retry times
task-commit-retry-times: 5
# master commit task interval
task-commit-interval: 1s
state-wheel-interval: 5s
# master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
max-cpu-load-avg: -1
# master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
reserved-memory: 0.3
# failover interval
failover-interval: 10m
# kill yarn jon when failover taskInstance, default true
kill-yarn-job-when-task-failover: true
worker:
# worker listener port
listen-port: 1234
# worker execute thread number to limit task instances in parallel
exec-threads: 10
# worker heartbeat interval
heartbeat-interval: 10s
# Worker heart beat task error threshold, if the continuous error count exceed this count, the worker will close.
heartbeat-error-threshold: 5
# worker host weight to dispatch tasks, default value 100
host-weight: 100
[Fix-10181] Fix the logic of judging that the tenant does not exist (#10185) * [Fix-10181] Fix the logic of judging that the tenant does not exist Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add parameter description to configuration.md Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add parameter description to configuration.md Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add parameter description to configuration.md Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac Co-authored-by: ouyangl <ouyangl@tebon.com.cn>
2 years ago
# tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
tenant-auto-create: true
[Fix-10181] Fix the logic of judging that the tenant does not exist (#10185) * [Fix-10181] Fix the logic of judging that the tenant does not exist Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add parameter description to configuration.md Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add parameter description to configuration.md Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add parameter description to configuration.md Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac Co-authored-by: ouyangl <ouyangl@tebon.com.cn>
2 years ago
#Scenes to be used for distributed users.For example,users created by FreeIpa are stored in LDAP.This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants.
tenant-distributed-user: false
# worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2
max-cpu-load-avg: -1
# worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G
reserved-memory: 0.3
# default worker groups separated by comma, like 'worker.groups=default,test'
groups:
- default
# alert server listen host
alert-listen-host: localhost
alert-listen-port: 50052
alert:
port: 50052
# Mark each alert of alert server if late after x milliseconds as failed.
# Define value is (0 = infinite), and alert server would be waiting alert result.
wait-timeout: 0
python-gateway:
# Weather enable python gateway server or not. The default value is true.
enabled: true
# The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
# between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
gateway-server-address: 0.0.0.0
# The port of Python gateway server start. Define which port you could connect to Python gateway server from
# Python API side.
gateway-server-port: 25333
# The address of Python callback client.
python-address: 127.0.0.1
# The port of Python callback client.
python-port: 25334
# Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
# and socket server would never close even though no requests accept
connect-timeout: 0
# Close each active connection of socket server if python program not active after x milliseconds. Define value is
# (0 = infinite), and socket server would never close even though no requests accept
read-timeout: 0
server:
port: 12345
servlet:
session:
timeout: 120m
context-path: /dolphinscheduler/
compression:
enabled: true
mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
jetty:
max-http-form-post-size: 5000000
management:
endpoints:
web:
exposure:
include: '*'
endpoint:
health:
enabled: true
show-details: always
health:
db:
enabled: true
defaults:
enabled: false
metrics:
tags:
application: ${spring.application.name}
audit:
enabled: true
metrics:
enabled: true
# Override by profile
---
spring:
config:
activate:
on-profile: postgresql
quartz:
properties:
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
datasource:
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
username: root
password: root
---
spring:
config:
activate:
on-profile: mysql
sql:
init:
schema-locations: classpath:sql/dolphinscheduler_mysql.sql
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
username: root
password: root