# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Default values for dolphinscheduler-chart. # This is a YAML-formatted file. # Declare variables to be passed into your templates. timezone: "Asia/Shanghai" # Used to detect whether dolphinscheduler dependent services such as database are ready initImage: pullPolicy: "IfNotPresent" busybox: "busybox:1.30.1" image: registry: "dolphinscheduler.docker.scarf.sh/apache" tag: "dev-SNAPSHOT" pullPolicy: "IfNotPresent" pullSecret: "" master: dolphinscheduler-master worker: dolphinscheduler-worker api: dolphinscheduler-api alert: dolphinscheduler-alert-server tools: dolphinscheduler-tools ## If not exists external database, by default, Dolphinscheduler's database will use it. postgresql: enabled: true postgresqlUsername: "root" postgresqlPassword: "root" postgresqlDatabase: "dolphinscheduler" params: "characterEncoding=utf8" persistence: enabled: false size: "20Gi" storageClass: "-" mysql: enabled: false auth: username: "ds" password: "ds" database: "dolphinscheduler" params: "characterEncoding=utf8" primary: persistence: enabled: false size: "20Gi" storageClass: "-" minio: enabled: true auth: rootUser: minioadmin rootPassword: minioadmin persistence: enabled: false defaultBuckets: "dolphinscheduler" ## If exists external database, and set postgresql.enable value to false. ## external database will be used, otherwise Dolphinscheduler's database will be used. externalDatabase: enabled: false type: "postgresql" host: "localhost" port: "5432" username: "root" password: "root" database: "dolphinscheduler" params: "characterEncoding=utf8" ## If not exists external registry, the zookeeper registry will be used by default. zookeeper: enabled: true service: port: 2181 fourlwCommandsWhitelist: "srvr,ruok,wchs,cons" persistence: enabled: false size: "20Gi" storageClass: "-" etcd: enabled: false endpoints: "" namespace: "dolphinscheduler" user: "" passWord: "" authority: "" # Please create a new folder: deploy/kubernetes/dolphinscheduler/etcd-certs ssl: enabled: false certFile: "etcd-certs/ca.crt" keyCertChainFile: "etcd-certs/client.crt" keyFile: "etcd-certs/client.pem" ## If exists external registry and set zookeeper.enable value to false, the external registry will be used. externalRegistry: registryPluginName: "zookeeper" registryServers: "127.0.0.1:2181" conf: common: # user data local directory path, please make sure the directory exists and have read write permissions data.basedir.path: /tmp/dolphinscheduler # resource storage type: HDFS, S3, OSS, GCS, ABS, NONE resource.storage.type: S3 # resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended resource.storage.upload.base.path: /dolphinscheduler # The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required resource.aws.access.key.id: minioadmin # The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required resource.aws.secret.access.key: minioadmin # The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required resource.aws.region: ca-central-1 # The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. resource.aws.s3.bucket.name: dolphinscheduler # You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn resource.aws.s3.endpoint: http://minio:9000 # alibaba cloud access key id, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.id: # alibaba cloud access key secret, required if you set resource.storage.type=OSS resource.alibaba.cloud.access.key.secret: # alibaba cloud region, required if you set resource.storage.type=OSS resource.alibaba.cloud.region: cn-hangzhou # oss bucket name, required if you set resource.storage.type=OSS resource.alibaba.cloud.oss.bucket.name: dolphinscheduler # oss bucket endpoint, required if you set resource.storage.type=OSS resource.alibaba.cloud.oss.endpoint: https://oss-cn-hangzhou.aliyuncs.com # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path resource.hdfs.root.user: hdfs # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir resource.hdfs.fs.defaultFS: hdfs://mycluster:8020 # whether to startup kerberos hadoop.security.authentication.startup.state: false # java.security.krb5.conf path java.security.krb5.conf.path: /opt/krb5.conf # login user from keytab username login.user.keytab.username: hdfs-mycluster@ESZ.COM # login user from keytab path login.user.keytab.path: /opt/hdfs.headless.keytab # kerberos expire time, the unit is hour kerberos.expire.time: 2 # resourcemanager port, the default value is 8088 if not specified resource.manager.httpaddress.port: 8088 # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty yarn.resourcemanager.ha.rm.ids: 192.168.xx.xx,192.168.xx.xx # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname yarn.application.status.address: http://ds1:%s/ws/v1/cluster/apps/%s # job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) yarn.job.history.status.address: http://ds1:19888/ws/v1/history/mapreduce/jobs/%s # datasource encryption enable datasource.encryption.enable: false # datasource encryption salt datasource.encryption.salt: '!@#$%^&*' # data quality option data-quality.jar.name: dolphinscheduler-data-quality-dev-SNAPSHOT.jar # Whether hive SQL is executed in the same session support.hive.oneSession: false # use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions sudo.enable: true # development state development.state: false # rpc port alert.rpc.port: 50052 # set path of conda.sh conda.path: /opt/anaconda3/etc/profile.d/conda.sh # Task resource limit state task.resource.limit.state: false # mlflow task plugin preset repository ml.mlflow.preset_repository: https://github.com/apache/dolphinscheduler-mlflow # mlflow task plugin preset repository version ml.mlflow.preset_repository_version: "main" # way to collect applicationId: log, aop appId.collect: log common: ## Configmap configmap: DOLPHINSCHEDULER_OPTS: "" DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" RESOURCE_UPLOAD_PATH: "/dolphinscheduler" # dolphinscheduler env HADOOP_HOME: "/opt/soft/hadoop" HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" SPARK_HOME: "/opt/soft/spark" PYTHON_HOME: "/usr/bin/python" JAVA_HOME: "/opt/java/openjdk" HIVE_HOME: "/opt/soft/hive" FLINK_HOME: "/opt/soft/flink" DATAX_HOME: "/opt/soft/datax" ## Shared storage persistence mounted into api, master and worker, such as Hadoop, Spark, Flink and DataX binary package sharedStoragePersistence: enabled: false mountPath: "/opt/soft" accessModes: - "ReadWriteMany" ## storageClassName must support the access mode: ReadWriteMany storageClassName: "-" storage: "20Gi" ## If RESOURCE_STORAGE_TYPE is HDFS and FS_DEFAULT_FS is file:///, fsFileResourcePersistence should be enabled for resource storage fsFileResourcePersistence: enabled: false accessModes: - "ReadWriteMany" ## storageClassName must support the access mode: ReadWriteMany storageClassName: "-" storage: "20Gi" master: ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. podManagementPolicy: "Parallel" ## Replicas is the desired number of replicas of the given Template. replicas: "3" ## You can use annotations to attach arbitrary non-identifying metadata to objects. ## Clients such as tools and libraries can retrieve this metadata. annotations: {} ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} ## NodeSelector is a selector which must be true for the pod to fit on a node. ## Selector which must match a node's labels for the pod to be scheduled on that node. ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] ## Compute Resources required by this container. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container resources: {} # resources: # limits: # memory: "8Gi" # cpu: "4" # requests: # memory: "2Gi" # cpu: "500m" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: enabled: true initialDelaySeconds: "30" periodSeconds: "30" timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes readinessProbe: enabled: true initialDelaySeconds: "30" periodSeconds: "30" timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. ## A claim in this list takes precedence over any volumes in the template, with the same name. persistentVolumeClaim: enabled: false accessModes: - "ReadWriteOnce" storageClassName: "-" storage: "20Gi" env: JAVA_OPTS: "-Xms1g -Xmx1g -Xmn512m" MASTER_EXEC_THREADS: "100" MASTER_EXEC_TASK_NUM: "20" MASTER_DISPATCH_TASK_NUM: "3" MASTER_HOST_SELECTOR: "LowerWeight" MASTER_HEARTBEAT_INTERVAL: "10s" MASTER_HEARTBEAT_ERROR_THRESHOLD: "5" MASTER_TASK_COMMIT_RETRYTIMES: "5" MASTER_TASK_COMMIT_INTERVAL: "1s" MASTER_STATE_WHEEL_INTERVAL: "5s" MASTER_MAX_CPU_LOAD_AVG: "1" MASTER_RESERVED_MEMORY: "0.3" MASTER_FAILOVER_INTERVAL: "10m" MASTER_KILL_APPLICATION_WHEN_HANDLE_FAILOVER: "true" worker: ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. podManagementPolicy: "Parallel" ## Replicas is the desired number of replicas of the given Template. replicas: "3" ## You can use annotations to attach arbitrary non-identifying metadata to objects. ## Clients such as tools and libraries can retrieve this metadata. annotations: {} ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} ## NodeSelector is a selector which must be true for the pod to fit on a node. ## Selector which must match a node's labels for the pod to be scheduled on that node. ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] ## Compute Resources required by this container. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container resources: {} # resources: # limits: # memory: "8Gi" # cpu: "4" # requests: # memory: "2Gi" # cpu: "500m" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: enabled: true initialDelaySeconds: "30" periodSeconds: "30" timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes readinessProbe: enabled: true initialDelaySeconds: "30" periodSeconds: "30" timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. ## A claim in this list takes precedence over any volumes in the template, with the same name. persistentVolumeClaim: enabled: false ## dolphinscheduler data volume dataPersistentVolume: enabled: false accessModes: - "ReadWriteOnce" storageClassName: "-" storage: "20Gi" ## dolphinscheduler logs volume logsPersistentVolume: enabled: false accessModes: - "ReadWriteOnce" storageClassName: "-" storage: "20Gi" env: WORKER_MAX_CPU_LOAD_AVG: "1" WORKER_RESERVED_MEMORY: "0.3" WORKER_EXEC_THREADS: "100" WORKER_HEARTBEAT_INTERVAL: "10s" WORKER_HEART_ERROR_THRESHOLD: "5" WORKER_HOST_WEIGHT: "100" keda: enabled: false namespaceLabels: { } # How often KEDA polls the DolphinScheduler DB to report new scale requests to the HPA pollingInterval: 5 # How many seconds KEDA will wait before scaling to zero. # Note that HPA has a separate cooldown period for scale-downs cooldownPeriod: 30 # Minimum number of workers created by keda minReplicaCount: 0 # Maximum number of workers created by keda maxReplicaCount: 3 # Specify HPA related options advanced: { } # horizontalPodAutoscalerConfig: # behavior: # scaleDown: # stabilizationWindowSeconds: 300 # policies: # - type: Percent # value: 100 # periodSeconds: 15 alert: ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. replicas: 1 ## The deployment strategy to use to replace existing pods with new ones. strategy: type: "RollingUpdate" rollingUpdate: maxSurge: "25%" maxUnavailable: "25%" ## You can use annotations to attach arbitrary non-identifying metadata to objects. ## Clients such as tools and libraries can retrieve this metadata. annotations: {} ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} ## NodeSelector is a selector which must be true for the pod to fit on a node. ## Selector which must match a node's labels for the pod to be scheduled on that node. ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] ## Compute Resources required by this container. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container resources: {} # resources: # limits: # memory: "2Gi" # cpu: "1" # requests: # memory: "1Gi" # cpu: "500m" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: enabled: true initialDelaySeconds: "30" periodSeconds: "30" timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes readinessProbe: enabled: true initialDelaySeconds: "30" periodSeconds: "30" timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. ## More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims persistentVolumeClaim: enabled: false accessModes: - "ReadWriteOnce" storageClassName: "-" storage: "20Gi" env: JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m" api: ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. replicas: "1" ## The deployment strategy to use to replace existing pods with new ones. strategy: type: "RollingUpdate" rollingUpdate: maxSurge: "25%" maxUnavailable: "25%" ## You can use annotations to attach arbitrary non-identifying metadata to objects. ## Clients such as tools and libraries can retrieve this metadata. annotations: {} ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} ## NodeSelector is a selector which must be true for the pod to fit on a node. ## Selector which must match a node's labels for the pod to be scheduled on that node. ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] ## Compute Resources required by this container. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container resources: {} # resources: # limits: # memory: "2Gi" # cpu: "1" # requests: # memory: "1Gi" # cpu: "500m" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: enabled: true initialDelaySeconds: "30" periodSeconds: "30" timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes readinessProbe: enabled: true initialDelaySeconds: "30" periodSeconds: "30" timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. ## More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims persistentVolumeClaim: enabled: false accessModes: - "ReadWriteOnce" storageClassName: "-" storage: "20Gi" service: ## type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer type: "ClusterIP" ## clusterIP is the IP address of the service and is usually assigned randomly by the master clusterIP: "" ## nodePort is the port on each node on which this api service is exposed when type=NodePort nodePort: "" ## pythonNodePort is the port on each node on which this python api service is exposed when type=NodePort pythonNodePort: "" ## externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service externalIPs: [] ## externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service, requires Type to be ExternalName externalName: "" ## loadBalancerIP when service.type is LoadBalancer. LoadBalancer will get created with the IP specified in this field loadBalancerIP: "" ## annotations may need to be set when service.type is LoadBalancer ## service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:EXAMPLE_CERT annotations: {} env: JAVA_OPTS: "-Xms512m -Xmx512m -Xmn256m" ingress: enabled: false host: "dolphinscheduler.org" path: "/dolphinscheduler" annotations: {} tls: enabled: false secretName: "dolphinscheduler-tls"