From 3627bf25eff1ca20d0c2e18e414a35167068d76b Mon Sep 17 00:00:00 2001 From: xiaojingXU <68894048+xxjingcd@users.noreply.github.com> Date: Mon, 6 Feb 2023 10:55:19 +0800 Subject: [PATCH] [Improvement-13472][Api] Removes the overhead of idle Quartz Threadpool of Api (#13475) * [fix-13472][Api] change Threadpool of Quartz Scheduler to ZeroSizeThreadPool * update doc about quartz threadpool configuration --- docs/docs/en/architecture/configuration.md | 48 ++++++++++++------- docs/docs/zh/architecture/configuration.md | 46 +++++++++++------- .../src/main/resources/application.yaml | 5 +- .../quartz/QuartzZeroSizeThreadPool.java | 35 ++++++++++++++ 4 files changed, 96 insertions(+), 38 deletions(-) create mode 100644 dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzZeroSizeThreadPool.java diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 1dd5b17338..80cf3060e8 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -320,23 +320,37 @@ This part describes quartz configs and configure them based on your practical si The default configuration is as follows: -|Parameters | Default value| -|--|--| -|spring.quartz.properties.org.quartz.threadPool.threadPriority | 5| -|spring.quartz.properties.org.quartz.jobStore.isClustered | true| -|spring.quartz.properties.org.quartz.jobStore.class | org.quartz.impl.jdbcjobstore.JobStoreTX| -|spring.quartz.properties.org.quartz.scheduler.instanceId | AUTO| -|spring.quartz.properties.org.quartz.jobStore.tablePrefix | QRTZ_| -|spring.quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock|true| -|spring.quartz.properties.org.quartz.scheduler.instanceName | DolphinScheduler| -|spring.quartz.properties.org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool| -|spring.quartz.properties.org.quartz.jobStore.useProperties | false| -|spring.quartz.properties.org.quartz.threadPool.makeThreadsDaemons | true| -|spring.quartz.properties.org.quartz.threadPool.threadCount | 25| -|spring.quartz.properties.org.quartz.jobStore.misfireThreshold | 60000| -|spring.quartz.properties.org.quartz.scheduler.makeSchedulerThreadDaemon | true| -|spring.quartz.properties.org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.PostgreSQLDelegate| -|spring.quartz.properties.org.quartz.jobStore.clusterCheckinInterval | 5000| +| Parameters | Default value | +|-------------------------------------------------------------------------|-------------------------------------------------| +| spring.quartz.properties.org.quartz.jobStore.isClustered | true | +| spring.quartz.properties.org.quartz.jobStore.class | org.quartz.impl.jdbcjobstore.JobStoreTX | +| spring.quartz.properties.org.quartz.scheduler.instanceId | AUTO | +| spring.quartz.properties.org.quartz.jobStore.tablePrefix | QRTZ_ | +| spring.quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock | true | +| spring.quartz.properties.org.quartz.scheduler.instanceName | DolphinScheduler | +| spring.quartz.properties.org.quartz.jobStore.useProperties | false | +| spring.quartz.properties.org.quartz.jobStore.misfireThreshold | 60000 | +| spring.quartz.properties.org.quartz.scheduler.makeSchedulerThreadDaemon | true | +| spring.quartz.properties.org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.PostgreSQLDelegate | +| spring.quartz.properties.org.quartz.jobStore.clusterCheckinInterval | 5000 | + +The above configuration items is the same in *Master Server* and *Api Server*, but their *Quartz Scheduler* threadpool configuration is different. + +The default quartz threadpool configuration in *Master Server* is as follows: + +| Parameters | Default value | +|-------------------------------------------------------------------|-----------------------------------| +| spring.quartz.properties.org.quartz.threadPool.makeThreadsDaemons | true | +| spring.quartz.properties.org.quartz.threadPool.threadCount | 25 | +| spring.quartz.properties.org.quartz.threadPool.threadPriority | 5 | +| spring.quartz.properties.org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool | + +Since *Api Server* will not start *Quartz Scheduler* instance, as a client only, therefore it's threadpool is configured as `QuartzZeroSizeThreadPool` which has zero thread; +The default configuration is as follows: + +| Parameters | Default value | +|------------------------------------------------------|-----------------------------------------------------------------------| +| spring.quartz.properties.org.quartz.threadPool.class | org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool | ### dolphinscheduler_env.sh [load environment variables configs] diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 98b3dce7c4..105b433b7c 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -313,23 +313,35 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId 默认配置如下： -| 参数 | 默认值 | -|--|--| -|spring.quartz.properties.org.quartz.threadPool.threadPriority | 5| -|spring.quartz.properties.org.quartz.jobStore.isClustered | true| -|spring.quartz.properties.org.quartz.jobStore.class | org.quartz.impl.jdbcjobstore.JobStoreTX| -|spring.quartz.properties.org.quartz.scheduler.instanceId | AUTO| -|spring.quartz.properties.org.quartz.jobStore.tablePrefix | QRTZ_| -|spring.quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock|true| -|spring.quartz.properties.org.quartz.scheduler.instanceName | DolphinScheduler| -|spring.quartz.properties.org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool| -|spring.quartz.properties.org.quartz.jobStore.useProperties | false| -|spring.quartz.properties.org.quartz.threadPool.makeThreadsDaemons | true| -|spring.quartz.properties.org.quartz.threadPool.threadCount | 25| -|spring.quartz.properties.org.quartz.jobStore.misfireThreshold | 60000| -|spring.quartz.properties.org.quartz.scheduler.makeSchedulerThreadDaemon | true| -|spring.quartz.properties.org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.PostgreSQLDelegate| -|spring.quartz.properties.org.quartz.jobStore.clusterCheckinInterval | 5000| +| 参数 | 默认值 | +|-------------------------------------------------------------------------|-------------------------------------------------| +| spring.quartz.properties.org.quartz.jobStore.isClustered | true | +| spring.quartz.properties.org.quartz.jobStore.class | org.quartz.impl.jdbcjobstore.JobStoreTX | +| spring.quartz.properties.org.quartz.scheduler.instanceId | AUTO | +| spring.quartz.properties.org.quartz.jobStore.tablePrefix | QRTZ_ | +| spring.quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock | true | +| spring.quartz.properties.org.quartz.scheduler.instanceName | DolphinScheduler | +| spring.quartz.properties.org.quartz.jobStore.useProperties | false | +| spring.quartz.properties.org.quartz.jobStore.misfireThreshold | 60000 | +| spring.quartz.properties.org.quartz.scheduler.makeSchedulerThreadDaemon | true | +| spring.quartz.properties.org.quartz.jobStore.driverDelegateClass | org.quartz.impl.jdbcjobstore.PostgreSQLDelegate | +| spring.quartz.properties.org.quartz.jobStore.clusterCheckinInterval | 5000 | + +上述配置项在*Master Server* 和 *Api Server*是相同的，但他们的Quartz线程池配置部分却是不一样的。 +*Master Server* 的Quartz线程池默认配置如下： + +| Parameters | Default value | +|-------------------------------------------------------------------|-----------------------------------| +| spring.quartz.properties.org.quartz.threadPool.makeThreadsDaemons | true | +| spring.quartz.properties.org.quartz.threadPool.threadCount | 25 | +| spring.quartz.properties.org.quartz.threadPool.threadPriority | 5 | +| spring.quartz.properties.org.quartz.threadPool.class | org.quartz.simpl.SimpleThreadPool | + +因为*Api Server*不会启动*Quartz Scheduler*实例，只会作为Scheduler客户端使用，因此它的Quartz线程池将会使用`QuartzZeroSizeThreadPool`。`QuartzZeroSizeThreadPool`不会启动任何线程。具体的默认配置如下： + +| Parameters | Default value | +|------------------------------------------------------|-----------------------------------------------------------------------| +| spring.quartz.properties.org.quartz.threadPool.class | org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool | ## dolphinscheduler_env.sh [环境变量配置] diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 676f471d08..56a6db7bf5 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -61,17 +61,14 @@ spring: jdbc: initialize-schema: never properties: - org.quartz.threadPool.threadPriority: 5 org.quartz.jobStore.isClustered: true org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore org.quartz.scheduler.instanceId: AUTO org.quartz.jobStore.tablePrefix: QRTZ_ org.quartz.jobStore.acquireTriggersWithinLock: true org.quartz.scheduler.instanceName: DolphinScheduler - org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool + org.quartz.threadPool.class: org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool org.quartz.jobStore.useProperties: false - org.quartz.threadPool.makeThreadsDaemons: true - org.quartz.threadPool.threadCount: 25 org.quartz.jobStore.misfireThreshold: 60000 org.quartz.scheduler.makeSchedulerThreadDaemon: true org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzZeroSizeThreadPool.java b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzZeroSizeThreadPool.java new file mode 100644 index 0000000000..8f38f2d0f9 --- /dev/null +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzZeroSizeThreadPool.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.scheduler.quartz; + +import org.quartz.simpl.ZeroSizeThreadPool; + +public class QuartzZeroSizeThreadPool extends ZeroSizeThreadPool { + + /** + * fix spring bug : add getter、setter method for threadCount field + * @param count never use + */ + public void setThreadCount(int count) { + // do nothing + } + + public int getThreadCount() { + return -1; + } +}