From ce2853ad4c7b31b9a8427e8c46dcbd0d5bd0b775 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 19 Mar 2020 00:07:36 +0800 Subject: [PATCH] add master and worker properties and remove data quality config (#2211) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update logback * update log * refactor worker registry (#2107) * Refactor worker (#2115) * refactor worker registry * refactor master server * Modify workgroupid parameter name (#2105) * Delete worker group management page * Modify workgroupid parameter name * Refactor worker (#2121) * refactor worker registry * refactor master server * refactor MasterSchedulerService * cancelTaskInstance set TaskExecutionContext host,logPath,executePath (#2126) * 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * add license * modify javadoc error * TaskExecutionContext create modify * buildAckCommand taskInstanceId not set modify * java doc error modify * add comment * ExecutorManager interface add generic type * add TaskInstanceCacheManager receive Worker report result * TaskInstance setExecutePath * add TaskInstanceCacheManager to receive Worker Task result report * TaskInstanceCacheManager add remove method * add license * add dispatcht task method * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * 1，worker TaskPros use TaskExecutionContext replase 2，Master kill Task , KillTaskProcessor modify * worker remove db * ShellTask modify * master persistence processId and appIds * master persistence processId and appIds * master add kill task logic * master add kill task logic * master add kill task logic * javadoc error modify * remove chinese log * executeDirectly method add Override * remote module modify * TaskKillResponseProcessor command type modify * create buildKillCommand * host add host:port format * host add host:port format * TaskAckProcessor modify * TaskAckProcessor modify * task prioriry refator * remove ITaskQueue * task prioriry refator * remove ITaskQueue * TaskPriority refactor * remove logs * WorkerServer refactor * MasterSchedulerService modify * WorkerConfig listen port modify * modify master and worker listen port * cancelTaskInstance set TaskExecutionContext host,logPath,executePath * cancelTaskInstance set TaskExecutionContext host,logPath,executePath Co-authored-by: qiaozhanwei * not exist in openjdk,just delete * add master and worker properties * add master and worker properties * add master and worker properties Co-authored-by: Tboy Co-authored-by: break60 <790061044@qq.com> Co-authored-by: qiaozhanwei Co-authored-by: qiaozhanwei --- .gitignore | 1 + .../dolphinscheduler/dao/TaskRecordDao.java | 2 +- .../src/main/resources/application.properties | 9 ---- .../remote/command/Command.java | 2 - .../server/master/config/MasterConfig.java | 12 ++++-- .../server/worker/config/WorkerConfig.java | 19 ++++++--- .../src/main/resources/master.properties | 41 +++++++++++++++++++ .../src/main/resources/worker.properties | 34 +++++++++++++++ 8 files changed, 99 insertions(+), 21 deletions(-) create mode 100644 dolphinscheduler-server/src/main/resources/master.properties create mode 100644 dolphinscheduler-server/src/main/resources/worker.properties diff --git a/.gitignore b/.gitignore index 7cf1d4d7db..fbc1dc25ed 100644 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,4 @@ dolphinscheduler-ui/dist/lib/external/ dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue /dolphinscheduler-dao/src/main/resources/dao/data_source.properties +!/zookeeper_data/ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java index f7ba1054a5..e0c3761032 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java @@ -61,7 +61,7 @@ public class TaskRecordDao { * @return whether startup taskrecord */ public static boolean getTaskRecordFlag(){ - return conf.getBoolean(Constants.TASK_RECORD_FLAG); + return conf.getBoolean(Constants.TASK_RECORD_FLAG,false); } /** * create connection diff --git a/dolphinscheduler-dao/src/main/resources/application.properties b/dolphinscheduler-dao/src/main/resources/application.properties index 1bf79b71d8..b79c1edf60 100644 --- a/dolphinscheduler-dao/src/main/resources/application.properties +++ b/dolphinscheduler-dao/src/main/resources/application.properties @@ -71,12 +71,3 @@ spring.datasource.spring.datasource.filters=stat,wall,log4j spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 -# data quality analysis is not currently in use. please ignore the following configuration -# task record -task.record.flag=false -task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8 -task.record.datasource.username=xx -task.record.datasource.password=xx - -# Logger Config -#logging.level.org.apache.dolphinscheduler.dao=debug diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java index 86ba79c884..ed46e1ff51 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java @@ -16,8 +16,6 @@ */ package org.apache.dolphinscheduler.remote.command; -import com.sun.org.apache.regexp.internal.RE; - import java.io.Serializable; import java.util.concurrent.atomic.AtomicLong; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java index b81e458a62..5b4b5daef1 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java @@ -16,10 +16,13 @@ */ package org.apache.dolphinscheduler.server.master.config; +import org.apache.dolphinscheduler.common.Constants; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Component; @Component +@PropertySource(value = "master.properties") public class MasterConfig { @Value("${master.exec.threads:100}") @@ -37,10 +40,10 @@ public class MasterConfig { @Value("${master.task.commit.interval:1000}") private int masterTaskCommitInterval; - @Value("${master.max.cpuload.avg:100}") + @Value("${master.max.cpuload.avg:-1}") private double masterMaxCpuloadAvg; - @Value("${master.reserved.memory:0.1}") + @Value("${master.reserved.memory:0.3}") private double masterReservedMemory; @Value("${master.host.selector:lowerWeight}") @@ -106,6 +109,9 @@ public class MasterConfig { } public double getMasterMaxCpuloadAvg() { + if (masterMaxCpuloadAvg == -1){ + return Constants.DEFAULT_MASTER_CPU_LOAD; + } return masterMaxCpuloadAvg; } @@ -120,4 +126,4 @@ public class MasterConfig { public void setMasterReservedMemory(double masterReservedMemory) { this.masterReservedMemory = masterReservedMemory; } -} +} \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index d2fd92665d..7f4d93fdf8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -1,3 +1,4 @@ + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -16,25 +17,28 @@ */ package org.apache.dolphinscheduler.server.worker.config; +import org.apache.dolphinscheduler.common.Constants; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Component; @Component +@PropertySource(value = "worker.properties") public class WorkerConfig { - @Value("${worker.exec.threads: 100}") + @Value("${worker.exec.threads:100}") private int workerExecThreads; - @Value("${worker.heartbeat.interval: 10}") + @Value("${worker.heartbeat.interval:10}") private int workerHeartbeatInterval; - @Value("${worker.fetch.task.num: 3}") + @Value("${worker.fetch.task.num:3}") private int workerFetchTaskNum; - @Value("${worker.max.cpuload.avg:100}") + @Value("${worker.max.cpuload.avg:-1}") private int workerMaxCpuloadAvg; - @Value("${worker.reserved.memory:0.1}") + @Value("${worker.reserved.memory:0.5}") private double workerReservedMemory; @Value("${worker.group: default}") @@ -92,10 +96,13 @@ public class WorkerConfig { } public int getWorkerMaxCpuloadAvg() { + if (workerMaxCpuloadAvg == -1){ + return Constants.DEFAULT_WORKER_CPU_LOAD; + } return workerMaxCpuloadAvg; } public void setWorkerMaxCpuloadAvg(int workerMaxCpuloadAvg) { this.workerMaxCpuloadAvg = workerMaxCpuloadAvg; } -} +} \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/resources/master.properties b/dolphinscheduler-server/src/main/resources/master.properties new file mode 100644 index 0000000000..2f75aa50ad --- /dev/null +++ b/dolphinscheduler-server/src/main/resources/master.properties @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# master execute thread num +#master.exec.threads=100 + +# master execute task number in parallel +#master.exec.task.num=20 + +# master heartbeat interval +#master.heartbeat.interval=10 + +# master commit task retry times +#master.task.commit.retryTimes=5 + +# master commit task interval +#master.task.commit.interval=1000 + + +# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 +#master.max.cpuload.avg=100 + +# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. +#master.reserved.memory=0.3 + +# master listen port +#master.listen.port=5678 \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties new file mode 100644 index 0000000000..ca7c27860f --- /dev/null +++ b/dolphinscheduler-server/src/main/resources/worker.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# worker execute thread num +#worker.exec.threads=100 + +# worker heartbeat interval +#worker.heartbeat.interval=10 + +# submit the number of tasks at a time +#worker.fetch.task.num = 3 + +# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 +#worker.max.cpuload.avg=100 + +# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G. +#worker.reserved.memory=0.3 + +# worker listener port +#worker.listen.port: 1234 \ No newline at end of file