From fd4b59ad6c06cb8a558143e0ea16c6dd686b5c04 Mon Sep 17 00:00:00 2001 From: wenjun <861923274@qq.com> Date: Wed, 10 Mar 2021 08:55:04 +0800 Subject: [PATCH 1/4] [Improvement][Worker] Rename worker.weight to worker.host.weight (#4997) (#5014) --- .../DOLPHIN/1.3.3/configuration/dolphin-worker.xml | 4 ++-- docker/build/README.md | 2 +- docker/build/README_zh_CN.md | 2 +- .../conf/dolphinscheduler/worker.properties.tpl | 4 ++-- docker/build/startup-init-conf.sh | 2 +- docker/docker-swarm/docker-compose.yml | 2 +- docker/docker-swarm/docker-stack.yml | 2 +- docker/kubernetes/dolphinscheduler/README.md | 2 +- .../templates/configmap-dolphinscheduler-worker.yaml | 2 +- .../statefulset-dolphinscheduler-worker.yaml | 2 +- docker/kubernetes/dolphinscheduler/values.yaml | 2 +- .../server/worker/config/WorkerConfig.java | 12 ++++++------ .../server/worker/registry/WorkerRegistry.java | 2 +- .../src/main/resources/worker.properties | 4 ++-- 14 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml index e5a3adf9f6..aaa5463ad4 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml @@ -67,12 +67,12 @@ - worker.weigth + worker.host.weigth 100 int - worker weight + worker host weight \ No newline at end of file diff --git a/docker/build/README.md b/docker/build/README.md index 6f168a3423..69273f9dac 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -309,7 +309,7 @@ This environment variable sets port for `worker-server`. The default value is `1 This environment variable sets groups for `worker-server`. The default value is `default`. -**`WORKER_WEIGHT`** +**`WORKER_HOST_WEIGHT`** This environment variable sets weight for `worker-server`. The default value is `100`. diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index defb2578ff..96bfb80dd2 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -309,7 +309,7 @@ DolphinScheduler Docker 容器通过环境变量进行配置，缺省时将会 配置`worker-server`的分组，默认值 `default`。 -**`WORKER_WEIGHT`** +**`WORKER_HOST_WEIGHT`** 配置`worker-server`的权重，默认之`100`。 diff --git a/docker/build/conf/dolphinscheduler/worker.properties.tpl b/docker/build/conf/dolphinscheduler/worker.properties.tpl index cab729b6aa..ec0c4abb49 100644 --- a/docker/build/conf/dolphinscheduler/worker.properties.tpl +++ b/docker/build/conf/dolphinscheduler/worker.properties.tpl @@ -33,8 +33,8 @@ worker.listen.port=${WORKER_LISTEN_PORT} # default worker groups worker.groups=${WORKER_GROUPS} -# default worker weight -worker.weight=${WORKER_WEIGHT} +# default worker host weight +worker.host.weight=${WORKER_HOST_WEIGHT} # alert server listener host alert.listen.host=${ALERT_LISTEN_HOST} diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 3d0eb4a31b..95491268a9 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -84,7 +84,7 @@ export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"100"} export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.1"} export WORKER_LISTEN_PORT=${WORKER_LISTEN_PORT:-"1234"} export WORKER_GROUPS=${WORKER_GROUPS:-"default"} -export WORKER_WEIGHT=${WORKER_WEIGHT:-"100"} +export WORKER_HOST_WEIGHT=${WORKER_HOST_WEIGHT:-"100"} export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"} #============================================================================ diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index b4c97cbe88..e8601708eb 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -171,7 +171,7 @@ services: WORKER_MAX_CPULOAD_AVG: "100" WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUPS: "default" - WORKER_WEIGHT: "100" + WORKER_HOST_WEIGHT: "100" ALERT_LISTEN_HOST: dolphinscheduler-alert HADOOP_HOME: "/opt/soft/hadoop" HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index fddf279205..094890dfac 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -165,7 +165,7 @@ services: WORKER_MAX_CPULOAD_AVG: "100" WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUPS: "default" - WORKER_WEIGHT: "100" + WORKER_HOST_WEIGHT: "100" ALERT_LISTEN_HOST: dolphinscheduler-alert HADOOP_HOME: "/opt/soft/hadoop" HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md index 87b04ef69f..a6f474af62 100644 --- a/docker/kubernetes/dolphinscheduler/README.md +++ b/docker/kubernetes/dolphinscheduler/README.md @@ -173,7 +173,7 @@ The Configuration file is `values.yaml`, and the following tables lists the conf | `worker.configmap.WORKER_RESERVED_MEMORY` | Only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G | `0.1` | | `worker.configmap.WORKER_LISTEN_PORT` | Worker listen port | `1234` | | `worker.configmap.WORKER_GROUPS` | Worker groups | `default` | -| `worker.configmap.WORKER_WEIGHT` | Worker weight | `100` | +| `worker.configmap.WORKER_HOST_WEIGHT` | Worker host weight | `100` | | `worker.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `worker.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `worker.livenessProbe.periodSeconds` | How often to perform the probe | `30` | diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml index 2b6a36615c..61d208e8f8 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml @@ -31,5 +31,5 @@ data: WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }} WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }} WORKER_GROUPS: {{ .Values.worker.configmap.WORKER_GROUPS | quote }} - WORKER_WEIGHT: {{ .Values.worker.configmap.WORKER_WEIGHT | quote }} + WORKER_HOST_WEIGHT: {{ .Values.worker.configmap.WORKER_HOST_WEIGHT | quote }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 271aaa10c7..d616b5421c 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -111,7 +111,7 @@ spec: valueFrom: configMapKeyRef: name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_WEIGHT + key: WORKER_HOST_WEIGHT - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH valueFrom: configMapKeyRef: diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index a8902593f8..896ae772d0 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -203,7 +203,7 @@ worker: WORKER_RESERVED_MEMORY: "0.1" WORKER_LISTEN_PORT: "1234" WORKER_GROUPS: "default" - WORKER_WEIGHT: "100" + WORKER_HOST_WEIGHT: "100" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index a32d4c8ff3..228f6ab755 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -50,8 +50,8 @@ public class WorkerConfig { @Value("${worker.listen.port: 1234}") private int listenPort; - @Value("${worker.weight:100}") - private int weight; + @Value("${worker.host.weight:100}") + private int hostWeight; @Value("${alert.listen.host:localhost}") private String alertListenHost; @@ -115,12 +115,12 @@ public class WorkerConfig { this.workerMaxCpuloadAvg = workerMaxCpuloadAvg; } - public int getWeight() { - return weight; + public int getHostWeight() { + return hostWeight; } - public void setWeight(int weight) { - this.weight = weight; + public void setHostWeight(int weight) { + this.hostWeight = weight; } public String getAlertListenHost() { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java index b763497a04..87c6af6734 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java @@ -152,7 +152,7 @@ public class WorkerRegistry { String address = getLocalAddress(); String workerZkPathPrefix = this.zookeeperRegistryCenter.getWorkerPath(); - int weight = workerConfig.getWeight(); + int weight = workerConfig.getHostWeight(); long workerStartTime = System.currentTimeMillis(); for (String workGroup : this.workerGroups) { diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties index fd249e26bb..7fd11fe6c2 100644 --- a/dolphinscheduler-server/src/main/resources/worker.properties +++ b/dolphinscheduler-server/src/main/resources/worker.properties @@ -33,8 +33,8 @@ # default worker groups #worker.groups=default -# default worker weight -#worker.weight=100 +# default worker host weight +#worker.host.weight=100 # alert server listener host alert.listen.host=localhost From 28dfde7002afc04fccc86b70cfa12fafbefc465d Mon Sep 17 00:00:00 2001 From: dddyszy <913406953@qq.com> Date: Wed, 10 Mar 2021 09:21:34 +0800 Subject: [PATCH 2/4] =?UTF-8?q?[Fix-4862][Server]=20Kill=20yarn=20applicat?= =?UTF-8?q?ion=20command=20won't=20be=20execute=20whe=E2=80=A6=20(#4936)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Fix-4862][Server] Kill yarn application command won't be execute when kill error --- .../server/utils/ProcessUtils.java | 7 ++-- .../worker/processor/TaskKillProcessor.java | 33 +++++++++---------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index 5b191edabf..aca2afedab 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -60,7 +60,7 @@ public class ProcessUtils { /** * Expression of PID recognition in Windows scene */ - private static final Pattern WINDOWSATTERN = Pattern.compile("(\\d+)"); + private static final Pattern WINDOWSATTERN = Pattern.compile("\\w+\\((\\d+)\\)"); private static final String LOCAL_PROCESS_EXEC = "jdk.lang.Process.allowAmbiguousCommands"; @@ -391,12 +391,11 @@ public class ProcessUtils { OSUtils.exeCmd(cmd); - // find log and kill yarn job - killYarnJob(taskExecutionContext); - } catch (Exception e) { logger.error("kill task failed", e); } + // find log and kill yarn job + killYarnJob(taskExecutionContext); } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java index a3665b33e0..84109ccbc8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java @@ -111,10 +111,11 @@ public class TaskKillProcessor implements NettyRequestProcessor { * @return kill result */ private Pair> doKill(TaskKillRequestCommand killCommand) { + boolean processFlag = true; List appIds = Collections.emptyList(); + int taskInstanceId = killCommand.getTaskInstanceId(); + TaskExecutionContext taskExecutionContext = taskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); try { - int taskInstanceId = killCommand.getTaskInstanceId(); - TaskExecutionContext taskExecutionContext = taskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); Integer processId = taskExecutionContext.getProcessId(); if (processId.equals(0)) { workerManager.killTaskBeforeExecuteByInstanceId(taskInstanceId); @@ -128,18 +129,16 @@ public class TaskKillProcessor implements NettyRequestProcessor { logger.info("process id:{}, cmd:{}", taskExecutionContext.getProcessId(), cmd); OSUtils.exeCmd(cmd); - - // find log and kill yarn job - appIds = killYarnJob(Host.of(taskExecutionContext.getHost()).getIp(), - taskExecutionContext.getLogPath(), - taskExecutionContext.getExecutePath(), - taskExecutionContext.getTenantCode()); - - return Pair.of(true, appIds); } catch (Exception e) { + processFlag = false; logger.error("kill task error", e); } - return Pair.of(false, appIds); + // find log and kill yarn job + Pair> yarnResult = killYarnJob(Host.of(taskExecutionContext.getHost()).getIp(), + taskExecutionContext.getLogPath(), + taskExecutionContext.getExecutePath(), + taskExecutionContext.getTenantCode()); + return Pair.of(processFlag && yarnResult.getLeft(), yarnResult.getRight()); } /** @@ -170,26 +169,26 @@ public class TaskKillProcessor implements NettyRequestProcessor { * @param logPath logPath * @param executePath executePath * @param tenantCode tenantCode - * @return List appIds + * @return Pair> yarn kill result */ - private List killYarnJob(String host, String logPath, String executePath, String tenantCode) { + private Pair> killYarnJob(String host, String logPath, String executePath, String tenantCode) { LogClientService logClient = null; try { logClient = new LogClientService(); logger.info("view log host : {},logPath : {}", host, logPath); String log = logClient.viewLog(host, Constants.RPC_PORT, logPath); - + List appIds = Collections.emptyList(); if (StringUtils.isNotEmpty(log)) { - List appIds = LoggerUtils.getAppIds(log, logger); + appIds = LoggerUtils.getAppIds(log, logger); if (StringUtils.isEmpty(executePath)) { logger.error("task instance execute path is empty"); throw new RuntimeException("task instance execute path is empty"); } if (appIds.size() > 0) { ProcessUtils.cancelApplication(appIds, logger, tenantCode, executePath); - return appIds; } } + return Pair.of(true, appIds); } catch (Exception e) { logger.error("kill yarn job error", e); } finally { @@ -197,7 +196,7 @@ public class TaskKillProcessor implements NettyRequestProcessor { logClient.close(); } } - return Collections.EMPTY_LIST; + return Pair.of(false, Collections.emptyList()); } } From 3197b321a15deeee781031d21a607452f7259888 Mon Sep 17 00:00:00 2001 From: fredster33 <64927044+fredster33@users.noreply.github.com> Date: Tue, 9 Mar 2021 19:06:17 -0800 Subject: [PATCH 3/4] Grammar changes for README (#5021) --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0c933f642f..b52e00ef1f 100644 --- a/README.md +++ b/README.md @@ -76,23 +76,23 @@ dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release ## Thanks -DolphinScheduler is based on a lot of excellent open-source projects, such as google guava, guice, grpc, netty, ali bonecp, quartz, and many open-source projects of Apache and so on. +DolphinScheduler is based on a lot of excellent open-source projects, such as Google guava, guice, grpc, netty, ali bonecp, quartz, and many open-source projects of Apache and so on. We would like to express our deep gratitude to all the open-source projects used in Dolphin Scheduler. We hope that we are not only the beneficiaries of open-source, but also give back to the community. Besides, we hope everyone who have the same enthusiasm and passion for open source could join in and contribute to the open-source community! ## Get Help -1. Submit an [[issue](https://github.com/apache/incubator-dolphinscheduler/issues/new/choose)] -1. Subscribe to this mail list: https://dolphinscheduler.apache.org/en-us/community/development/subscribe.html, then email dev@dolphinscheduler.apache.org +1. Submit an [issue](https://github.com/apache/incubator-dolphinscheduler/issues/new/choose) +1. Subscribe to this mailing list: https://dolphinscheduler.apache.org/en-us/community/development/subscribe.html, then email dev@dolphinscheduler.apache.org ## Community -You are so much welcomed to communicate with the developers and users of Dolphin Scheduler freely. There are two ways to find them: -1. Join the slack channel by [this invitation link](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-mzqu52gi-rCggPkSHQ0DZYkwbTxO1Gw). -2. Follow the [twitter account of Dolphin Scheduler](https://twitter.com/dolphinschedule) and get the latest news just on time. +You are very welcome to communicate with the developers and users of Dolphin Scheduler. There are two ways to find them: +1. Join the Slack channel by [this invitation link](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-mzqu52gi-rCggPkSHQ0DZYkwbTxO1Gw). +2. Follow the [Twitter account of Dolphin Scheduler](https://twitter.com/dolphinschedule) and get the latest news on time. ## How to Contribute -The community welcomes everyone to participate in contributing, please refer to this website to find out more: [[How to contribute](https://dolphinscheduler.apache.org/en-us/community/development/contribute.html)] +The community welcomes everyone to contribute, please refer to this page to find out more: [How to contribute](https://dolphinscheduler.apache.org/en-us/community/development/contribute.html). ## License From 9a3ae4839641d23f32057715ab1e88619d8dfdf8 Mon Sep 17 00:00:00 2001 From: fredster33 <64927044+fredster33@users.noreply.github.com> Date: Tue, 9 Mar 2021 19:40:21 -0800 Subject: [PATCH 4/4] Update PULL_REQUEST_TEMPLATE.md (#5022) When commented out, even if the PR creator forgets to remove the tips, they will not appear in the PR when it is submitted. --- .github/PULL_REQUEST_TEMPLATE.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c150c845c3..61603fadc6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,19 +1,18 @@ -## *Tips* -- *Thanks very much for contributing to Apache DolphinScheduler.* -- *Please review https://dolphinscheduler.apache.org/en-us/community/development/pull-request.html before opening a pull request.* + -## What is the purpose of the pull request -*(For example: This pull request adds checkstyle plugin.)* +## Purpose of the pull request + + ## Brief change log -*(for example:)* + ## Verify this pull request -*(Please pick either of the following options)* + This pull request is code cleanup without any test coverage. @@ -25,8 +24,7 @@ This pull request is already covered by existing tests, such as *(please describ This change added tests and can be verified as follows: -*(example:)* - +