From eac00ca008475b124ab35c5624454e1958a0833e Mon Sep 17 00:00:00 2001 From: zhanglong Date: Tue, 31 Mar 2020 16:52:49 +0800 Subject: [PATCH 1/6] Remove invalid code --- .../dolphinscheduler/api/controller/ResourcesController.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 40effb641d..1d83fcea2b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -154,8 +154,7 @@ public class ResourcesController extends BaseController{ @ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = true, dataType ="Int", example = "100"), @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType ="ResourceType"), @ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType ="String"), - @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType ="String"), - @ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true,dataType = "MultipartFile") + @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType ="String") }) @PostMapping(value = "/update") public Result updateResource(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, From d6602f21b51dd78e4cf9425b65bc810e4458a97b Mon Sep 17 00:00:00 2001 From: zhanglong Date: Wed, 8 Apr 2020 17:48:11 +0800 Subject: [PATCH 2/6] fix document errors --- .../dolphinscheduler/api/controller/SchedulerController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java index 96038dcf8c..d246a0f73a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java @@ -76,7 +76,7 @@ public class SchedulerController extends BaseController { @ApiOperation(value = "createSchedule", notes= "CREATE_SCHEDULE_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "processDefinitionId", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100"), - @ApiImplicitParam(name = "schedule", value = "SCHEDULE", dataType = "Int", example = "100"), + @ApiImplicitParam(name = "schedule", value = "SCHEDULE", dataType = "String", example = "{'startTime':'2019-06-10 00:00:00','endTime':'2019-06-13 00:00:00','crontab':'0 0 3/6 * * ? *'}"), @ApiImplicitParam(name = "warningType", value = "WARNING_TYPE", type ="WarningType"), @ApiImplicitParam(name = "warningGroupId", value = "WARNING_GROUP_ID", dataType = "Int", example = "100"), @ApiImplicitParam(name = "failureStrategy", value = "FAILURE_STRATEGY", type ="FailureStrategy"), @@ -132,7 +132,7 @@ public class SchedulerController extends BaseController { @ApiOperation(value = "updateSchedule", notes= "UPDATE_SCHEDULE_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "id", value = "SCHEDULE_ID", required = true, dataType = "Int", example = "100"), - @ApiImplicitParam(name = "schedule", value = "SCHEDULE", dataType = "Int", example = "100"), + @ApiImplicitParam(name = "schedule", value = "SCHEDULE", dataType = "String", example = "{'startTime':'2019-06-10 00:00:00','endTime':'2019-06-13 00:00:00','crontab':'0 0 3/6 * * ? *'}"), @ApiImplicitParam(name = "warningType", value = "WARNING_TYPE", type ="WarningType"), @ApiImplicitParam(name = "warningGroupId", value = "WARNING_GROUP_ID", dataType = "Int", example = "100"), @ApiImplicitParam(name = "failureStrategy", value = "FAILURE_STRATEGY", type ="FailureStrategy"), From 772a8e2b3750ea7a0a615fb4a0a775fbd5af8fb4 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Tue, 19 May 2020 10:18:21 +0800 Subject: [PATCH 3/6] add retryCallSilent to RetryerUtils making it easy to use (#2729) * add retryCallSilent to RetryerUtils making it easy to use * add more detail to java doc of retryCallSilent --- .../common/utils/RetryerUtils.java | 30 +++++ .../common/utils/RetryerUtilsTest.java | 109 ++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java index e6f4c502de..a3a935831f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java @@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.common.utils; import com.github.rholder.retry.*; import org.apache.dolphinscheduler.common.Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -28,6 +30,7 @@ import java.util.concurrent.TimeUnit; * The Retryer util. */ public class RetryerUtils { + private static final Logger logger = LoggerFactory.getLogger(RetryerUtils.class); private static Retryer defaultRetryerResultCheck; private static Retryer defaultRetryerResultNoCheck; @@ -105,4 +108,31 @@ public class RetryerUtils { public static Boolean retryCall(final Callable callable) throws ExecutionException, RetryException { return retryCall(callable, true); } + + /** + * Retry call silent without exceptions thrown + * + * @param callable the callable + * @param checkResult whether check result + * @return if no exceptions ,it's result returned by callable ,else always false + */ + public static boolean retryCallSilent(final Callable callable, boolean checkResult) { + boolean result = false; + try { + result = getDefaultRetryer(checkResult).call(callable); + } catch (ExecutionException | RetryException e) { + logger.warn("Retry call {} failed {}", callable, e.getMessage(), e); + } + return result; + } + + /** + * Retry call silent without exceptions thrown + * + * @param callable the callable + * @return if no exceptions ,it's result returned by callable ,else always false + */ + public static boolean retryCallSilent(final Callable callable) { + return retryCallSilent(callable, true); + } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java index 7841e46585..19b7853de3 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java @@ -213,4 +213,113 @@ public class RetryerUtilsTest { testRetryExceptionWithPara(true); testRetryExceptionWithPara(false); } + + @Test + public void testRetrySilent() { + try { + for (int execTarget = 1; execTarget <= 3; execTarget++) { + int finalExecTarget = execTarget; + int[] execTime = {0}; + boolean result = RetryerUtils.retryCallSilent(() -> { + execTime[0]++; + return execTime[0] == finalExecTarget; + }); + Assert.assertEquals(finalExecTarget, execTime[0]); + Assert.assertTrue(result); + } + } catch (Exception e) { + Assert.fail("Unexpected exception " + e.getMessage()); + } + int[] execTime = {0}; + try { + boolean result = RetryerUtils.retryCallSilent(() -> { + execTime[0]++; + return execTime[0] == 4; + }); + Assert.assertFalse(result); + } catch (Exception e) { + Assert.fail("Unexpected exception " + e.getMessage()); + } + } + + @Test + public void testRetrySilentWithPara() { + try { + for (int execTarget = 1; execTarget <= 3; execTarget++) { + int finalExecTarget = execTarget; + int[] execTime = {0}; + boolean result = RetryerUtils.retryCallSilent(() -> { + execTime[0]++; + return execTime[0] == finalExecTarget; + }, true); + Assert.assertEquals(finalExecTarget, execTime[0]); + Assert.assertTrue(result); + } + } catch (Exception e) { + Assert.fail("Unexpected exception " + e.getMessage()); + } + int[] execTime = {0}; + try { + boolean result = RetryerUtils.retryCallSilent(() -> { + execTime[0]++; + return execTime[0] == 4; + }, true); + Assert.assertFalse(result); + } catch (Exception e) { + Assert.fail("Unexpected exception " + e.getMessage()); + } + } + @Test + public void testRetrySilentNoCheckResult(){ + try { + for (int execTarget = 1; execTarget <= 5; execTarget++) { + int[] execTime = {0}; + boolean result = RetryerUtils.retryCallSilent(() -> { + execTime[0]++; + return execTime[0] > 1; + }, false); + Assert.assertEquals(1, execTime[0]); + Assert.assertFalse(result); + } + } catch (Exception e) { + Assert.fail("Unexpected exception " + e.getMessage()); + } + } + private void testRetrySilentExceptionWithPara(boolean checkResult) { + try { + for (int execTarget = 1; execTarget <= 3; execTarget++) { + int finalExecTarget = execTarget; + int[] execTime = {0}; + boolean result = RetryerUtils.retryCallSilent(() -> { + execTime[0]++; + if (execTime[0] != finalExecTarget) { + throw new IllegalArgumentException(String.valueOf(execTime[0])); + } + return true; + }, checkResult); + Assert.assertEquals(finalExecTarget, execTime[0]); + Assert.assertTrue(result); + } + } catch (Exception e) { + Assert.fail("Unexpected exception " + e.getMessage()); + } + int[] execTime = {0}; + try { + boolean result = RetryerUtils.retryCallSilent(() -> { + execTime[0]++; + if (execTime[0] != 4) { + throw new IllegalArgumentException(String.valueOf(execTime[0])); + } + return true; + }, checkResult); + Assert.assertFalse(result); + } catch (Exception e) { + Assert.fail("Unexpected exception " + e.getMessage()); + } + } + @Test + public void testRetrySilentException() { + testRetrySilentExceptionWithPara(true); + testRetrySilentExceptionWithPara(false); + } } From 1a78c1150c89d3bbea9029bcc9b2489b6460b278 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Tue, 19 May 2020 19:49:39 +0800 Subject: [PATCH 4/6] simply NettyExecutorManager.execute logic making it readable (#2744) * simply NettyExecutorManager.execute logic making it readable * fix NPE * remove unused import Co-authored-by: dailidong --- .../executor/NettyExecutorManager.java | 54 +++++++------------ 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java index 7ded3b0056..6fc3f45ec4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.dispatch.executor; -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -36,10 +35,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; +import java.util.*; /** * netty executor manager @@ -87,17 +83,11 @@ public class NettyExecutorManager extends AbstractExecutorManager{ */ @Override public Boolean execute(ExecutionContext context) throws ExecuteException { - - /** - * all nodes - */ - Set allNodes = getAllNodes(context); - - /** - * fail nodes - */ - Set failNodeSet = new HashSet<>(); - + LinkedList allNodes = new LinkedList<>(); + Set nodes = getAllNodes(context); + if (nodes != null) { + allNodes.addAll(nodes); + } /** * build command accord executeContext */ @@ -106,31 +96,27 @@ public class NettyExecutorManager extends AbstractExecutorManager{ /** * execute task host */ - Host host = context.getHost(); + String startHostAddress = context.getHost().getAddress(); + // remove start host address and add it to head + allNodes.remove(startHostAddress); + allNodes.addFirst(startHostAddress); + boolean success = false; - while (!success) { + for (String address : allNodes) { try { - doExecute(host,command); + Host host = Host.of(address); + doExecute(host, command); success = true; context.setHost(host); + break; } catch (ExecuteException ex) { - logger.error(String.format("execute command : %s error", command), ex); - try { - failNodeSet.add(host.getAddress()); - Set tmpAllIps = new HashSet<>(allNodes); - Collection remained = CollectionUtils.subtract(tmpAllIps, failNodeSet); - if (remained != null && remained.size() > 0) { - host = Host.of(remained.iterator().next()); - logger.error("retry execute command : {} host : {}", command, host); - } else { - throw new ExecuteException("fail after try all nodes"); - } - } catch (Throwable t) { - throw new ExecuteException("fail after try all nodes"); - } + logger.error("retry execute command : {} host : {}", command, address); } } - + if (!success) { + throw new ExecuteException("fail after try all nodes"); + } + return success; } From 20007264b1e1ab634be2b2e0e17ef10d57163a8b Mon Sep 17 00:00:00 2001 From: BoYiZhang Date: Thu, 21 May 2020 09:45:50 +0800 Subject: [PATCH 5/6] Sync file name --- .../src/js/conf/home/pages/dag/_source/dag.scss | 6 +++--- ...toobar_CONDITIONS.png => toolbar_CONDITIONS.png} | Bin .../dag/img/{toobar_FLINK.png => toolbar_FLINK.png} | Bin .../dag/img/{toobar_HTTP.png => toolbar_HTTP.png} | Bin 4 files changed, 3 insertions(+), 3 deletions(-) rename dolphinscheduler-ui/src/js/conf/home/pages/dag/img/{toobar_CONDITIONS.png => toolbar_CONDITIONS.png} (100%) rename dolphinscheduler-ui/src/js/conf/home/pages/dag/img/{toobar_FLINK.png => toolbar_FLINK.png} (100%) rename dolphinscheduler-ui/src/js/conf/home/pages/dag/img/{toobar_HTTP.png => toolbar_HTTP.png} (100%) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index 886ee692bf..5b171bb4ca 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -87,7 +87,7 @@ background: url("../img/toolbar_SPARK.png") no-repeat 50% 50%; } .icos-FLINK { - background: url("../img/toobar_FLINK.png") no-repeat 50% 50%; + background: url("../img/toolbar_FLINK.png") no-repeat 50% 50%; } .icos-MR { background: url("../img/toolbar_MR.png") no-repeat 50% 50%; @@ -99,7 +99,7 @@ background: url("../img/toolbar_DEPENDENT.png") no-repeat 50% 50%; } .icos-HTTP { - background: url("../img/toobar_HTTP.png") no-repeat 50% 50%; + background: url("../img/toolbar_HTTP.png") no-repeat 50% 50%; } .icos-DATAX { background: url("../img/toolbar_DATAX.png") no-repeat 50% 50%; @@ -108,7 +108,7 @@ background: url("../img/toolbar_SQOOP.png") no-repeat 50% 50%; } .icos-CONDITIONS { - background: url("../img/toobar_CONDITIONS.png") no-repeat 50% 50%; + background: url("../img/toolbar_CONDITIONS.png") no-repeat 50% 50%; } .toolbar { width: 60px; diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toobar_CONDITIONS.png b/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_CONDITIONS.png similarity index 100% rename from dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toobar_CONDITIONS.png rename to dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_CONDITIONS.png diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toobar_FLINK.png b/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_FLINK.png similarity index 100% rename from dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toobar_FLINK.png rename to dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_FLINK.png diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toobar_HTTP.png b/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_HTTP.png similarity index 100% rename from dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toobar_HTTP.png rename to dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_HTTP.png From 714ad5d4451c6e99befb045b320193443c3d6b16 Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Thu, 21 May 2020 10:22:47 +0800 Subject: [PATCH 6/6] update e2e chrome version (#2778) * add LoginTest license * Delete useless packages * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * Update worker_group_id to worker_group in init.sql * Update worker_group_id to worker_group in init.sql * Update worker_group_id to worker_group * Increase dataX environment variable, sslTrust default value * modify dataX environment variable * update e2e chrome version Co-authored-by: chenxingchun <438044805@qq.com> --- .github/workflows/ci_e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 58069a7259..82c81ef4e1 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -58,7 +58,7 @@ jobs: wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb sudo dpkg -i google-chrome*.deb sudo apt-get install -f -y - wget -N https://chromedriver.storage.googleapis.com/80.0.3987.106/chromedriver_linux64.zip + wget -N https://chromedriver.storage.googleapis.com/83.0.4103.39/chromedriver_linux64.zip unzip chromedriver_linux64.zip sudo mv -f chromedriver /usr/local/share/chromedriver sudo ln -s /usr/local/share/chromedriver /usr/local/bin/chromedriver