Merge remote-tracking branch 'origin/dev' into json_split_two

# Conflicts: # .github/actions/translate-on-issue # docker/build/conf/dolphinscheduler/registry.properties.tpl # docker/build/startup-init-conf.sh # docker/docker-swarm/config.env.sh # docker/kubernetes/dolphinscheduler/templates/_helpers.tpl # dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java # dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryCenter.java # dolphinscheduler-service/src/main/resources/registry.properties
3 years ago · fc203fb41c
8 changed files with 87 additions and 72 deletions
--- a/docker/build/conf/dolphinscheduler/registry.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/registry.properties.tpl
@ -15,9 +15,13 @@
 # limitations under the License.
 #

-registry.plugin.name=${REGISTRY_PLUGIN_NAME}
+#registry.plugin.dir config the Registry Plugin dir.
 registry.plugin.dir=${REGISTRY_PLUGIN_DIR}
-registry.plugin.binding=registry
+
+registry.plugin.name=${REGISTRY_PLUGIN_NAME}
 registry.servers=${REGISTRY_SERVERS}

+#maven.local.repository=/usr/local/localRepository

+#registry.plugin.binding config the Registry Plugin need be load when development and run in IDE
+#registry.plugin.binding=./dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml
--- a/docker/build/startup-init-conf.sh
+++ b/docker/build/startup-init-conf.sh
@ -37,7 +37,7 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
 #============================================================================
 # Registry
 #============================================================================
-export REGISTRY_PLUGIN_DIR=${REGISTRY_PLUGIN_DIR:-"lib/plugin/registry/zookeeper"}
+export REGISTRY_PLUGIN_DIR=${REGISTRY_PLUGIN_DIR:-"lib/plugin/registry"}
 export REGISTRY_PLUGIN_NAME=${REGISTRY_PLUGIN_NAME:-"zookeeper"}
 export REGISTRY_SERVERS=${REGISTRY_SERVERS:-"127.0.0.1:2181"}

--- a/docker/docker-swarm/config.env.sh
+++ b/docker/docker-swarm/config.env.sh
@ -37,10 +37,9 @@ DATABASE_PARAMS=characterEncoding=utf8
 # DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8

 #============================================================================
-# ZooKeeper
+# Registry
 #============================================================================
-
-REGISTRY_PLUGIN_DIR=lib/plugin/registry/zookeeper
+REGISTRY_PLUGIN_DIR=lib/plugin/registry
 REGISTRY_PLUGIN_NAME=zookeeper
 REGISTRY_SERVERS=dolphinscheduler-zookeeper:2181

--- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
+++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
@ -162,21 +162,27 @@ Create a database environment variables.
  {{- end }}
 {{- end -}}

-{{/* todo
-Create a rregistry environment variables.
+{{/*
+Create a registry environment variables.
 */}}
-{{- define "dolphinscheduler.zookeeper.env_vars" -}}
- name: ZOOKEEPER_QUORUM
+{{- define "dolphinscheduler.registry.env_vars" -}}
+- name: REGISTRY_PLUGIN_DIR
  {{- if .Values.zookeeper.enabled }}
-  value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
+  value: "lib/plugin/registry"
+  {{- else }}
+  value: {{ .Values.externalRegistry.registryPluginDir }}
+  {{- end }}
+- name: REGISTRY_PLUGIN_NAME
+  {{- if .Values.zookeeper.enabled }}
+  value: "zookeeper"
  {{- else }}
-  value: {{ .Values.externalZookeeper.zookeeperQuorum }}
+  value: {{ .Values.externalRegistry.registryPluginName }}
  {{- end }}
- name: ZOOKEEPER_ROOT
+- name: REGISTRY_SERVERS
  {{- if .Values.zookeeper.enabled }}
-  value: {{ .Values.zookeeper.zookeeperRoot }}
+  value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
  {{- else }}
-  value: {{ .Values.externalZookeeper.zookeeperRoot }}
+  value: {{ .Values.externalRegistry.registryServers }}
  {{- end }}
 {{- end -}}

--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/Result.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/Result.java
@ -72,6 +72,18 @@ public class Result<T> {
        return new Result<>(data);
    }

+    public boolean isSuccess() {
+        return this.isStatus(Status.SUCCESS);
+    }
+
+    public boolean isFailed() {
+        return !this.isSuccess();
+    }
+
+    public boolean isStatus(Status status) {
+        return this.code != null && this.code.equals(status.getCode());
+    }
+
    /**
     * Call this function if there is any error
     *
@ -120,10 +132,11 @@ public class Result<T> {

    @Override
    public String toString() {
-        return "Status{" +
-                "code='" + code + '\'' +
-                ", msg='" + msg + '\'' +
-                ", data=" + data +
-                '}';
+        return "Status{"
+                + "code='" + code
+                + '\'' + ", msg='"
+                + msg + '\''
+                + ", data=" + data
+                + '}';
    }
 }
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java
@ -73,28 +73,27 @@ public class ProcessDefinitionControllerTest {

    @Test
    public void testCreateProcessDefinition() throws Exception {
-        String json = "[{\"name\":\"\",\"pre_task_code\":0,\"pre_task_version\":0,\"post_task_code\":123456789,\"post_task_version\":1,"
-                + "\"condition_type\":0,\"condition_params\":{}},{\"name\":\"\",\"pre_task_code\":123456789,\"pre_task_version\":1,"
-                + "\"post_task_code\":123451234,\"post_task_version\":1,\"condition_type\":0,\"condition_params\":{}}]";
+        String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\""
+                + ":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\"
+                + "necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\""
+                + ",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},"
+                + "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}";
+        String locations = "{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}";

        String projectName = "test";
        String name = "dag_test";
        String description = "desc test";
-        String globalParams = "[]";
        String connects = "[]";
-        String locations = "[]";
-        int timeout = 0;
-        String tenantCode = "root";
        Map<String, Object> result = new HashMap<>();
        putMsg(result, Status.SUCCESS);
        result.put(Constants.DATA_LIST, 1);

-        Mockito.when(processDefinitionService.createProcessDefinition(user, projectName, name, description, globalParams,
-                connects, locations, timeout, tenantCode, json)).thenReturn(result);
+        Mockito.when(processDefinitionService.createProcessDefinition(user, projectName, name, json,
+                description, locations, connects)).thenReturn(result);

-        Result response = processDefinitionController.createProcessDefinition(user, projectName, name, description, globalParams,
-                connects, locations, timeout, tenantCode, json);
-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Result response = processDefinitionController.createProcessDefinition(user, projectName, name, json,
+                locations, connects, description);
+        Assert.assertTrue(response.isSuccess());
    }

    private void putMsg(Map<String, Object> result, Status status, Object... statusParams) {
@ -117,35 +116,34 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.verifyProcessDefinitionName(user, projectName, name)).thenReturn(result);

        Result response = processDefinitionController.verifyProcessDefinitionName(user, projectName, name);
-        Assert.assertEquals(Status.PROCESS_DEFINITION_NAME_EXIST.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response.isStatus(Status.PROCESS_DEFINITION_NAME_EXIST));

    }

    @Test
-    public void updateProcessDefinition() {
-        String json = "[{\"name\":\"\",\"pre_task_code\":0,\"pre_task_version\":0,\"post_task_code\":123456789,\"post_task_version\":1,"
-                + "\"condition_type\":0,\"condition_params\":{}},{\"name\":\"\",\"pre_task_code\":123456789,\"pre_task_version\":1,"
-                + "\"post_task_code\":123451234,\"post_task_version\":1,\"condition_type\":0,\"condition_params\":{}}]";
+    public void updateProcessDefinition() throws Exception {
+
+        String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\""
+                + ",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"}"
+                + ",\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\""
+                + ":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\""
+                + ":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}";
        String locations = "{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}";
        String projectName = "test";
        String name = "dag_test";
        String description = "desc test";
        String connects = "[]";
-        String globalParams = "[]";
-        int timeout = 0;
-        String tenantCode = "root";
-        long code = 123L;
+        int id = 1;
        Map<String, Object> result = new HashMap<>();
        putMsg(result, Status.SUCCESS);
        result.put("processDefinitionId", 1);

-        Mockito.when(processDefinitionService.updateProcessDefinition(user, projectName, name, code, description, globalParams,
-                connects, locations, timeout, tenantCode, json)).thenReturn(result);
+        Mockito.when(processDefinitionService.updateProcessDefinition(user, projectName, id, name, json,
+                description, locations, connects)).thenReturn(result);

-        Result response = processDefinitionController.updateProcessDefinition(user, projectName, name, code, description, globalParams,
-                connects, locations, timeout, tenantCode, json, ReleaseState.OFFLINE);
-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
-    }
+        Result response = processDefinitionController.updateProcessDefinition(user, projectName, name, id, json,
+                locations, connects, description,ReleaseState.OFFLINE);
+        Assert.assertTrue(response != null && response.isSuccess());    }

    @Test
    public void testReleaseProcessDefinition() throws Exception {
@ -156,7 +154,7 @@ public class ProcessDefinitionControllerTest {

        Mockito.when(processDefinitionService.releaseProcessDefinition(user, projectName, id, ReleaseState.OFFLINE)).thenReturn(result);
        Result response = processDefinitionController.releaseProcessDefinition(user, projectName, id, ReleaseState.OFFLINE);
-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    @Test
@ -189,7 +187,7 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.queryProcessDefinitionById(user, projectName, id)).thenReturn(result);
        Result response = processDefinitionController.queryProcessDefinitionById(user, projectName, id);

-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    @Test
@ -205,7 +203,7 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.batchCopyProcessDefinition(user, projectName, id, targetProjectId)).thenReturn(result);
        Result response = processDefinitionController.copyProcessDefinition(user, projectName, id, targetProjectId);

-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    @Test
@ -221,7 +219,7 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.batchMoveProcessDefinition(user, projectName, id, targetProjectId)).thenReturn(result);
        Result response = processDefinitionController.moveProcessDefinition(user, projectName, id, targetProjectId);

-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    @Test
@ -237,7 +235,7 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.queryProcessDefinitionList(user, projectName)).thenReturn(result);
        Result response = processDefinitionController.queryProcessDefinitionList(user, projectName);

-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    public List<ProcessDefinition> getDefinitionList() {
@ -292,7 +290,7 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.deleteProcessDefinitionById(user, projectName, id)).thenReturn(result);
        Result response = processDefinitionController.deleteProcessDefinitionById(user, projectName, id);

-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    @Test
@ -306,7 +304,7 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.getTaskNodeListByDefinitionCode(code)).thenReturn(result);
        Result response = processDefinitionController.getNodeListByDefinitionCode(user, projectName, code);

-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    @Test
@ -320,7 +318,7 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.getTaskNodeListByDefinitionCodeList(codeList)).thenReturn(result);
        Result response = processDefinitionController.getNodeListByDefinitionCodeList(user, projectName, codeList);

-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    @Test
@ -332,7 +330,7 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.queryProcessDefinitionAllByProjectId(projectId)).thenReturn(result);
        Result response = processDefinitionController.queryProcessDefinitionAllByProjectId(user, projectId);

-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    @Test
@ -346,7 +344,7 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.viewTree(processId, limit)).thenReturn(result);
        Result response = processDefinitionController.viewTree(user, projectName, processId, limit);

-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    @Test
@ -364,7 +362,7 @@ public class ProcessDefinitionControllerTest {
        Mockito.when(processDefinitionService.queryProcessDefinitionListPaging(user, projectName, searchVal, pageNo, pageSize, userId)).thenReturn(result);
        Result response = processDefinitionController.queryProcessDefinitionListPaging(user, projectName, pageNo, searchVal, userId, pageSize);

-        Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue());
+        Assert.assertTrue(response != null && response.isSuccess());
    }

    @Test
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryCenter.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryCenter.java
@ -124,12 +124,9 @@ public class RegistryCenter {
        DolphinPluginManagerConfig registryPluginManagerConfig = new DolphinPluginManagerConfig();
        registryPluginManagerConfig.setPlugins(PropertyUtils.getString(REGISTRY_PLUGIN_BINDING));
        if (StringUtils.isNotBlank(PropertyUtils.getString(REGISTRY_PLUGIN_DIR))) {
-            registryPluginManagerConfig.setPlugins(PropertyUtils.getString(REGISTRY_PLUGIN_DIR, REGISTRY_PLUGIN_PATH).trim());
+            registryPluginManagerConfig.setInstalledPluginsDir(PropertyUtils.getString(REGISTRY_PLUGIN_DIR, REGISTRY_PLUGIN_PATH).trim());
        }

-        if (StringUtils.isNotBlank(PropertyUtils.getString(MAVEN_LOCAL_REPOSITORY))) {
-            registryPluginManagerConfig.setMavenLocalRepository(PropertyUtils.getString(MAVEN_LOCAL_REPOSITORY).trim());
-        }
        if (StringUtils.isNotBlank(PropertyUtils.getString(MAVEN_LOCAL_REPOSITORY))) {
            registryPluginManagerConfig.setMavenLocalRepository(PropertyUtils.getString(MAVEN_LOCAL_REPOSITORY).trim());
        }
--- a/dolphinscheduler-service/src/main/resources/registry.properties
+++ b/dolphinscheduler-service/src/main/resources/registry.properties
@ -15,15 +15,13 @@
 # limitations under the License.
 #

-#registry.plugin.dir config the Alert Plugin dir . AlertServer while find and load the Alert Plugin Jar from this dir when deploy and start AlertServer on the server .
-#registry.plugin.dir=/Users/kris/workspace/incubator-dolphinscheduler/dolphinscheduler-dist/target/dolphinscheduler-dist-1.3.6-SNAPSHOT/lib/plugin/registry/zookeeper
-#registry.plugin.name=zookeeper
-#registry.plugin.binding=registry
-#registry.servers=127.0.0.1:2181
-#maven.local.repository=/Users/gaojun/Documents/jianguoyun/localRepository
+#registry.plugin.dir config the Registry Plugin dir.
+registry.plugin.dir=lib/plugin/registry

-#registry.plugin.binding config the Alert Plugin need be load when development and run in IDE
-#registry.plugin.binding=\
-#  ./dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml
+registry.plugin.name=zookeeper
+registry.servers=127.0.0.1:2181

+#maven.local.repository=/usr/local/localRepository

+#registry.plugin.binding config the Registry Plugin need be load when development and run in IDE
+#registry.plugin.binding=./dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml