diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index f7f6e1cf57..0246aaf80e 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -64,7 +64,7 @@ jobs: - name: Compile run: | export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx5g' - mvn test -B -Dmaven.test.skip=false + mvn clean verify -B -Dmaven.test.skip=false - name: Upload coverage report to codecov run: | CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) diff --git a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml index 7a8d6ac8be..0fa1a8b5c9 100644 --- a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml @@ -49,8 +49,8 @@ spec: {{- end }} {{- if .Values.ingress.tls.enabled }} tls: - hosts: + - hosts: - {{ .Values.ingress.host }} - secretName: {{ .Values.ingress.tls.secretName }} + secretName: {{ .Values.ingress.tls.secretName }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/pom.xml index 84b39b2d87..cb2d7fba13 100644 --- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/pom.xml +++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/pom.xml @@ -68,6 +68,13 @@ jar test + + + org.jacoco + org.jacoco.agent + runtime + test + diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml index 492a621da2..74dedf4e0f 100644 --- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml +++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml @@ -118,6 +118,13 @@ + + + org.jacoco + org.jacoco.agent + runtime + test + diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml index 8155435764..ef47874c36 100644 --- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml +++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml @@ -68,6 +68,13 @@ jar test + + + org.jacoco + org.jacoco.agent + runtime + test + diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/pom.xml index aff9388182..83c4d41f7f 100644 --- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/pom.xml +++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/pom.xml @@ -62,6 +62,13 @@ jar test + + + org.jacoco + org.jacoco.agent + runtime + test + diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/pom.xml index 0088cc85fd..eb64e25787 100644 --- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/pom.xml +++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/pom.xml @@ -64,6 +64,13 @@ jar test + + + org.jacoco + org.jacoco.agent + runtime + test + diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-slack/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-slack/pom.xml index 5fe7f77680..3b1d8067e6 100644 --- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-slack/pom.xml +++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-slack/pom.xml @@ -69,6 +69,13 @@ jar test + + + org.jacoco + org.jacoco.agent + runtime + test + diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/pom.xml index 4b94f18077..fccf0e28da 100644 --- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/pom.xml +++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/pom.xml @@ -64,6 +64,13 @@ test + + org.jacoco + org.jacoco.agent + runtime + test + + diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index 0007da1276..cf586c38d0 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -108,6 +108,13 @@ + + + org.jacoco + org.jacoco.agent + runtime + test + diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index e4db57ce43..67d7d090c7 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -247,5 +247,12 @@ + + org.jacoco + org.jacoco.agent + runtime + test + + diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParam.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParam.java deleted file mode 100644 index 8572d7b482..0000000000 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParam.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.utils.exportprocess; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.dolphinscheduler.common.enums.TaskType; -import org.apache.dolphinscheduler.dao.entity.DataSource; -import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.List; - -/** - * task node add datasource param strategy - */ -@Service -public class DataSourceParam implements ProcessAddTaskParam, InitializingBean { - - private static final String PARAMS = "params"; - @Autowired - private DataSourceMapper dataSourceMapper; - - /** - * add datasource params - * @param taskNode task node json object - * @return task node json object - */ - @Override - public JsonNode addExportSpecialParam(JsonNode taskNode) { - // add sqlParameters - ObjectNode sqlParameters = (ObjectNode) taskNode.path(PARAMS); - DataSource dataSource = dataSourceMapper.selectById(sqlParameters.get("datasource").asInt()); - if (null != dataSource) { - sqlParameters.put("datasourceName", dataSource.getName()); - } - ((ObjectNode)taskNode).set(PARAMS, sqlParameters); - - return taskNode; - } - - /** - * import process add datasource params - * @param taskNode task node json object - * @return task node json object - */ - @Override - public JsonNode addImportSpecialParam(JsonNode taskNode) { - ObjectNode sqlParameters = (ObjectNode) taskNode.path(PARAMS); - List dataSources = dataSourceMapper.queryDataSourceByName(sqlParameters.path("datasourceName").asText()); - if (!dataSources.isEmpty()) { - DataSource dataSource = dataSources.get(0); - sqlParameters.put("datasource", dataSource.getId()); - } - ((ObjectNode)taskNode).set(PARAMS, sqlParameters); - return taskNode; - } - - - /** - * put datasource strategy - */ - @Override - public void afterPropertiesSet() { - TaskNodeParamFactory.register(TaskType.SQL.getDesc(), this); - TaskNodeParamFactory.register(TaskType.PROCEDURE.getDesc(), this); - } -} \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java deleted file mode 100644 index 29746f8f2a..0000000000 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.utils.exportprocess; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.dolphinscheduler.common.enums.TaskType; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; -import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; -import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -/** - * task node add dependent param strategy - */ -@Service -public class DependentParam implements ProcessAddTaskParam, InitializingBean { - - private static final String DEPENDENCE = "dependence"; - - @Autowired - ProcessDefinitionMapper processDefineMapper; - - @Autowired - ProjectMapper projectMapper; - - /** - * add dependent param - * @param taskNode task node json object - * @return task node json object - */ - @Override - public JsonNode addExportSpecialParam(JsonNode taskNode) { - // add dependent param - ObjectNode dependentParameters = JSONUtils.parseObject(taskNode.path(DEPENDENCE).asText()); - - if (null != dependentParameters) { - ArrayNode dependTaskList = (ArrayNode) dependentParameters.get("dependTaskList"); - for (int j = 0; j < dependTaskList.size(); j++) { - JsonNode dependentTaskModel = dependTaskList.path(j); - ArrayNode dependItemList = (ArrayNode) dependentTaskModel.get("dependItemList"); - for (int k = 0; k < dependItemList.size(); k++) { - ObjectNode dependentItem = (ObjectNode) dependItemList.path(k); - int definitionId = dependentItem.path("definitionId").asInt(); - ProcessDefinition definition = processDefineMapper.queryByDefineId(definitionId); - if (null != definition) { - dependentItem.put("projectName", definition.getProjectName()); - dependentItem.put("definitionName", definition.getName()); - } - } - } - ((ObjectNode)taskNode).set(DEPENDENCE, dependentParameters); - } - - return taskNode; - } - - /** - * import process add dependent param - * @param taskNode task node json object - * @return - */ - @Override - public JsonNode addImportSpecialParam(JsonNode taskNode) { - ObjectNode dependentParameters = JSONUtils.parseObject(taskNode.path(DEPENDENCE).asText()); - if(dependentParameters != null){ - ArrayNode dependTaskList = (ArrayNode) dependentParameters.path("dependTaskList"); - for (int h = 0; h < dependTaskList.size(); h++) { - ObjectNode dependentTaskModel = (ObjectNode) dependTaskList.path(h); - ArrayNode dependItemList = (ArrayNode) dependentTaskModel.get("dependItemList"); - for (int k = 0; k < dependItemList.size(); k++) { - ObjectNode dependentItem = (ObjectNode) dependItemList.path(k); - Project dependentItemProject = projectMapper.queryByName(dependentItem.path("projectName").asText()); - if(dependentItemProject != null){ - ProcessDefinition definition = processDefineMapper.queryByDefineName(dependentItemProject.getCode(),dependentItem.path("definitionName").asText()); - if(definition != null){ - dependentItem.put("projectId",dependentItemProject.getId()); - dependentItem.put("definitionId",definition.getId()); - } - } - } - } - ((ObjectNode)taskNode).set(DEPENDENCE, dependentParameters); - } - return taskNode; - } - - /** - * put dependent strategy - */ - @Override - public void afterPropertiesSet() { - TaskNodeParamFactory.register(TaskType.DEPENDENT.getDesc(), this); - } -} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/ProcessAddTaskParam.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/ProcessAddTaskParam.java deleted file mode 100644 index 8e408556b0..0000000000 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/ProcessAddTaskParam.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.utils.exportprocess; - -import com.fasterxml.jackson.databind.JsonNode; - -/** - * ProcessAddTaskParam - */ -public interface ProcessAddTaskParam { - - /** - * add export task special param: sql task dependent task - * @param taskNode task node json object - * @return task node json object - */ - JsonNode addExportSpecialParam(JsonNode taskNode); - - /** - * add task special param: sql task dependent task - * @param taskNode task node json object - * @return task node json object - */ - JsonNode addImportSpecialParam(JsonNode taskNode); -} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/TaskNodeParamFactory.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/TaskNodeParamFactory.java deleted file mode 100644 index b8f7b03dee..0000000000 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/TaskNodeParamFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.utils.exportprocess; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * task node param factory - */ -public class TaskNodeParamFactory { - - private static Map taskServices = new ConcurrentHashMap<>(); - - public static ProcessAddTaskParam getByTaskType(String taskType){ - return taskServices.get(taskType); - } - - static void register(String taskType, ProcessAddTaskParam addSpecialTaskParam){ - if (null != taskType) { - taskServices.put(taskType, addSpecialTaskParam); - } - } -} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java deleted file mode 100644 index ceee22fc2c..0000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.api.utils.exportprocess; - -import org.apache.dolphinscheduler.api.controller.AbstractControllerTest; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; - -import org.json.JSONException; -import org.junit.Test; -import org.skyscreamer.jsonassert.JSONAssert; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - -/** - * DataSourceParamTest - */ -public class DataSourceParamTest extends AbstractControllerTest { - - @Test - public void testAddExportDependentSpecialParam() throws JSONException { - - String sqlJson = "{\"type\":\"SQL\",\"id\":\"tasks-27297\",\"name\":\"sql\"," - + "\"params\":{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from test\"," - + "\"udfs\":\"\",\"sqlType\":\"1\",\"title\":\"\",\"receivers\":\"\",\"receiversCc\":\"\",\"showType\":\"TABLE\"" - + ",\"localParams\":[],\"connParams\":\"\"," - + "\"preStatements\":[],\"postStatements\":[]}," - + "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," - + "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\"," - + "\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," - + "\"preTasks\":[\"dependent\"]}"; - - ObjectNode taskNode = JSONUtils.parseObject(sqlJson); - if (StringUtils.isNotEmpty(taskNode.path("type").asText())) { - String taskType = taskNode.path("type").asText(); - - ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); - - JsonNode sql = addTaskParam.addExportSpecialParam(taskNode); - - JSONAssert.assertEquals(taskNode.toString(), sql.toString(), false); - } - } - - @Test - public void testAddImportDependentSpecialParam() throws JSONException { - String sqlJson = "{\"workerGroupId\":-1,\"description\":\"\",\"runFlag\":\"NORMAL\"," - + "\"type\":\"SQL\",\"params\":{\"postStatements\":[]," - + "\"connParams\":\"\",\"receiversCc\":\"\",\"udfs\":\"\"," - + "\"type\":\"MYSQL\",\"title\":\"\",\"sql\":\"show tables\",\"" - + "preStatements\":[],\"sqlType\":\"1\",\"receivers\":\"\",\"datasource\":1," - + "\"showType\":\"TABLE\",\"localParams\":[],\"datasourceName\":\"dsmetadata\"},\"timeout\"" - + ":{\"enable\":false,\"strategy\":\"\"},\"maxRetryTimes\":\"0\"," - + "\"taskInstancePriority\":\"MEDIUM\",\"name\":\"mysql\",\"dependence\":{}," - + "\"retryInterval\":\"1\",\"preTasks\":[\"dependent\"],\"id\":\"tasks-8745\"}"; - - ObjectNode taskNode = JSONUtils.parseObject(sqlJson); - if (StringUtils.isNotEmpty(taskNode.path("type").asText())) { - String taskType = taskNode.path("type").asText(); - - ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); - - JsonNode sql = addTaskParam.addImportSpecialParam(taskNode); - - JSONAssert.assertEquals(taskNode.toString(), sql.toString(), false); - } - } -} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java deleted file mode 100644 index 531856cb28..0000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.api.utils.exportprocess; - -import org.apache.dolphinscheduler.api.controller.AbstractControllerTest; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; - -import org.json.JSONException; -import org.junit.Test; -import org.skyscreamer.jsonassert.JSONAssert; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - -/** - * DependentParamTest - */ -public class DependentParamTest extends AbstractControllerTest { - - @Test - public void testAddExportDependentSpecialParam() throws JSONException { - String dependentJson = "{\"type\":\"DEPENDENT\",\"id\":\"tasks-33787\"," - + "\"name\":\"dependent\",\"params\":{},\"description\":\"\",\"runFlag\":\"NORMAL\"," - + "\"dependence\":{\"relation\":\"AND\",\"dependTaskList\":[{\"relation\":\"AND\"," - + "\"dependItemList\":[{\"projectId\":2,\"definitionId\":46,\"depTasks\":\"ALL\"," - + "\"cycle\":\"day\",\"dateValue\":\"today\"}]}]}}"; - - ObjectNode taskNode = JSONUtils.parseObject(dependentJson); - if (StringUtils.isNotEmpty(taskNode.path("type").asText())) { - String taskType = taskNode.path("type").asText(); - - ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); - - JsonNode dependent = addTaskParam.addExportSpecialParam(taskNode); - - JSONAssert.assertEquals(taskNode.toString(), dependent.toString(), false); - } - - String dependentEmpty = "{\"type\":\"DEPENDENT\",\"id\":\"tasks-33787\"," - + "\"name\":\"dependent\",\"params\":{},\"description\":\"\",\"runFlag\":\"NORMAL\"}"; - - ObjectNode taskEmpty = JSONUtils.parseObject(dependentEmpty); - if (StringUtils.isNotEmpty(taskEmpty.path("type").asText())) { - String taskType = taskEmpty.path("type").asText(); - - ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); - - JsonNode dependent = addTaskParam.addImportSpecialParam(taskEmpty); - - JSONAssert.assertEquals(taskEmpty.toString(), dependent.toString(), false); - } - - } - - @Test - public void testAddImportDependentSpecialParam() throws JSONException { - String dependentJson = "{\"workerGroupId\":-1,\"description\":\"\",\"runFlag\":\"NORMAL\"" - + ",\"type\":\"DEPENDENT\",\"params\":{},\"timeout\":{\"enable\":false," - + "\"strategy\":\"\"},\"maxRetryTimes\":\"0\",\"taskInstancePriority\":\"MEDIUM\"" - + ",\"name\":\"dependent\"," - + "\"dependence\":{\"dependTaskList\":[{\"dependItemList\":[{\"dateValue\":\"today\"," - + "\"definitionName\":\"shell-1\",\"depTasks\":\"shell-1\",\"projectName\":\"test\"," - + "\"projectId\":1,\"cycle\":\"day\",\"definitionId\":7}],\"relation\":\"AND\"}]," - + "\"relation\":\"AND\"},\"retryInterval\":\"1\",\"preTasks\":[],\"id\":\"tasks-55485\"}"; - - ObjectNode taskNode = JSONUtils.parseObject(dependentJson); - if (StringUtils.isNotEmpty(taskNode.path("type").asText())) { - String taskType = taskNode.path("type").asText(); - - ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); - - JsonNode dependent = addTaskParam.addImportSpecialParam(taskNode); - - JSONAssert.assertEquals(taskNode.toString(), dependent.toString(), false); - } - - String dependentEmpty = "{\"workerGroupId\":-1,\"description\":\"\",\"runFlag\":\"NORMAL\"" - + ",\"type\":\"DEPENDENT\",\"params\":{},\"timeout\":{\"enable\":false," - + "\"strategy\":\"\"},\"maxRetryTimes\":\"0\",\"taskInstancePriority\":\"MEDIUM\"" - + ",\"name\":\"dependent\",\"retryInterval\":\"1\",\"preTasks\":[],\"id\":\"tasks-55485\"}"; - - JsonNode taskNodeEmpty = JSONUtils.parseObject(dependentEmpty); - if (StringUtils.isNotEmpty(taskNodeEmpty.path("type").asText())) { - String taskType = taskNodeEmpty.path("type").asText(); - - ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); - - JsonNode dependent = addTaskParam.addImportSpecialParam(taskNode); - - JSONAssert.assertEquals(taskNodeEmpty.toString(), dependent.toString(), false); - } - - } -} diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 6d55afe682..fe1ed3aac2 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -90,6 +90,13 @@ + + org.jacoco + org.jacoco.agent + runtime + test + + commons-configuration commons-configuration diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index e818c87adc..592fa6c5bc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -1095,4 +1095,17 @@ public final class Constants { public static final boolean DOCKER_MODE = StringUtils.isNotEmpty(System.getenv("DOCKER")); public static final boolean KUBERNETES_MODE = StringUtils.isNotEmpty(System.getenv("KUBERNETES_SERVICE_HOST")) && StringUtils.isNotEmpty(System.getenv("KUBERNETES_SERVICE_PORT")); + /** + * task parameter keys + */ + public static final String TASK_PARAMS = "params"; + public static final String TASK_PARAMS_DATASOURCE = "datasource"; + public static final String TASK_PARAMS_DATASOURCE_NAME = "datasourceName"; + public static final String TASK_DEPENDENCE = "dependence"; + public static final String TASK_DEPENDENCE_DEPEND_TASK_LIST = "dependTaskList"; + public static final String TASK_DEPENDENCE_DEPEND_ITEM_LIST = "dependItemList"; + public static final String TASK_DEPENDENCE_PROJECT_ID = "projectId"; + public static final String TASK_DEPENDENCE_PROJECT_NAME = "projectName"; + public static final String TASK_DEPENDENCE_DEFINITION_ID = "definitionId"; + public static final String TASK_DEPENDENCE_DEFINITION_NAME = "definitionName"; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java index d03c13d864..a9d3bcef36 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java @@ -30,7 +30,7 @@ public abstract class AbstractDatasourceProcessor implements DatasourceProcessor private static final Pattern DATABASE_PATTER = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); - private static final Pattern PARAMS_PATTER = Pattern.compile("^[a-zA-Z0-9]+$"); + private static final Pattern PARAMS_PATTER = Pattern.compile("^[a-zA-Z0-9\\-\\_\\/]+$"); @Override public void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java index 8ebc5b17d4..1b8b59cd99 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.common.datasource; +import java.util.HashMap; +import java.util.Map; import org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam; import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO; import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor; @@ -44,7 +46,11 @@ public class DatasourceUtilTest { MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); mysqlDatasourceParamDTO.setHost("localhost"); mysqlDatasourceParamDTO.setDatabase("default"); - mysqlDatasourceParamDTO.setOther(null); + Map other = new HashMap<>(); + other.put("serverTimezone", "Asia/Shanghai"); + other.put("queryTimeout", "-1"); + other.put("characterEncoding", "utf8"); + mysqlDatasourceParamDTO.setOther(other); DatasourceUtil.checkDatasourceParam(mysqlDatasourceParamDTO); Assert.assertTrue(true); } diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index f095a9ba54..2cf3de29ac 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -48,6 +48,12 @@ junit test + + org.jacoco + org.jacoco.agent + runtime + test + com.baomidou mybatis-plus diff --git a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml b/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml index d632a04ccc..e2e158f20a 100644 --- a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml +++ b/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml @@ -70,6 +70,13 @@ test + + org.jacoco + org.jacoco.agent + runtime + test + + diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index d1e9a7ffca..5f13a329e1 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -73,6 +73,13 @@ test + + org.jacoco + org.jacoco.agent + runtime + test + + com.google.guava guava diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 4d762383fd..03544ad713 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -77,6 +77,12 @@ mockito-core test + + org.jacoco + org.jacoco.agent + runtime + test + org.springframework spring-test diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java index cbf663fce2..57abf0b4e2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java @@ -43,69 +43,15 @@ public class ParamUtils { /** * parameter conversion - * @param globalParams global params - * @param globalParamsMap global params map - * @param localParams local params - * @param commandType command type - * @param scheduleTime schedule time - * @return global params - */ - public static Map convert(Map globalParams, - Map globalParamsMap, - Map localParams, - Map varParams, - CommandType commandType, - Date scheduleTime) { - if (globalParams == null && localParams == null) { - return null; - } - // if it is a complement, - // you need to pass in the task instance id to locate the time - // of the process instance complement - Map timeParams = BusinessTimeUtils - .getBusinessTime(commandType, - scheduleTime); - - if (globalParamsMap != null) { - timeParams.putAll(globalParamsMap); - } - - if (globalParams != null && localParams != null) { - localParams.putAll(globalParams); - globalParams = localParams; - } else if (globalParams == null && localParams != null) { - globalParams = localParams; - } - if (varParams != null) { - varParams.putAll(globalParams); - globalParams = varParams; - } - Iterator> iter = globalParams.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry en = iter.next(); - Property property = en.getValue(); - - if (StringUtils.isNotEmpty(property.getValue()) - && property.getValue().startsWith("$")) { - /** - * local parameter refers to global parameter with the same name - * note: the global parameters of the process instance here are solidified parameters, - * and there are no variables in them. - */ - String val = property.getValue(); - val = ParameterUtils.convertParameterPlaceholders(val, timeParams); - property.setValue(val); - } - } - - return globalParams; - } - - /** - * parameter conversion + * Warning: + * When you first invoke the function of convert, the variables of localParams and varPool in the ShellParameters will be modified. + * But in the whole system the variables of localParams and varPool have been used in other functions. I'm not sure if this current + * situation is wrong. So I cannot modify the original logic. + * * @param taskExecutionContext the context of this task instance * @param parameters the parameters * @return global params + * */ public static Map convert(TaskExecutionContext taskExecutionContext, AbstractParameters parameters) { Preconditions.checkNotNull(taskExecutionContext); @@ -115,8 +61,11 @@ public class ParamUtils { CommandType commandType = CommandType.of(taskExecutionContext.getCmdTypeIfComplement()); Date scheduleTime = taskExecutionContext.getScheduleTime(); + // combining local and global parameters Map localParams = parameters.getLocalParametersMap(); + Map varParams = parameters.getVarPoolMap(); + if (globalParams == null && localParams == null) { return null; } @@ -141,6 +90,10 @@ public class ParamUtils { } else if (globalParams == null && localParams != null) { globalParams = localParams; } + if (varParams != null) { + varParams.putAll(globalParams); + globalParams = varParams; + } Iterator> iter = globalParams.entrySet().iterator(); while (iter.hasNext()) { Map.Entry en = iter.next(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java deleted file mode 100644 index 2d033c6fa7..0000000000 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.utils; - -import org.apache.dolphinscheduler.service.registry.RegistryClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.context.annotation.ComponentScan; - -@ComponentScan("org.apache.dolphinscheduler") -public class RemoveZKNode implements CommandLineRunner { - - private static final Integer ARGS_LENGTH = 1; - - private static final Logger logger = LoggerFactory.getLogger(RemoveZKNode.class); - - /** - * zookeeper operator - */ - private RegistryClient registryClient = RegistryClient.getInstance(); - - public static void main(String[] args) { - new SpringApplicationBuilder(RemoveZKNode.class).web(WebApplicationType.NONE).run(args); - } - - @Override - public void run(String... args) throws Exception { - if (args.length != ARGS_LENGTH) { - logger.error("Usage: "); - return; - } - - registryClient.remove(args[0]); - registryClient.close(); - } -} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index b785cb5d49..c30326d03e 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.worker.task.datax; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.common.datasource.DatasourceUtil; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.process.Property; @@ -154,13 +153,8 @@ public class DataxTask extends AbstractTask { String threadLoggerInfoName = String.format("TaskLogInfo-%s", taskExecutionContext.getTaskAppId()); Thread.currentThread().setName(threadLoggerInfoName); - // combining local and global parameters - Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()), - taskExecutionContext.getDefinedParams(), - dataXParameters.getLocalParametersMap(), - dataXParameters.getVarPoolMap(), - CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), - taskExecutionContext.getScheduleTime()); + // replace placeholder,and combine local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); // run datax procesDataSourceService.s String jsonFilePath = buildDataxJsonFile(paramsMap); @@ -258,7 +252,7 @@ public class DataxTask extends AbstractTask { } ArrayNode urlArr = readerConn.putArray("jdbcUrl"); - urlArr.add(DatasourceUtil.getJdbcUrl(DbType.valueOf(dataXParameters.getDtType()), dataSourceCfg)); + urlArr.add(DatasourceUtil.getJdbcUrl(DbType.valueOf(dataXParameters.getDsType()), dataSourceCfg)); readerConnArr.add(readerConn); @@ -276,7 +270,7 @@ public class DataxTask extends AbstractTask { ArrayNode tableArr = writerConn.putArray("table"); tableArr.add(dataXParameters.getTargetTable()); - writerConn.put("jdbcUrl", DatasourceUtil.getJdbcUrl(DbType.valueOf(dataXParameters.getDsType()), dataTargetCfg)); + writerConn.put("jdbcUrl", DatasourceUtil.getJdbcUrl(DbType.valueOf(dataXParameters.getDtType()), dataTargetCfg)); writerConnArr.add(writerConn); ObjectNode writerParam = JSONUtils.createObjectNode(); @@ -443,31 +437,31 @@ public class DataxTask extends AbstractTask { } public String loadJvmEnv(DataxParameters dataXParameters) { - int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms(); - int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx(); + int xms = Math.max(dataXParameters.getXms(), 1); + int xmx = Math.max(dataXParameters.getXmx(), 1); return String.format(JVM_PARAM, xms, xmx); } /** * parsing synchronized column names in SQL statements * - * @param dsType the database type of the data source - * @param dtType the database type of the data target + * @param sourceType the database type of the data source + * @param targetType the database type of the data target * @param dataSourceCfg the database connection parameters of the data source * @param sql sql for data synchronization * @return Keyword converted column names */ - private String[] parsingSqlColumnNames(DbType dsType, DbType dtType, BaseConnectionParam dataSourceCfg, String sql) { - String[] columnNames = tryGrammaticalAnalysisSqlColumnNames(dsType, sql); + private String[] parsingSqlColumnNames(DbType sourceType, DbType targetType, BaseConnectionParam dataSourceCfg, String sql) { + String[] columnNames = tryGrammaticalAnalysisSqlColumnNames(sourceType, sql); if (columnNames == null || columnNames.length == 0) { logger.info("try to execute sql analysis query column name"); - columnNames = tryExecuteSqlResolveColumnNames(dataSourceCfg, sql); + columnNames = tryExecuteSqlResolveColumnNames(sourceType, dataSourceCfg, sql); } notNull(columnNames, String.format("parsing sql columns failed : %s", sql)); - return DataxUtils.convertKeywordsColumns(dtType, columnNames); + return DataxUtils.convertKeywordsColumns(targetType, columnNames); } /** @@ -548,13 +542,13 @@ public class DataxTask extends AbstractTask { * @param sql sql for data synchronization * @return column name array */ - public String[] tryExecuteSqlResolveColumnNames(BaseConnectionParam baseDataSource, String sql) { + public String[] tryExecuteSqlResolveColumnNames(DbType sourceType, BaseConnectionParam baseDataSource, String sql) { String[] columnNames; sql = String.format("SELECT t.* FROM ( %s ) t WHERE 0 = 1", sql); sql = sql.replace(";", ""); try ( - Connection connection = DatasourceUtil.getConnection(DbType.valueOf(dataXParameters.getDtType()), baseDataSource); + Connection connection = DatasourceUtil.getConnection(sourceType, baseDataSource); PreparedStatement stmt = connection.prepareStatement(sql); ResultSet resultSet = stmt.executeQuery()) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java index 27e5b42f4c..863b91aaf7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.task.flink; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; @@ -80,13 +79,8 @@ public class FlinkTask extends AbstractYarnTask { if (StringUtils.isNotEmpty(flinkParameters.getMainArgs())) { String args = flinkParameters.getMainArgs(); - // replace placeholder - Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()), - taskExecutionContext.getDefinedParams(), - flinkParameters.getLocalParametersMap(), - flinkParameters.getVarPoolMap(), - CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), - taskExecutionContext.getScheduleTime()); + // combining local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); logger.info("param Map : {}", paramsMap); if (paramsMap != null) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index 01ac50bd09..4e34741577 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.server.worker.task.http; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.HttpMethod; import org.apache.dolphinscheduler.common.enums.HttpParametersType; import org.apache.dolphinscheduler.common.process.HttpProperty; @@ -137,13 +136,9 @@ public class HttpTask extends AbstractTask { protected CloseableHttpResponse sendRequest(CloseableHttpClient client) throws IOException { RequestBuilder builder = createRequestBuilder(); - // replace placeholder - Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()), - taskExecutionContext.getDefinedParams(), - httpParameters.getLocalParametersMap(), - httpParameters.getVarPoolMap(), - CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), - taskExecutionContext.getScheduleTime()); + // replace placeholder,and combine local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); + List httpPropertyList = new ArrayList<>(); if (CollectionUtils.isNotEmpty(httpParameters.getHttpParams())) { for (HttpProperty httpProperty : httpParameters.getHttpParams()) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java index ce908df596..5e8f3ca932 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.server.worker.task.mr; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.ResourceInfo; @@ -84,13 +83,8 @@ public class MapReduceTask extends AbstractYarnTask { mapreduceParameters.setQueue(taskExecutionContext.getQueue()); setMainJarName(); - // replace placeholder - Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()), - taskExecutionContext.getDefinedParams(), - mapreduceParameters.getLocalParametersMap(), - mapreduceParameters.getVarPoolMap(), - CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), - taskExecutionContext.getScheduleTime()); + // replace placeholder,and combine local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); if (paramsMap != null) { String args = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getMainArgs(), ParamUtils.convert(paramsMap)); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/procedure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/procedure/ProcedureTask.java index 3748c7a492..1a1573ca97 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/procedure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/procedure/ProcedureTask.java @@ -30,7 +30,6 @@ import static org.apache.dolphinscheduler.common.enums.DataType.VARCHAR; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.datasource.ConnectionParam; import org.apache.dolphinscheduler.common.datasource.DatasourceUtil; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Direct; @@ -119,12 +118,7 @@ public class ProcedureTask extends AbstractTask { connection = DatasourceUtil.getConnection(dbType, connectionParam); // combining local and global parameters - Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()), - taskExecutionContext.getDefinedParams(), - procedureParameters.getLocalParametersMap(), - procedureParameters.getVarPoolMap(), - CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), - taskExecutionContext.getScheduleTime()); + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); // call method stmt = connection.prepareCall(procedureParameters.getMethod()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java index e784a79b24..0ee480d7df 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java @@ -14,25 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.worker.task.python; +package org.apache.dolphinscheduler.server.worker.task.python; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.python.PythonParameters; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.VarPoolUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.CommandExecuteResult; import org.apache.dolphinscheduler.server.worker.task.PythonCommandExecutor; -import org.slf4j.Logger; import java.util.Map; +import org.slf4j.Logger; + /** * python task */ @@ -115,13 +116,8 @@ public class PythonTask extends AbstractTask { private String buildCommand() throws Exception { String rawPythonScript = pythonParameters.getRawScript().replaceAll("\\r\\n", "\n"); - // replace placeholder - Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()), - taskExecutionContext.getDefinedParams(), - pythonParameters.getLocalParametersMap(), - pythonParameters.getVarPoolMap(), - CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), - taskExecutionContext.getScheduleTime()); + // combining local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); try { rawPythonScript = VarPoolUtils.convertPythonScriptPlaceholders(rawPythonScript); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index f7887df41e..32c2ad18fe 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -21,7 +21,6 @@ import static java.util.Calendar.DAY_OF_MONTH; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters; @@ -42,10 +41,8 @@ import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; -import java.util.ArrayList; import java.util.Date; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; @@ -166,7 +163,7 @@ public class ShellTask extends AbstractTask { private String parseScript(String script) { // combining local and global parameters - Map paramsMap = ParamUtils.convert(taskExecutionContext,shellParameters); + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); // replace variable TIME with $[YYYYmmddd...] in shell file when history run job and batch complement job if (taskExecutionContext.getScheduleTime() != null) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java index a5a641cca9..6939439ef6 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.task.spark; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.SparkVersion; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.ResourceInfo; @@ -109,13 +108,8 @@ public class SparkTask extends AbstractYarnTask { // other parameters args.addAll(SparkArgsUtils.buildArgs(sparkParameters)); - // replace placeholder - Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()), - taskExecutionContext.getDefinedParams(), - sparkParameters.getLocalParametersMap(), - sparkParameters.getVarPoolMap(), - CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), - taskExecutionContext.getScheduleTime()); + // replace placeholder, and combining local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); String command = null; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index b174734e01..9dd8b516ed 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.worker.task.sql; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.common.datasource.DatasourceUtil; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -166,14 +165,8 @@ public class SqlTask extends AbstractTask { Map sqlParamsMap = new HashMap<>(); StringBuilder sqlBuilder = new StringBuilder(); - // find process instance by task id - - Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()), - taskExecutionContext.getDefinedParams(), - sqlParameters.getLocalParametersMap(), - sqlParameters.getVarPoolMap(), - CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), - taskExecutionContext.getScheduleTime()); + // combining local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); // spell SQL according to the final user-defined variable if (paramsMap == null) { @@ -276,7 +269,6 @@ public class SqlTask extends AbstractTask { } } - public String setNonQuerySqlReturn(String updateResult, List properties) { String result = null; for (Property info :properties) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java index 1d1b32de00..2f3e48dc4c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; @@ -73,12 +72,8 @@ public class SqoopTask extends AbstractYarnTask { SqoopJobGenerator generator = new SqoopJobGenerator(); String script = generator.generateSqoopJob(sqoopParameters, sqoopTaskExecutionContext); - Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(sqoopTaskExecutionContext.getDefinedParams()), - sqoopTaskExecutionContext.getDefinedParams(), - sqoopParameters.getLocalParametersMap(), - sqoopParameters.getVarPoolMap(), - CommandType.of(sqoopTaskExecutionContext.getCmdTypeIfComplement()), - sqoopTaskExecutionContext.getScheduleTime()); + // combining local and global parameters + Map paramsMap = ParamUtils.convert(sqoopTaskExecutionContext,getParameters()); if (paramsMap != null) { String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java index 12613c61c6..c3fa0fc547 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java @@ -14,27 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.master; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.enums.DataType; -import org.apache.dolphinscheduler.common.enums.Direct; -import org.apache.dolphinscheduler.common.process.Property; -import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; -import org.apache.dolphinscheduler.server.utils.ParamUtils; import java.util.Calendar; import java.util.Date; -import java.util.HashMap; import java.util.Map; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * user define param */ @@ -42,9 +36,8 @@ public class ParamsTest { private static final Logger logger = LoggerFactory.getLogger(ParamsTest.class); - @Test - public void systemParamsTest()throws Exception{ + public void systemParamsTest() throws Exception { String command = "${system.biz.date}"; // start process @@ -56,12 +49,10 @@ public class ParamsTest { logger.info("start process : {}",command); - Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date()); calendar.add(Calendar.DAY_OF_MONTH, -5); - command = "${system.biz.date}"; // complement data timeParams = BusinessTimeUtils @@ -71,40 +62,4 @@ public class ParamsTest { logger.info("complement data : {}",command); } - - @Test - public void convertTest() throws Exception { - Map globalParams = new HashMap<>(); - Property property = new Property(); - property.setProp("global_param"); - property.setDirect(Direct.IN); - property.setType(DataType.VARCHAR); - property.setValue("${system.biz.date}"); - globalParams.put("global_param", property); - - Map globalParamsMap = new HashMap<>(); - globalParamsMap.put("global_param", "${system.biz.date}"); - - Map localParams = new HashMap<>(); - Property localProperty = new Property(); - localProperty.setProp("local_param"); - localProperty.setDirect(Direct.IN); - localProperty.setType(DataType.VARCHAR); - localProperty.setValue("${global_param}"); - localParams.put("local_param", localProperty); - - Map varPoolParams = new HashMap<>(); - Property varProperty = new Property(); - varProperty.setProp("local_param"); - varProperty.setDirect(Direct.IN); - varProperty.setType(DataType.VARCHAR); - varProperty.setValue("${global_param}"); - varPoolParams.put("varPool", varProperty); - - Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, - localParams,varPoolParams, CommandType.START_PROCESS, new Date()); - logger.info(JSONUtils.toJsonString(paramsMap)); - - - } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java index 99a6eb2111..4d7bc93b41 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java @@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.enums.TaskType; @@ -85,7 +84,7 @@ public class ParamUtilsTest { localParams.put("local_param", localProperty); Property varProperty = new Property(); - varProperty.setProp("local_param"); + varProperty.setProp("varPool"); varProperty.setDirect(Direct.IN); varProperty.setType(DataType.VARCHAR); varProperty.setValue("${global_param}"); @@ -93,42 +92,72 @@ public class ParamUtilsTest { } /** - * Test convert + * This is basic test case for ParamUtils.convert. + * Warning: + * As you can see,this case invokes the function of convert in different situations. When you first invoke the function of convert, + * the variables of localParams and varPool in the ShellParameters will be modified. But in the whole system the variables of localParams + * and varPool have been used in other functions. I'm not sure if this current situation is wrong. So I cannot modify the original logic. */ @Test public void testConvert() { //The expected value - String expected = "{\"varPool\":{\"prop\":\"local_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}," + String expected = "{\"varPool\":{\"prop\":\"varPool\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}," + "\"global_param\":{\"prop\":\"global_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}," + "\"local_param\":{\"prop\":\"local_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; + //The expected value when globalParams is null but localParams is not null - String expected1 = "{\"varPool\":{\"prop\":\"local_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}," - + "\"global_param\":{\"prop\":\"global_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}," + String expected1 = "{\"varPool\":{\"prop\":\"varPool\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}," + "\"local_param\":{\"prop\":\"local_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; //Define expected date , the month is 0-base Calendar calendar = Calendar.getInstance(); calendar.set(2019, 11, 30); Date date = calendar.getTime(); + List globalParamList = globalParams.values().stream().collect(Collectors.toList()); + List localParamList = localParams.values().stream().collect(Collectors.toList()); + List varPoolParamList = varPoolParams.values().stream().collect(Collectors.toList()); + + TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setTaskInstanceId(1); + taskExecutionContext.setTaskName("params test"); + taskExecutionContext.setTaskType(TaskType.SHELL.getDesc()); + taskExecutionContext.setHost("127.0.0.1:1234"); + taskExecutionContext.setExecutePath("/tmp/test"); + taskExecutionContext.setLogPath("/log"); + taskExecutionContext.setProcessInstanceId(1); + taskExecutionContext.setExecutorId(1); + taskExecutionContext.setCmdTypeIfComplement(0); + taskExecutionContext.setScheduleTime(date); + taskExecutionContext.setGlobalParams(JSONUtils.toJsonString(globalParamList)); + taskExecutionContext.setDefinedParams(globalParamsMap); + taskExecutionContext.setVarPool("[{\"prop\":\"varPool\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"${global_param}\"}]"); + taskExecutionContext.setTaskParams( + "{\"rawScript\":\"#!/bin/sh\\necho $[yyyy-MM-dd HH:mm:ss]\\necho \\\" ${task_execution_id} \\\"\\necho \\\" ${task_execution_path}\\\"\\n\"," + + "\"localParams\":" + + "[],\"resourceList\":[]}"); + + ShellParameters shellParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), ShellParameters.class); + shellParameters.setLocalParams(localParamList); + + String varPoolParamsJson = JSONUtils.toJsonString(varPoolParams,SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + shellParameters.setVarPool(taskExecutionContext.getVarPool()); + shellParameters.dealOutParam(varPoolParamsJson); + //Invoke convert - Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, varPoolParams,CommandType.START_PROCESS, date); + Map paramsMap = ParamUtils.convert(taskExecutionContext, shellParameters); String result = JSONUtils.toJsonString(paramsMap); assertEquals(expected, result); - for (Map.Entry entry : paramsMap.entrySet()) { - - String key = entry.getKey(); - Property prop = entry.getValue(); - logger.info(key + " : " + prop.getValue()); - } - //Invoke convert with null globalParams - Map paramsMap1 = ParamUtils.convert(null, globalParamsMap, localParams,varPoolParams, CommandType.START_PROCESS, date); + taskExecutionContext.setDefinedParams(null); + Map paramsMap1 = ParamUtils.convert(taskExecutionContext, shellParameters); + String result1 = JSONUtils.toJsonString(paramsMap1); assertEquals(expected1, result1); - //Null check, invoke convert with null globalParams and null localParams - Map paramsMap2 = ParamUtils.convert(null, globalParamsMap, null, varPoolParams,CommandType.START_PROCESS, date); + // Null check, invoke convert with null globalParams and null localParams + shellParameters.setLocalParams(null); + Map paramsMap2 = ParamUtils.convert(taskExecutionContext, shellParameters); assertNull(paramsMap2); } diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 415ce53c05..84771399f2 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -94,5 +94,11 @@ mockito-core test + + org.jacoco + org.jacoco.agent + runtime + test + diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java index 8775a272e5..8d630beeb0 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java @@ -25,8 +25,7 @@ import java.util.concurrent.TimeUnit; import org.springframework.stereotype.Service; /** - * A singleton of a task queue implemented with zookeeper - * tasks queue implementation + * A singleton of a task queue implemented using PriorityBlockingQueue */ @Service public class TaskPriorityQueueImpl implements TaskPriorityQueue { diff --git a/dolphinscheduler-spi/pom.xml b/dolphinscheduler-spi/pom.xml index 0893abe86a..c3f746c21e 100644 --- a/dolphinscheduler-spi/pom.xml +++ b/dolphinscheduler-spi/pom.xml @@ -57,6 +57,12 @@ junit test + + org.jacoco + org.jacoco.agent + runtime + test + com.google.guava guava diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js index e98cda22fd..66509a65ed 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js @@ -110,6 +110,7 @@ Dag.prototype.toolbarEvent = function ({ item, code, is }) { * Echo data display */ Dag.prototype.backfill = function (arg) { + const that = this if (arg) { const marginX = 100 const g = new dagre.graphlib.Graph() @@ -144,7 +145,7 @@ Dag.prototype.backfill = function (arg) { instance: this.instance, options: { onRemoveNodes ($id) { - this.dag.removeEventModelById($id) + that.dag.removeEventModelById($id) } } }) @@ -167,7 +168,7 @@ Dag.prototype.backfill = function (arg) { instance: this.instance, options: { onRemoveNodes ($id) { - this.dag.removeEventModelById($id) + that.dag.removeEventModelById($id) } } }) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 9a90658fca..73a3738bf8 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -893,4 +893,12 @@ .operBtn { padding: 8px 6px; } + + .el-drawer__body { + ::selection { + background: #409EFF; + color: white; + } + } + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/resourceTree.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/resourceTree.js new file mode 100644 index 0000000000..3f03bf900d --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/resourceTree.js @@ -0,0 +1,44 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +export function diGuiTree (items) { // Recursive convenience tree structure + items.forEach(item => { + item.children === '' || item.children === undefined || item.children === null || item.children.length === 0 + ? operationTree(item) : diGuiTree(item.children) + }) +} + +export function operationTree (item) { + if (item.dirctory) { + item.isDisabled = true + } + delete item.children +} + +export function searchTree (element, id) { + // 根据id查找节点 + if (element.id === id) { + return element + } else if (element.children) { + let i + let result = null + for (i = 0; result === null && i < element.children.length; i++) { + result = searchTree(element.children[i], id) + } + return result + } + return null +} diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index 3031dcca92..6b3c6e6999 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -204,6 +204,7 @@ import '@riophae/vue-treeselect/dist/vue-treeselect.css' import disabledState from '@/module/mixin/disabledState' import Clipboard from 'clipboard' + import { diGuiTree, searchTree } from './_source/resourceTree' export default { name: 'flink', @@ -398,40 +399,14 @@ }) return true }, - diGuiTree (item) { // Recursive convenience tree structure - item.forEach(item => { - item.children === '' || item.children === undefined || item.children === null || item.children.length === 0 - ? this.operationTree(item) : this.diGuiTree(item.children) - }) - }, - operationTree (item) { - if (item.dirctory) { - item.isDisabled = true - } - delete item.children - }, - searchTree (element, id) { - // 根据id查找节点 - if (element.id === id) { - return element - } else if (element.children !== null) { - let i - let result = null - for (i = 0; result === null && i < element.children.length; i++) { - result = this.searchTree(element.children[i], id) - } - return result - } - return null - }, dataProcess (backResource) { let isResourceId = [] let resourceIdArr = [] if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.mainJarList.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -503,8 +478,8 @@ if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.mainJarList.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -538,8 +513,8 @@ created () { let item = this.store.state.dag.resourcesListS let items = this.store.state.dag.resourcesListJar - this.diGuiTree(item) - this.diGuiTree(items) + diGuiTree(item) + diGuiTree(items) this.mainJarList = item this.mainJarLists = items let o = this.backfillItem diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index 253a596327..d6718f1c8e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -117,6 +117,7 @@ import '@riophae/vue-treeselect/dist/vue-treeselect.css' import disabledState from '@/module/mixin/disabledState' import Clipboard from 'clipboard' + import { diGuiTree, searchTree } from './_source/resourceTree' export default { name: 'mr', @@ -210,40 +211,14 @@ _onCacheResourcesData (a) { this.cacheResourceList = a }, - diGuiTree (item) { // Recursive convenience tree structure - item.forEach(item => { - item.children === '' || item.children === undefined || item.children === null || item.children.length === 0 - ? this.operationTree(item) : this.diGuiTree(item.children) - }) - }, - operationTree (item) { - if (item.dirctory) { - item.isDisabled = true - } - delete item.children - }, - searchTree (element, id) { - // 根据id查找节点 - if (element.id === id) { - return element - } else if (element.children !== null) { - let i - let result = null - for (i = 0; result === null && i < element.children.length; i++) { - result = this.searchTree(element.children[i], id) - } - return result - } - return null - }, dataProcess (backResource) { let isResourceId = [] let resourceIdArr = [] if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.mainJarList.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -359,8 +334,8 @@ if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.mainJarList.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -388,8 +363,8 @@ created () { let item = this.store.state.dag.resourcesListS let items = this.store.state.dag.resourcesListJar - this.diGuiTree(item) - this.diGuiTree(items) + diGuiTree(item) + diGuiTree(items) this.mainJarList = item this.mainJarLists = items let o = this.backfillItem diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 78b4985754..85892f4c60 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -66,6 +66,8 @@ import disabledState from '@/module/mixin/disabledState' import codemirror from '@/conf/home/pages/resource/pages/file/pages/_source/codemirror' import Clipboard from 'clipboard' + import { diGuiTree, searchTree } from './_source/resourceTree' + let editor export default { @@ -198,40 +200,14 @@ return editor }, - diGuiTree (item) { // Recursive convenience tree structure - item.forEach(item => { - item.children === '' || item.children === undefined || item.children === null || item.children.length === 0 - ? this.operationTree(item) : this.diGuiTree(item.children) - }) - }, - operationTree (item) { - if (item.dirctory) { - item.isDisabled = true - } - delete item.children - }, - searchTree (element, id) { - // 根据id查找节点 - if (element.id === id) { - return element - } else if (element.children !== null) { - let i - let result = null - for (i = 0; result === null && i < element.children.length; i++) { - result = this.searchTree(element.children[i], id) - } - return result - } - return null - }, dataProcess (backResource) { let isResourceId = [] let resourceIdArr = [] if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.resourceOptions.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -297,8 +273,8 @@ if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.resourceOptions.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -317,7 +293,7 @@ }, created () { let item = this.store.state.dag.resourcesListS - this.diGuiTree(item) + diGuiTree(item) this.resourceOptions = item let o = this.backfillItem diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 64181913b0..8f0b85a1e1 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -69,6 +69,8 @@ import '@riophae/vue-treeselect/dist/vue-treeselect.css' import codemirror from '@/conf/home/pages/resource/pages/file/pages/_source/codemirror' import Clipboard from 'clipboard' + import { diGuiTree, searchTree } from './_source/resourceTree' + let editor export default { @@ -208,40 +210,14 @@ return editor }, - diGuiTree (item) { // Recursive convenience tree structure - item.forEach(item => { - item.children === '' || item.children === undefined || item.children === null || item.children.length === 0 - ? this.operationTree(item) : this.diGuiTree(item.children) - }) - }, - operationTree (item) { - if (item.dirctory) { - item.isDisabled = true - } - delete item.children - }, - searchTree (element, id) { - // 根据id查找节点 - if (element.id === id) { - return element - } else if (element.children !== null) { - let i - let result = null - for (i = 0; result === null && i < element.children.length; i++) { - result = this.searchTree(element.children[i], id) - } - return result - } - return null - }, dataProcess (backResource) { let isResourceId = [] let resourceIdArr = [] if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.options.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -307,8 +283,8 @@ if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.options.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -327,7 +303,7 @@ }, created () { let item = this.store.state.dag.resourcesListS - this.diGuiTree(item) + diGuiTree(item) this.options = item let o = this.backfillItem diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue index bc299a372f..e2e0d068b2 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue @@ -205,6 +205,8 @@ import '@riophae/vue-treeselect/dist/vue-treeselect.css' import disabledState from '@/module/mixin/disabledState' import Clipboard from 'clipboard' + import { diGuiTree, searchTree } from './_source/resourceTree' + export default { name: 'spark', data () { @@ -313,40 +315,14 @@ _onCacheResourcesData (a) { this.cacheResourceList = a }, - diGuiTree (item) { // Recursive convenience tree structure - item.forEach(item => { - item.children === '' || item.children === undefined || item.children === null || item.children.length === 0 - ? this.operationTree(item) : this.diGuiTree(item.children) - }) - }, - operationTree (item) { - if (item.dirctory) { - item.isDisabled = true - } - delete item.children - }, - searchTree (element, id) { - // 根据id查找节点 - if (element.id === id) { - return element - } else if (element.children !== null) { - let i - let result = null - for (i = 0; result === null && i < element.children.length; i++) { - result = this.searchTree(element.children[i], id) - } - return result - } - return null - }, dataProcess (backResource) { let isResourceId = [] let resourceIdArr = [] if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.mainJarList.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -521,8 +497,8 @@ if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.mainJarList.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -557,8 +533,8 @@ created () { let item = this.store.state.dag.resourcesListS let items = this.store.state.dag.resourcesListJar - this.diGuiTree(item) - this.diGuiTree(items) + diGuiTree(item) + diGuiTree(items) this.mainJarList = item this.mainJarLists = items let o = this.backfillItem diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/waterdrop.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/waterdrop.vue index ad4ff5737d..a5ebc3443f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/waterdrop.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/waterdrop.vue @@ -99,6 +99,7 @@ import disabledState from '@/module/mixin/disabledState' import Treeselect from '@riophae/vue-treeselect' import '@riophae/vue-treeselect/dist/vue-treeselect.css' + import { diGuiTree, searchTree } from './_source/resourceTree' export default { name: 'waterdrop', @@ -228,40 +229,14 @@ return true }, - diGuiTree (item) { // Recursive convenience tree structure - item.forEach(item => { - item.children === '' || item.children === undefined || item.children === null || item.children.length === 0 - ? this.operationTree(item) : this.diGuiTree(item.children) - }) - }, - operationTree (item) { - if (item.dirctory) { - item.isDisabled = true - } - delete item.children - }, - searchTree (element, id) { - // 根据id查找节点 - if (element.id === id) { - return element - } else if (element.children !== null) { - let i - let result = null - for (i = 0; result === null && i < element.children.length; i++) { - result = this.searchTree(element.children[i], id) - } - return result - } - return null - }, dataProcess (backResource) { let isResourceId = [] let resourceIdArr = [] if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.options.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -340,8 +315,8 @@ if (this.resourceList.length > 0) { this.resourceList.forEach(v => { this.options.forEach(v1 => { - if (this.searchTree(v1, v)) { - isResourceId.push(this.searchTree(v1, v)) + if (searchTree(v1, v)) { + isResourceId.push(searchTree(v1, v)) } }) }) @@ -364,7 +339,7 @@ }, created () { let item = this.store.state.dag.resourcesListS - this.diGuiTree(item) + diGuiTree(item) this.options = item let o = this.backfillItem diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue index 91b2ca3566..9196e9670b 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue @@ -281,6 +281,11 @@ max-height: 600px; overflow-y: scroll; padding:0 20px; + + ::selection { + background: #409EFF ; + color: white; + } } .title { line-height: 36px; diff --git a/pom.xml b/pom.xml index 668e6d5c76..ec7867a861 100644 --- a/pom.xml +++ b/pom.xml @@ -409,6 +409,14 @@ + + org.jacoco + org.jacoco.agent + ${jacoco.version} + runtime + test + + mysql mysql-connector-java @@ -826,6 +834,9 @@ maven-surefire-plugin ${maven-surefire-plugin.version} + + ${project.build.directory}/jacoco.exec + **/plugin/registry/zookeeper/ZookeeperRegistryTest.java @@ -1093,19 +1104,23 @@ jacoco-maven-plugin ${jacoco.version} - target/jacoco.exec - target/jacoco.exec + ${project.build.directory}/jacoco.exec - jacoco-initialize + default-instrument + + instrument + + + + default-restore-instrumented-classes - prepare-agent + restore-instrumented-classes - jacoco-site - test + default-report report diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index 5f2814c600..a6e3f977e5 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -305,7 +305,7 @@ CREATE TABLE `t_ds_alertgroup`( `create_time` datetime DEFAULT NULL COMMENT 'create time', `update_time` datetime DEFAULT NULL COMMENT 'update time', PRIMARY KEY (`id`), - UNIQUE KEY `t_ds_alertgroup_name_UN` (`group_name`) + UNIQUE KEY `t_ds_alertgroup_name_un` (`group_name`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; -- ---------------------------- @@ -352,7 +352,7 @@ CREATE TABLE `t_ds_datasource` ( `create_time` datetime NOT NULL COMMENT 'create time', `update_time` datetime DEFAULT NULL COMMENT 'update time', PRIMARY KEY (`id`), - UNIQUE KEY `t_ds_datasource_name_UN` (`name`, `type`) + UNIQUE KEY `t_ds_datasource_name_un` (`name`, `type`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; -- ---------------------------- diff --git a/sql/dolphinscheduler_postgre.sql b/sql/dolphinscheduler_postgre.sql index 3393938c83..88cf1431e9 100644 --- a/sql/dolphinscheduler_postgre.sql +++ b/sql/dolphinscheduler_postgre.sql @@ -209,7 +209,7 @@ CREATE TABLE t_ds_alertgroup( create_time timestamp DEFAULT NULL, update_time timestamp DEFAULT NULL, PRIMARY KEY (id), - CONSTRAINT t_ds_alertgroup_name_UN UNIQUE (group_name) + CONSTRAINT t_ds_alertgroup_name_un UNIQUE (group_name) ) ; -- @@ -250,7 +250,7 @@ CREATE TABLE t_ds_datasource ( create_time timestamp NOT NULL , update_time timestamp DEFAULT NULL , PRIMARY KEY (id), - CONSTRAINT t_ds_datasource_name_UN UNIQUE (name, type) + CONSTRAINT t_ds_datasource_name_un UNIQUE (name, type) ) ; -- diff --git a/sql/upgrade/1.4.0_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.4.0_schema/mysql/dolphinscheduler_ddl.sql index 5071d148b7..70b874278c 100644 --- a/sql/upgrade/1.4.0_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.4.0_schema/mysql/dolphinscheduler_ddl.sql @@ -285,9 +285,9 @@ BEGIN IF NOT EXISTS (SELECT 1 FROM information_schema.STATISTICS WHERE TABLE_NAME='t_ds_alertgroup' AND TABLE_SCHEMA=(SELECT DATABASE()) - AND INDEX_NAME ='t_ds_alertgroup_name_UN') + AND INDEX_NAME ='t_ds_alertgroup_name_un') THEN - ALTER TABLE t_ds_alertgroup ADD UNIQUE KEY `t_ds_alertgroup_name_UN` (`group_name`); + ALTER TABLE t_ds_alertgroup ADD UNIQUE KEY `t_ds_alertgroup_name_un` (`group_name`); END IF; END; @@ -302,12 +302,12 @@ drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_datasource_A_add_UN_datasourceName; delimiter d// CREATE PROCEDURE uc_dolphin_T_t_ds_datasource_A_add_UN_datasourceName() BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + IF NOT EXISTS (SELECT 1 FROM information_schema.STATISTICS WHERE TABLE_NAME='t_ds_datasource' AND TABLE_SCHEMA=(SELECT DATABASE()) - AND COLUMN_NAME ='t_ds_datasource_name_UN') + AND INDEX_NAME ='t_ds_datasource_name_un') THEN - ALTER TABLE t_ds_datasource ADD UNIQUE KEY `t_ds_datasource_name_UN` (`name`, `type`); + ALTER TABLE t_ds_datasource ADD UNIQUE KEY `t_ds_datasource_name_un` (`name`, `type`); END IF; END; diff --git a/sql/upgrade/1.4.0_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.4.0_schema/postgresql/dolphinscheduler_ddl.sql index 21a82b2513..419e00efb2 100644 --- a/sql/upgrade/1.4.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.4.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -274,9 +274,9 @@ CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_alertgroup_A_add_UN_groupName() RET BEGIN IF NOT EXISTS (SELECT 1 FROM pg_stat_all_indexes WHERE relname='t_ds_alertgroup' - AND indexrelname ='t_ds_alertgroup_name_UN') + AND indexrelname ='t_ds_alertgroup_name_un') THEN - ALTER TABLE t_ds_alertgroup ADD CONSTRAINT t_ds_alertgroup_name_UN UNIQUE (group_name); + ALTER TABLE t_ds_alertgroup ADD CONSTRAINT t_ds_alertgroup_name_un UNIQUE (group_name); END IF; END; $$ LANGUAGE plpgsql; @@ -292,9 +292,9 @@ CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_datasource_A_add_UN_datasourceName( BEGIN IF NOT EXISTS (SELECT 1 FROM pg_stat_all_indexes WHERE relname='t_ds_datasource' - AND indexrelname ='t_ds_datasource_name_UN') + AND indexrelname ='t_ds_datasource_name_un') THEN - ALTER TABLE t_ds_datasource ADD CONSTRAINT t_ds_datasource_name_UN UNIQUE (name, type); + ALTER TABLE t_ds_datasource ADD CONSTRAINT t_ds_datasource_name_un UNIQUE (name, type); END IF; END; $$ LANGUAGE plpgsql;