From ab9caec50eca3d51bdeecbd517f84a89a1c0d48c Mon Sep 17 00:00:00 2001 From: zsmdata <3656562@qq.com> Date: Wed, 11 Dec 2019 18:03:02 +0800 Subject: [PATCH 01/11] fix #1441 --- .../main/java/org/apache/dolphinscheduler/common/Constants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 440446e567..791c0bb558 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -315,7 +315,7 @@ public final class Constants { /** * user name regex */ - public static final Pattern REGEX_USER_NAME = Pattern.compile("^[a-zA-Z0-9_-]{3,20}$"); + public static final Pattern REGEX_USER_NAME = Pattern.compile("^[a-zA-Z0-9._-]{3,20}$"); /** * email regex From b6928fe04529012f750810a6cfc3bb0894ccb168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Wed, 18 Mar 2020 15:30:21 +0800 Subject: [PATCH 02/11] support custom datax config --- .../common/task/datax/DataxParameters.java | 40 ++++++++++++--- .../server/worker/task/datax/DataxTask.java | 39 ++++++++++++--- .../worker/task/datax/DataxTaskTest.java | 49 ++++++++++++++----- 3 files changed, 100 insertions(+), 28 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java index 95dd505c02..505e34b0a1 100755 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java @@ -27,6 +27,16 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; */ public class DataxParameters extends AbstractParameters { + /** + * if custom json config，eg 0, 1 + */ + private Integer customConfig; + + /** + * if customConfig eq 1 ,then json is usable + */ + private String json; + /** * data source type，eg MYSQL, POSTGRES ... */ @@ -157,16 +167,32 @@ public class DataxParameters extends AbstractParameters { this.jobSpeedRecord = jobSpeedRecord; } + public Integer getCustomConfig() { + return customConfig; + } + + public void setCustomConfig(Integer customConfig) { + this.customConfig = customConfig; + } + + public String getJson() { + return json; + } + + public void setJson(String json) { + this.json = json; + } + @Override public boolean checkParameters() { - if (!(dataSource != 0 - && dataTarget != 0 - && StringUtils.isNotEmpty(sql) - && StringUtils.isNotEmpty(targetTable))) { - return false; + if (customConfig == 0) { + return dataSource != 0 + && dataTarget != 0 + && StringUtils.isNotEmpty(sql) + && StringUtils.isNotEmpty(targetTable); + } else { + return StringUtils.isNotEmpty(json); } - - return true; } @Override diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 952030ea62..8083bb6869 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -192,24 +192,47 @@ public class DataxTask extends AbstractTask { throws Exception { // generate json String fileName = String.format("%s/%s_job.json", taskDir, taskProps.getTaskAppId()); + String json; Path path = new File(fileName).toPath(); if (Files.exists(path)) { return fileName; } - JSONObject job = new JSONObject(); - job.put("content", buildDataxJobContentJson()); - job.put("setting", buildDataxJobSettingJson()); - JSONObject root = new JSONObject(); - root.put("job", job); - root.put("core", buildDataxCoreJson()); - logger.debug("datax job json : {}", root.toString()); + if (dataXParameters.getCustomConfig() == 1){ + + json = dataXParameters.getJson().replaceAll("\\r\\n", "\n"); + + /** + * combining local and global parameters + */ + Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), + taskProps.getDefinedParams(), + dataXParameters.getLocalParametersMap(), + taskProps.getCmdTypeIfComplement(), + taskProps.getScheduleTime()); + if (paramsMap != null){ + json = ParameterUtils.convertParameterPlaceholders(json, ParamUtils.convert(paramsMap)); + } + + }else { + + JSONObject job = new JSONObject(); + job.put("content", buildDataxJobContentJson()); + job.put("setting", buildDataxJobSettingJson()); + + JSONObject root = new JSONObject(); + root.put("job", job); + root.put("core", buildDataxCoreJson()); + json = root.toString(); + } + + logger.debug("datax job json : {}", json); // create datax json file - FileUtils.writeStringToFile(new File(fileName), root.toString(), StandardCharsets.UTF_8); + FileUtils.writeStringToFile(new File(fileName), json, StandardCharsets.UTF_8); return fileName; } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index bd7f27530a..34a09934f4 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -44,6 +44,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; +import static org.apache.dolphinscheduler.common.enums.CommandType.START_PROCESS; + /** * DataxTask Tester. */ @@ -59,6 +61,8 @@ public class DataxTaskTest { private ApplicationContext applicationContext; + private TaskProps props = new TaskProps(); + @Before public void before() throws Exception { @@ -70,7 +74,6 @@ public class DataxTaskTest { springApplicationContext.setApplicationContext(applicationContext); Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); - TaskProps props = new TaskProps(); props.setTaskDir("/tmp"); props.setTaskAppId(String.valueOf(System.currentTimeMillis())); props.setTaskInstId(1); @@ -78,10 +81,8 @@ public class DataxTaskTest { props.setEnvFile(".dolphinscheduler_env.sh"); props.setTaskStartTime(new Date()); props.setTaskTimeout(0); - props.setTaskParams( - "{\"targetTable\":\"test\",\"postStatements\":[],\"jobSpeedByte\":1024,\"jobSpeedRecord\":1000,\"dtType\":\"MYSQL\",\"datasource\":1,\"dsType\":\"MYSQL\",\"datatarget\":2,\"jobSpeedByte\":0,\"sql\":\"select 1 as test from dual\",\"preStatements\":[\"delete from test\"],\"postStatements\":[\"delete from test\"]}"); - dataxTask = PowerMockito.spy(new DataxTask(props, logger)); - dataxTask.init(); + props.setCmdTypeIfComplement(START_PROCESS); + setTaskParems(0); Mockito.when(processService.findDataSourceById(1)).thenReturn(getDataSource()); Mockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource()); @@ -91,6 +92,22 @@ public class DataxTaskTest { Mockito.when(shellCommandExecutor.run(fileName, processService)).thenReturn(0); } + private void setTaskParems(Integer customConfig) { + if (customConfig == 1) { + props.setTaskParams( + "{\"customConfig\":1, \"localParams\":[{\"prop\":\"test\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"38294729\"}],\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"${test}\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); + +// "{\"customConfig\":1,\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"xxx\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); + } else { + props.setTaskParams( + "{\"customConfig\":0,\"targetTable\":\"test\",\"postStatements\":[],\"jobSpeedByte\":1024,\"jobSpeedRecord\":1000,\"dtType\":\"MYSQL\",\"datasource\":1,\"dsType\":\"MYSQL\",\"datatarget\":2,\"jobSpeedByte\":0,\"sql\":\"select 1 as test from dual\",\"preStatements\":[\"delete from test\"],\"postStatements\":[\"delete from test\"]}"); + + } + + dataxTask = PowerMockito.spy(new DataxTask(props, logger)); + dataxTask.init(); + } + private DataSource getDataSource() { DataSource dataSource = new DataSource(); dataSource.setType(DbType.MYSQL); @@ -102,7 +119,7 @@ public class DataxTaskTest { private ProcessInstance getProcessInstance() { ProcessInstance processInstance = new ProcessInstance(); - processInstance.setCommandType(CommandType.START_PROCESS); + processInstance.setCommandType(START_PROCESS); processInstance.setScheduleTime(new Date()); return processInstance; } @@ -229,18 +246,24 @@ public class DataxTaskTest { */ @Test public void testBuildDataxJsonFile() - throws Exception { + throws Exception { try { - Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); - method.setAccessible(true); - String filePath = (String) method.invoke(dataxTask, null); - Assert.assertNotNull(filePath); - } - catch (Exception e) { + setTaskParems(1); + buildDataJson(); + setTaskParems(0); + buildDataJson(); + } catch (Exception e) { Assert.fail(e.getMessage()); } } + public void buildDataJson() throws Exception { + Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); + method.setAccessible(true); + String filePath = (String) method.invoke(dataxTask, null); + Assert.assertNotNull(filePath); + } + /** * Method: buildDataxJobContentJson() */ From 8c4ba3a55753cde0f2cd741c0c6d8222e0c27396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Wed, 18 Mar 2020 15:38:45 +0800 Subject: [PATCH 03/11] support datax custom config --- .../common/task/datax/DataxParameters.java | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java index 505e34b0a1..c11852bd1f 100755 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java @@ -87,6 +87,22 @@ public class DataxParameters extends AbstractParameters { */ private int jobSpeedRecord; + public Integer getCustomConfig() { + return customConfig; + } + + public void setCustomConfig(Integer customConfig) { + this.customConfig = customConfig; + } + + public String getJson() { + return json; + } + + public void setJson(String json) { + this.json = json; + } + public String getDsType() { return dsType; } @@ -167,21 +183,6 @@ public class DataxParameters extends AbstractParameters { this.jobSpeedRecord = jobSpeedRecord; } - public Integer getCustomConfig() { - return customConfig; - } - - public void setCustomConfig(Integer customConfig) { - this.customConfig = customConfig; - } - - public String getJson() { - return json; - } - - public void setJson(String json) { - this.json = json; - } @Override public boolean checkParameters() { @@ -203,7 +204,9 @@ public class DataxParameters extends AbstractParameters { @Override public String toString() { return "DataxParameters{" + - "dsType='" + dsType + '\'' + + "customConfig=" + customConfig + '\'' + + ", json=" + json + '\'' + + ", dsType='" + dsType + '\'' + ", dataSource=" + dataSource + ", dtType='" + dtType + '\'' + ", dataTarget=" + dataTarget + From b32a3f31a112b89babcfc01998e348cdf0945b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Wed, 18 Mar 2020 17:10:30 +0800 Subject: [PATCH 04/11] support datax custom config --- .../dolphinscheduler/common/task/datax/DataxParameters.java | 4 ++-- .../server/worker/task/datax/DataxTaskTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java index c11852bd1f..e1fb5a0bac 100755 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java @@ -204,8 +204,8 @@ public class DataxParameters extends AbstractParameters { @Override public String toString() { return "DataxParameters{" + - "customConfig=" + customConfig + '\'' + - ", json=" + json + '\'' + + "customConfig=" + customConfig + + ", json='" + json + '\'' + ", dsType='" + dsType + '\'' + ", dataSource=" + dataSource + ", dtType='" + dtType + '\'' + diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index 34a09934f4..c2dbd268e6 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -95,7 +95,7 @@ public class DataxTaskTest { private void setTaskParems(Integer customConfig) { if (customConfig == 1) { props.setTaskParams( - "{\"customConfig\":1, \"localParams\":[{\"prop\":\"test\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"38294729\"}],\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"${test}\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); + "{\"customConfig\":1, \"localParams\":[{\"prop\":\"test\",\"value\":\"38294729\"}],\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"${test}\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); // "{\"customConfig\":1,\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"xxx\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); } else { From 74708b3b84f494facfb989ccf40ff20eb4cf94da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Wed, 18 Mar 2020 17:59:10 +0800 Subject: [PATCH 05/11] support datax custom config --- .../org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java | 1 + .../dolphinscheduler/common/task/datax/DataxParameters.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java index 24a0ed31d6..308ed8e9b6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java @@ -211,6 +211,7 @@ public class CheckUtilsTest { // DataxParameters DataxParameters dataxParameters = new DataxParameters(); assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(dataxParameters), TaskType.DATAX.toString())); + dataxParameters.setCustomConfig(0); dataxParameters.setDataSource(111); dataxParameters.setDataTarget(333); dataxParameters.setSql("sql"); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java index e1fb5a0bac..f153360d63 100755 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java @@ -186,6 +186,7 @@ public class DataxParameters extends AbstractParameters { @Override public boolean checkParameters() { + if (customConfig == null) return false; if (customConfig == 0) { return dataSource != 0 && dataTarget != 0 @@ -204,7 +205,7 @@ public class DataxParameters extends AbstractParameters { @Override public String toString() { return "DataxParameters{" + - "customConfig=" + customConfig + + "customConfig=" + customConfig + ", json='" + json + '\'' + ", dsType='" + dsType + '\'' + ", dataSource=" + dataSource + From 8ed2646e65b0a72f27d61efa5202c22ac6aa2801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Wed, 22 Apr 2020 14:10:34 +0800 Subject: [PATCH 06/11] fix #2450 --- .../apache/dolphinscheduler/api/service/ExecutorService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index d290886572..51f5420ac5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -499,7 +499,7 @@ public class ExecutorService extends BaseService{ // determine whether to complement if(commandType == CommandType.COMPLEMENT_DATA){ runMode = (runMode == null) ? RunMode.RUN_MODE_SERIAL : runMode; - if(null != start && null != end && start.before(end)){ + if(null != start && null != end && !start.after(end)){ if(runMode == RunMode.RUN_MODE_SERIAL){ cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(start)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(end)); From 798b133ec11b80585d0859c7f2dbf20e0373d74a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Wed, 29 Apr 2020 14:48:38 +0800 Subject: [PATCH 07/11] fix #2481 --- .../src/main/resources/config/install_config.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index cba117e048..fab6b32d51 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -27,6 +27,9 @@ dbhost="192.168.xx.xx:3306" # db username username="xx" +# database name +dbname="dolphinscheduler" + # db passwprd # NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[` password="xx" From 9d22d8beacb919c04ee2bc8f69a5035d3ae22e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Wed, 29 Apr 2020 14:50:20 +0800 Subject: [PATCH 08/11] fix #2481 --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index a1323952a7..7fad7fa77a 100644 --- a/install.sh +++ b/install.sh @@ -35,7 +35,7 @@ if [ $dbtype == "postgresql" ];then datasourceDriverClassname="org.postgresql.Driver" fi sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=${datasourceDriverClassname}#g" conf/datasource.properties -sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/dolphinscheduler?characterEncoding=UTF-8#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8&allowMultiQueries=true#g" conf/datasource.properties sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/datasource.properties sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${password}#g" conf/datasource.properties From f3fbdfc8e7391f98f3b263e599ecda4e9d06986e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Thu, 30 Apr 2020 15:03:00 +0800 Subject: [PATCH 09/11] 1 --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 7fad7fa77a..a74fe67531 100644 --- a/install.sh +++ b/install.sh @@ -35,7 +35,7 @@ if [ $dbtype == "postgresql" ];then datasourceDriverClassname="org.postgresql.Driver" fi sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=${datasourceDriverClassname}#g" conf/datasource.properties -sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8&allowMultiQueries=true#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true#g" conf/datasource.properties sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/datasource.properties sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${password}#g" conf/datasource.properties From 82fee46d709593799d54f1587b8e783b7c6bfc7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Wed, 6 May 2020 19:31:56 +0800 Subject: [PATCH 10/11] unique check before import ProcessDefinitions --- .../api/service/ProcessDefinitionService.java | 7 +++++++ sql/dolphinscheduler_mysql.sql | 1 + 2 files changed, 8 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 4fdafa46fa..6727ccd894 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -782,6 +782,13 @@ public class ProcessDefinitionService extends BaseDAGService { processDefinitionName, 1); } + // unique check + result = verifyProcessDefinitionName(loginUser, currentProjectName, processDefinitionName); + Status status = (Status) result.get(Constants.STATUS); + if (!Status.SUCCESS.equals(status)) { + return false; + } + // get create process result Map createProcessResult = getCreateProcessResult(loginUser, diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index c0c81dcb21..8ff2667f5f 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -415,6 +415,7 @@ CREATE TABLE `t_ds_process_definition` ( `modify_by` varchar(255) DEFAULT NULL, `resource_ids` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`), + UNIQUE KEY `process_definition_unique` (`name`,`project_id`), KEY `process_definition_index` (`project_id`,`id`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; From 97d09e1d6af74f3f3a8c57cdfc332b1016752b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Wed, 6 May 2020 19:38:34 +0800 Subject: [PATCH 11/11] init --- .../api/service/ProcessDefinitionService.java | 7 ------- sql/dolphinscheduler_mysql.sql | 1 - 2 files changed, 8 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 6727ccd894..4fdafa46fa 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -782,13 +782,6 @@ public class ProcessDefinitionService extends BaseDAGService { processDefinitionName, 1); } - // unique check - result = verifyProcessDefinitionName(loginUser, currentProjectName, processDefinitionName); - Status status = (Status) result.get(Constants.STATUS); - if (!Status.SUCCESS.equals(status)) { - return false; - } - // get create process result Map createProcessResult = getCreateProcessResult(loginUser, diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index 8ff2667f5f..c0c81dcb21 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -415,7 +415,6 @@ CREATE TABLE `t_ds_process_definition` ( `modify_by` varchar(255) DEFAULT NULL, `resource_ids` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`), - UNIQUE KEY `process_definition_unique` (`name`,`project_id`), KEY `process_definition_index` (`project_id`,`id`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;