From eefb71855d580c36fd251a6b3bf0c3da912349c7 Mon Sep 17 00:00:00 2001 From: Yelli <51317527+Yeleights@users.noreply.github.com> Date: Thu, 26 Dec 2019 19:56:15 +0800 Subject: [PATCH 01/16] =?UTF-8?q?#747=20Worker=20Log=20desensitization(?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=84=B1=E6=95=8F)=20(#1568)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * modify FileUtils.readFile2Str * #1300 Add right alignment function in sql email content * cancel formatted for alert_mail_template.ftl * #747 sql task password Log desensitization * cancel mail_temple * edit ExcelUtils * modify test method name * #747 sql task password Log desensitization * Constants add DATASOURCE_PASSWORD_REGEX --- .../dolphinscheduler/common/Constants.java | 6 +- .../server/utils/SensitiveLogUtil.java | 39 ++++++++ .../worker/log/SensitiveDataConverter.java | 92 +++++++++++++++++++ .../src/main/resources/worker_logback.xml | 4 +- .../server/utils/SensitiveLogUtilTest.java | 37 ++++++++ .../log/SensitiveDataConverterTest.java | 92 +++++++++++++++++++ 6 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtil.java create mode 100644 dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverter.java create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtilTest.java create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverterTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index eafb14b255..6073f218a4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -36,7 +36,6 @@ public final class Constants { */ public static final String HADOOP_PROPERTIES_PATH = "/common/hadoop/hadoop.properties"; - /** * common properties path */ @@ -1007,4 +1006,9 @@ public final class Constants { public static final String RECEIVERS = "receivers"; public static final String RECEIVERS_CC = "receiversCc"; + + /** + * dataSource sensitive param + */ + public static final String DATASOURCE_PASSWORD_REGEX = "(?<=(\"password\":\")).*?(?=(\"))"; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtil.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtil.java new file mode 100644 index 0000000000..948e92cb24 --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtil.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.utils; + +import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.common.Constants; + +/** + * sensitive log Util + */ +public class SensitiveLogUtil { + + /** + * @param dataSourcePwd data source password + * @return String + */ + public static String maskDataSourcePwd(String dataSourcePwd){ + + if (StringUtils.isNotEmpty(dataSourcePwd)) { + dataSourcePwd = Constants.PASSWORD_DEFAULT; + } + return dataSourcePwd; + } + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverter.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverter.java new file mode 100644 index 0000000000..be8d3d12a0 --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverter.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.worker.log; + + +import ch.qos.logback.classic.pattern.MessageConverter; +import ch.qos.logback.classic.spi.ILoggingEvent; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.server.utils.SensitiveLogUtil; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * sensitive data log converter + */ +@Slf4j +public class SensitiveDataConverter extends MessageConverter { + + /** + * password pattern + */ + private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX); + + + @Override + public String convert(ILoggingEvent event) { + + // get original log + String requestLogMsg = event.getFormattedMessage(); + + // desensitization log + return convertMsg(requestLogMsg); + } + + /** + * deal with sensitive log + * + * @param oriLogMsg original log + */ + private String convertMsg(final String oriLogMsg) { + + String tempLogMsg = oriLogMsg; + + if (StringUtils.isNotEmpty(tempLogMsg)) { + tempLogMsg = passwordHandler(pwdPattern, tempLogMsg); + } + return tempLogMsg; + } + + /** + * password regex + * + * @param logMsg original log + */ + private String passwordHandler(Pattern pwdPattern, String logMsg) { + + Matcher matcher = pwdPattern.matcher(logMsg); + + StringBuffer sb = new StringBuffer(logMsg.length()); + + while (matcher.find()) { + + String password = matcher.group(); + + String maskPassword = SensitiveLogUtil.maskDataSourcePwd(password); + + matcher.appendReplacement(sb, maskPassword); + } + matcher.appendTail(sb); + + return sb.toString(); + } + + +} diff --git a/dolphinscheduler-server/src/main/resources/worker_logback.xml b/dolphinscheduler-server/src/main/resources/worker_logback.xml index 64d85d4565..7ba0c9b8ab 100644 --- a/dolphinscheduler-server/src/main/resources/worker_logback.xml +++ b/dolphinscheduler-server/src/main/resources/worker_logback.xml @@ -18,6 +18,8 @@ + @@ -31,7 +33,7 @@ INFO - + taskAppId ${log.base} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtilTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtilTest.java new file mode 100644 index 0000000000..2e5bfcf3e5 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtilTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.utils; + + +import org.apache.dolphinscheduler.common.Constants; +import org.junit.Assert; +import org.junit.Test; + + +public class SensitiveLogUtilTest { + + @Test + public void testMaskDataSourcePwd() { + + String password = "123456"; + String emptyPassword = ""; + + Assert.assertEquals(Constants.PASSWORD_DEFAULT, SensitiveLogUtil.maskDataSourcePwd(password)); + Assert.assertEquals("", SensitiveLogUtil.maskDataSourcePwd(emptyPassword)); + + } +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverterTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverterTest.java new file mode 100644 index 0000000000..fb564a22fb --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverterTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.worker.log; + + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.server.utils.SensitiveLogUtil; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SensitiveDataConverterTest { + + private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class); + + /** + * password pattern + */ + private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX); + + + /** + * mask sensitive logMsg - sql task datasource password + */ + @Test + public void testPwdLogMsgConverter() { + + String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + + "\"database\":\"carbond\"," + + "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + + "\"user\":\"view\"," + + "\"password\":\"view1\"}"; + + String maskLogMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + + "\"database\":\"carbond\"," + + "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + + "\"user\":\"view\"," + + "\"password\":\"******\"}"; + + + logger.info("parameter : {}", logMsg); + logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg)); + + Assert.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg)); + Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg)); + + } + + /** + * password regex test + * + * @param logMsg original log + */ + private static String passwordHandler(Pattern pattern, String logMsg) { + + Matcher matcher = pattern.matcher(logMsg); + + StringBuffer sb = new StringBuffer(logMsg.length()); + + while (matcher.find()) { + + String password = matcher.group(); + + String maskPassword = SensitiveLogUtil.maskDataSourcePwd(password); + + matcher.appendReplacement(sb, maskPassword); + } + matcher.appendTail(sb); + + return sb.toString(); + } + + +} From 90730133ac560770683c22d7e73713b43a0f41ea Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Thu, 26 Dec 2019 20:09:21 +0800 Subject: [PATCH 02/16] common/ParameterUtils Utils Test (#1583) * common/parameterUtils unit test --- .../common/utils/ParameterUtilsTest.java | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java new file mode 100644 index 0000000000..8bb64b03c8 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.lang.time.DateUtils; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DataType; +import org.apache.dolphinscheduler.common.enums.Direct; +import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.*; +import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_TIME; +import static org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils.replacePlaceholders; + + +public class ParameterUtilsTest { + public static final Logger logger = LoggerFactory.getLogger(ParameterUtilsTest.class); + + /** + * Test convertParameterPlaceholders + */ + @Test + public void testConvertParameterPlaceholders() throws Exception { + // parameterString,parameterMap is null + Assert.assertNull(ParameterUtils.convertParameterPlaceholders(null, null)); + + // parameterString is null,parameterMap is not null + Map parameterMap = new HashMap(); + parameterMap.put("testParameter","testParameter"); + Assert.assertNull(ParameterUtils.convertParameterPlaceholders(null, parameterMap)); + + // parameterString、parameterMap is not null + String parameterString = "test_parameter"; + Assert.assertEquals(parameterString, ParameterUtils.convertParameterPlaceholders(parameterString, parameterMap)); + + //replace variable ${} form + parameterMap.put("testParameter2","${testParameter}"); + Assert.assertEquals(parameterString,PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true)); + + // replace time $[...] form, eg. $[yyyyMMdd] + Date cronTime = new Date(); + Assert.assertEquals(parameterString, replacePlaceholders(parameterString, cronTime, true)); + + // replace time $[...] form, eg. $[yyyyMMdd] + Date cronTimeStr = DateUtils.parseDate("20191220145900", new String[]{PARAMETER_FORMAT_TIME}); + Assert.assertEquals(parameterString, replacePlaceholders(parameterString, cronTimeStr, true)); + } + + /** + * Test curingGlobalParams + */ + @Test + public void testCuringGlobalParams() throws Exception { + //define globalMap + Map globalParamMap = new HashMap<>(); + globalParamMap.put("globalParams1","Params1"); + + //define globalParamList + List globalParamList = new ArrayList<>(); + + //define scheduleTime + Date scheduleTime = DateUtils.parseDate("20191220145900", new String[]{PARAMETER_FORMAT_TIME}); + + //test globalParamList is null + String result = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); + Assert.assertNull(result); + Assert.assertNull(ParameterUtils.curingGlobalParams(null,null,CommandType.START_CURRENT_TASK_PROCESS,null)); + Assert.assertNull(ParameterUtils.curingGlobalParams(globalParamMap,null,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime)); + + //test globalParamList is not null + Property property=new Property("testGlobalParam", Direct.IN, DataType.VARCHAR,"testGlobalParam"); + globalParamList.add(property); + + String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime); + Assert.assertEquals(result2, JSONObject.toJSONString(globalParamList)); + + String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null); + Assert.assertEquals(result3, JSONObject.toJSONString(globalParamList)); + + String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); + Assert.assertEquals(result4, JSONObject.toJSONString(globalParamList)); + + //test var $ startsWith + globalParamMap.put("bizDate","${system.biz.date}"); + globalParamMap.put("b1zCurdate","${system.biz.curdate}"); + + + Property property2=new Property("testParamList1", Direct.IN, DataType.VARCHAR,"testParamList"); + Property property3=new Property("testParamList2", Direct.IN, DataType.VARCHAR,"{testParamList1}"); + Property property4=new Property("testParamList3", Direct.IN, DataType.VARCHAR,"${b1zCurdate}"); + + globalParamList.add(property2); + globalParamList.add(property3); + globalParamList.add(property4); + + String result5 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); + Assert.assertEquals(result5,JSONUtils.toJsonString(globalParamList)); + } + + /** + * Test handleEscapes + */ + @Test + public void testHandleEscapes() throws Exception { + Assert.assertNull(ParameterUtils.handleEscapes(null)); + Assert.assertEquals("",ParameterUtils.handleEscapes("")); + Assert.assertEquals("test Parameter",ParameterUtils.handleEscapes("test Parameter")); + Assert.assertEquals("////%test////%Parameter",ParameterUtils.handleEscapes("%test%Parameter")); + } + +} From 110de68fccd04e637fa44541538a13cfa02047f0 Mon Sep 17 00:00:00 2001 From: loushang Date: Fri, 27 Dec 2019 12:05:34 +0800 Subject: [PATCH 03/16] fix Stirng format bug --- .../org/apache/dolphinscheduler/common/zk/AbstractZKClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java index bfcfb53108..5b937ce46d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java @@ -461,7 +461,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ if (serverHost.equals(OSUtils.getHost())) { logger.error("{} server({}) of myself dead , stopping...", zkNodeType.toString(), serverHost); - stoppable.stop(String.format(" {} server {} of myself dead , stopping...", + stoppable.stop(String.format(" %s server %s of myself dead , stopping...", zkNodeType.toString(), serverHost)); return true; } From a526106952005dfa85b7f909019bd57bc5aa7576 Mon Sep 17 00:00:00 2001 From: gaojun2048 <32193458+gaojun2048@users.noreply.github.com> Date: Fri, 27 Dec 2019 14:32:24 +0800 Subject: [PATCH 04/16] Added the maven profile to build rpm packages (#1563) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update quick-start.md * 更新demo的地址 * add rpm plugin * delete the git ignore files * 测试git ignore * 测试ignore * optimize the .gitignore file * add rpm package, delete unuse copy jar action in assembly * delete docs dir * add rpm plugin support * add rpm plugin support * add rpm plugin support * dolphinscheduler-common have not bin dir * dolphinscheduler-common have not bin dir * delete unuse config about assembly * add defineSatement * add rpm plugin support * add rpm plugin support * add rpm plugin support * add rpm support * update install dirname , make front and backend together * update rpm name * update rpm name * update rpm config * add jars excludes * add jars excludes * add rpm plugion * add rpm plugion * add rpm plugion * add auto create /opt/soft --- dolphinscheduler-dist/pom.xml | 270 ++++++++++++++++++ .../main/assembly/dolphinscheduler-binary.xml | 19 +- pom.xml | 16 ++ 3 files changed, 288 insertions(+), 17 deletions(-) diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 3d01eff831..dbeddbb6b3 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -101,6 +101,276 @@ + + + rpmbuild + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${project.build.directory}/lib + false + false + true + provided + + + + copy-dependencies + package + + copy-dependencies + + + + + + + org.codehaus.mojo + rpm-maven-plugin + true + + + package + + attached-rpm + + + + + + apache-dolphinscheduler-incubating + 1 + apache dolphinscheduler incubating rpm + apache + dolphinscheduler + + /opt/soft + + + + __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g') + + + + /opt/soft/${project.build.finalName}/conf + 755 + root + root + + + + ${basedir}/../dolphinscheduler-alert/src/main/resources + + + **/*.properties + **/*.xml + **/*.json + **/*.ftl + + + + + + ${basedir}/../dolphinscheduler-common/src/main/resources + + + **/*.properties + **/*.xml + **/*.json + + + + + + ${basedir}/../dolphinscheduler-dao/src/main/resources + + + **/*.properties + **/*.xml + **/*.json + **/*.yml + + + + + + ${basedir}/../dolphinscheduler-api/src/main/resources + + + **/*.properties + **/*.xml + **/*.json + + + + + + ${basedir}/../dolphinscheduler-server/src/main/resources + + + **/*.properties + **/*.xml + **/*.json + + + + + + ${basedir}/../script + + + config/*.* + env/*.* + + + + + + + + /opt/soft/${project.build.finalName}/lib + 755 + root + root + + + + + ${basedir}/../dolphinscheduler-dist/target/lib + + + *.* + + + servlet-api-*.jar + slf4j-log4j12-${slf4j.log4j12.version}.jar + + + + + + /opt/soft/${project.build.finalName}/bin + 755 + root + root + + + + + ${basedir}/../script + + + start-all.sh + stop-all.sh + dolphinscheduler-daemon.sh + + + + + + /opt/soft/${project.build.finalName} + 755 + root + root + + + + ${basedir}/../ + + + *.sh + *.py + DISCLAIMER + + + + + + ${basedir}/../dolphinscheduler-ui + + + install-dolphinscheduler-ui.sh + + + + + ${basedir}/release-docs + + + **/* + + + + + + + /opt/soft/${project.build.finalName}/dist + 755 + root + root + + + + ${basedir}/../dolphinscheduler-ui/dist + + + **/*.* + + + + + + /opt/soft/${project.build.finalName}/sql + 755 + root + root + + + + ${basedir}/../sql + + + **/*.* + + + + + + + /opt/soft/${project.build.finalName}/script + 755 + root + root + + + + ${basedir}/../script + + + **/*.* + + + + + + + + + + + + + + + + + + + + diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml index 5cddadd4f5..2bf109698b 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml @@ -97,23 +97,8 @@ conf - - ${basedir}/../dolphinscheduler-common/src/main/resources - - **/*.properties - **/*.xml - **/*.json - - conf - - - ${basedir}/../dolphinscheduler-common/src/main/resources/bin - - *.* - - 755 - bin - + + ${basedir}/../dolphinscheduler-dao/src/main/resources diff --git a/pom.xml b/pom.xml index 83a6b45e10..3d002df1ed 100644 --- a/pom.xml +++ b/pom.xml @@ -108,6 +108,8 @@ 2.10.3 2.4 2.18.1 + 3.1.1 + 2.2.0 0.8.4 1.0 false @@ -524,6 +526,14 @@ + + + org.codehaus.mojo + rpm-maven-plugin + ${rpm-maven-plugion.version} + false + + org.apache.maven.plugins maven-compiler-plugin @@ -568,6 +578,12 @@ ${maven-source-plugin.version} + + org.apache.maven.plugins + maven-dependency-plugin + ${maven-dependency-plugin.version} + + From ef401b98e2072ce2d175f407ce16f4a9c3672075 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Fri, 27 Dec 2019 14:32:44 +0800 Subject: [PATCH 05/16] add AlertGroupMapperTest UT in github action (#1590) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove LogViewServiceGrpc.java file and pom modify * remove kazoo * remove kazoo * remove kazoo * remove common monitor package * add license * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * install.sh remove python kazoo * add system param whether repeat running * remove kazoo modify * BusinessTimeUtils remove whther repeat running inner param * add AccessTokenMapperTest UT * CI UT yml modify，start postgresql and zookeeper by default * add AlertGroupMapperTest UT in github action * Conflicts reslove --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 3d002df1ed..69f1ff0665 100644 --- a/pom.xml +++ b/pom.xml @@ -673,6 +673,7 @@ **/server/utils/SparkArgsUtilsTest.java **/server/utils/FlinkArgsUtilsTest.java **/dao/mapper/AccessTokenMapperTest.java + **/dao/mapper/AlertGroupMapperTest.java From fe9344d4be3c6b4bcf5e280fca770da8a8f37cee Mon Sep 17 00:00:00 2001 From: samz406 Date: Fri, 27 Dec 2019 14:42:21 +0800 Subject: [PATCH 06/16] mobile phone need 11 number (#1596) * misspell words * modify common queue TaskQueueZKImplTest.java unit test * extends BaseTaskQueueTest get zkServer * modify zk config * add MonitorServiceTest * mobile phone need 11 number * delete file --- .../java/org/apache/dolphinscheduler/api/utils/CheckUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index 6c9f714721..a7867f1ba2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -104,7 +104,7 @@ public class CheckUtils { * @return true if phone regex valid, otherwise return false */ public static boolean checkPhone(String phone) { - return StringUtils.isEmpty(phone) || phone.length() <= 11; + return StringUtils.isEmpty(phone) || phone.length() == 11; } From 9de928ea042d3159c5e875ae3a75e8de0ce65462 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 27 Dec 2019 15:00:55 +0800 Subject: [PATCH 07/16] add profile nginx in order to deploy frontend (#1599) * add profile nginx * add dolphinscheduler-nginx.xml --- dolphinscheduler-dist/pom.xml | 57 +++++ .../main/assembly/dolphinscheduler-nginx.xml | 236 ++++++++++++++++++ dolphinscheduler-ui/pom.xml | 165 ++++++++---- 3 files changed, 408 insertions(+), 50 deletions(-) create mode 100644 dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index dbeddbb6b3..2a7553e3a3 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -102,6 +102,63 @@ + + nginx + + + + maven-assembly-plugin + + + dolphinscheduler-nginx + package + + single + + + + + src/main/assembly/dolphinscheduler-nginx.xml + + true + + + + + src + package + + single + + + + src/main/assembly/dolphinscheduler-src.xml + + true + + + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + verify + + jar-no-fork + + + + + + + + + rpmbuild diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml new file mode 100644 index 0000000000..ebe5b2cb3f --- /dev/null +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml @@ -0,0 +1,236 @@ + + + + dolphinscheduler-nginx + + tar.gz + + true + ${project.build.finalName}-dolphinscheduler-bin + + + + + ${basedir}/../dolphinscheduler-alert/src/main/resources + + **/*.properties + **/*.xml + **/*.json + **/*.ftl + + ./conf + + + + + + src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + ${basedir}/../dolphinscheduler-common/src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + ${basedir}/../dolphinscheduler-common/src/main/resources/bin + + *.* + + 755 + bin + + + ${basedir}/../dolphinscheduler-dao/src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + ${basedir}/../dolphinscheduler-api/src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + + + + ${basedir}/../dolphinscheduler-server/src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + ${basedir}/../dolphinscheduler-common/src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + ${basedir}/../dolphinscheduler-common/src/main/resources/bin + + *.* + + 755 + bin + + + ${basedir}/../dolphinscheduler-dao/src/main/resources + + **/*.properties + **/*.xml + **/*.json + **/*.yml + + conf + + + + + ${basedir}/../dolphinscheduler-server/target/dolphinscheduler-server-${project.version} + + **/*.* + + . + + + + ${basedir}/../dolphinscheduler-api/target/dolphinscheduler-api-${project.version} + + **/*.* + + . + + + + ${basedir}/../dolphinscheduler-alert/target/dolphinscheduler-alert-${project.version} + + **/*.* + + . + + + + ${basedir}/../dolphinscheduler-ui/dist + + **/*.* + + ./ui/dist + + + + ${basedir}/../dolphinscheduler-ui + + install-dolphinscheduler-ui.sh + + ./ui + + + + ${basedir}/../sql + + **/* + + ./sql + + + + ${basedir}/../script + + *.* + + ./script + + + + ${basedir}/../script + + config/*.* + env/*.* + + ./conf + + + + ${basedir}/../script + + start-all.sh + stop-all.sh + dolphinscheduler-daemon.sh + + ./bin + + + + ${basedir}/.././ + + *.sh + *.py + DISCLAIMER + + . + + + + ${basedir}/release-docs + true + + **/* + + . + + + + + + + lib + true + + javax.servlet:servlet-api + org.eclipse.jetty.aggregate:jetty-all + org.slf4j:slf4j-log4j12 + + + + \ No newline at end of file diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index 9f338a2977..28f584f49a 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -32,55 +32,120 @@ v12.12.0 6.11.3 + + + release + + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + + install node and npm + + install-node-and-npm + + + ${node.version} + ${npm.version} + + + + npm install node-sass --unsafe-perm + + npm + + generate-resources + + install node-sass --unsafe-perm + + + + npm install + + npm + + generate-resources + + install + + + + npm run build:release + + npm + + + run build:release + + + + + + + + + + + nginx + + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + + install node and npm + + install-node-and-npm + + + ${node.version} + ${npm.version} + + + + npm install node-sass --unsafe-perm + + npm + + generate-resources + + install node-sass --unsafe-perm + + + + npm install + + npm + + generate-resources + + install + + + + npm run build + + npm + + + run build + + + + + + + + + + + + + - - - - com.github.eirslett - frontend-maven-plugin - ${frontend-maven-plugin.version} - - - install node and npm - - install-node-and-npm - - - ${node.version} - ${npm.version} - - - - npm install node-sass --unsafe-perm - - npm - - generate-resources - - install node-sass --unsafe-perm - - - - npm install - - npm - - generate-resources - - install - - - - npm run build:release - - npm - - - run build:release - - - - - - From 7826843620fb31d312f13a017dbf06e3d7ef5e25 Mon Sep 17 00:00:00 2001 From: samz406 Date: Fri, 27 Dec 2019 15:02:49 +0800 Subject: [PATCH 08/16] update token add exist check (#1598) * misspell words * modify common queue TaskQueueZKImplTest.java unit test * extends BaseTaskQueueTest get zkServer * modify zk config * updateToken add entity exist check --- .../dolphinscheduler/api/service/AccessTokenService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java index 4664b59763..897646ba70 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java @@ -154,8 +154,13 @@ public class AccessTokenService extends BaseService { */ public Map updateToken(int id,int userId, String expireTime, String token) { Map result = new HashMap<>(5); - AccessToken accessToken = new AccessToken(); - accessToken.setId(id); + + AccessToken accessToken = accessTokenMapper.selectById(id); + if (accessToken == null) { + logger.error("access token not exist, access token id {}", id); + putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); + return result; + } accessToken.setUserId(userId); accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); accessToken.setToken(token); From c17fe38a6ceea589109fa332909de89ce96b467d Mon Sep 17 00:00:00 2001 From: "DK.Pino" Date: Fri, 27 Dec 2019 16:41:34 +0800 Subject: [PATCH 09/16] remove lombok annocation (#1605) --- .../dolphinscheduler/common/enums/AlertType.java | 10 ++++++++-- .../dolphinscheduler/common/enums/FailureStrategy.java | 10 ++++++++-- .../dolphinscheduler/common/enums/ReleaseState.java | 10 ++++++++-- .../apache/dolphinscheduler/common/enums/RunMode.java | 10 ++++++++-- .../dolphinscheduler/common/enums/TaskDependType.java | 10 ++++++++-- .../dolphinscheduler/common/enums/WarningType.java | 10 ++++++++-- 6 files changed, 48 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java index e7c3b24a6c..3c757f5337 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * warning message notification method */ -@Getter public enum AlertType { /** * 0 email; 1 SMS @@ -39,4 +37,12 @@ public enum AlertType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/FailureStrategy.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/FailureStrategy.java index 6582d2056c..c9c0c32930 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/FailureStrategy.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/FailureStrategy.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * failure policy when some task node failed. */ -@Getter public enum FailureStrategy { /** @@ -40,4 +38,12 @@ public enum FailureStrategy { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java index 41662a4f67..51e9a3393b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * process define release state */ -@Getter public enum ReleaseState { /** @@ -50,4 +48,12 @@ public enum ReleaseState { //For values out of enum scope return null; } + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java index 1f751ffaeb..44fdb5b1b6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * complement data run mode */ -@Getter public enum RunMode { /** * 0 serial run @@ -39,4 +37,12 @@ public enum RunMode { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java index 590e178350..401fecf3ea 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * task node depend type */ -@Getter public enum TaskDependType { /** * 0 run current tasks only @@ -41,4 +39,12 @@ public enum TaskDependType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java index ce9124b8b8..3a65760716 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * types for whether to send warning when process ending; */ -@Getter public enum WarningType { /** * 0 do not send warning; @@ -44,4 +42,12 @@ public enum WarningType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } From aadf0aae46fd6788be59ce410af7e6a0cf59a7c7 Mon Sep 17 00:00:00 2001 From: "DK.Pino" Date: Fri, 27 Dec 2019 16:59:48 +0800 Subject: [PATCH 10/16] remove lombok annocation (#1600) --- .../org/apache/dolphinscheduler/common/enums/Flag.java | 10 ++++++++-- .../apache/dolphinscheduler/common/enums/Priority.java | 9 +++++++-- .../dolphinscheduler/common/enums/ResourceType.java | 10 ++++++++-- .../apache/dolphinscheduler/common/enums/ShowType.java | 10 ++++++++-- .../apache/dolphinscheduler/common/enums/TaskType.java | 9 +++++++-- .../apache/dolphinscheduler/common/enums/UserType.java | 10 ++++++++-- 6 files changed, 46 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java index a4a0d41162..622e9d17d4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * have_script @@ -27,7 +26,6 @@ import lombok.Getter; * have_map_variables * have_alert */ -@Getter public enum Flag { /** * 0 no @@ -45,4 +43,12 @@ public enum Flag { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java index 9d20ed8eed..bdd7128eac 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * define process and task priority */ -@Getter public enum Priority { /** * 0 highest priority @@ -46,4 +44,11 @@ public enum Priority { private final int code; private final String descp; + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceType.java index 1b8c47bf92..043402c2ae 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * resource type */ -@Getter public enum ResourceType { /** * 0 file, 1 udf @@ -39,4 +37,12 @@ public enum ResourceType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ShowType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ShowType.java index c7d8e64a3e..19e552d765 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ShowType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ShowType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * show type for email */ -@Getter public enum ShowType { /** * 0 TABLE; @@ -44,4 +42,12 @@ public enum ShowType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java index f924825f19..45f36883e3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * task node type */ -@Getter public enum TaskType { /** * 0 SHELL @@ -61,4 +59,11 @@ public enum TaskType { return !(taskType == TaskType.SUB_PROCESS || taskType == TaskType.DEPENDENT); } + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java index 229a9bccd5..75a5df6fb9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * user type */ -@Getter public enum UserType { /** * 0 admin user; 1 general user @@ -39,5 +37,13 @@ public enum UserType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } From 8b2b5ba684090233294e23bbfafc57fbfffd5d4d Mon Sep 17 00:00:00 2001 From: Tboy Date: Fri, 27 Dec 2019 17:00:06 +0800 Subject: [PATCH 11/16] refactor TaskQueueZkImpl (#1591) * fix #1515 * sleep when resource in not satisfy. fix #1522 * add sleep 1s for no command * fix MasterBaseTaskExecThread submit method bug * updates * add log * delete lombok * remove duplicate code * refactor TaskQueueZkImpl * ignore First , we have to rewrite * updates --- .../common/queue/TaskQueueFactory.java | 3 +- .../common/queue/TaskQueueZkImpl.java | 163 +++--------------- .../utils/SpringApplicationContext.java | 2 +- .../common/zk/ZookeeperOperator.java | 3 - .../common/queue/TaskQueueZKImplTest.java | 4 +- .../dolphinscheduler/dao/ProcessDao.java | 3 +- .../dao/mapper/Application.java | 2 +- .../server/master/MasterServer.java | 3 +- .../runner/MasterBaseTaskExecThread.java | 2 +- .../master/runner/MasterExecThread.java | 1 - .../master/runner/MasterSchedulerThread.java | 2 +- .../server/worker/WorkerServer.java | 2 +- .../server/worker/runner/FetchTaskThread.java | 3 +- .../server/worker/task/AbstractYarnTask.java | 2 +- .../task/dependent/DependentExecute.java | 2 +- .../worker/task/dependent/DependentTask.java | 2 +- .../server/worker/task/http/HttpTask.java | 2 +- .../task/processdure/ProcedureTask.java | 2 +- .../server/worker/task/python/PythonTask.java | 2 +- .../server/worker/task/shell/ShellTask.java | 2 +- .../server/worker/task/sql/SqlTask.java | 2 +- .../shell/ShellCommandExecutorTest.java | 2 +- .../server/worker/sql/SqlExecutorTest.java | 2 +- 23 files changed, 46 insertions(+), 167 deletions(-) rename {dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common}/utils/SpringApplicationContext.java (96%) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java index a82a3098f2..0a2d943118 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.queue; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +44,7 @@ public class TaskQueueFactory { String queueImplValue = CommonUtils.getQueueImplValue(); if (StringUtils.isNotBlank(queueImplValue)) { logger.info("task queue impl use zookeeper "); - return TaskQueueZkImpl.getInstance(); + return SpringApplicationContext.getBean(TaskQueueZkImpl.class); }else{ logger.error("property dolphinscheduler.queue.impl can't be blank, system will exit "); System.exit(-1); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java index 5eca993120..45c6122341 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java @@ -17,22 +17,14 @@ package org.apache.dolphinscheduler.common.queue; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.Bytes; import org.apache.dolphinscheduler.common.utils.IpUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.zk.DefaultEnsembleProvider; -import org.apache.dolphinscheduler.common.zk.ZookeeperConfig; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.data.Stat; +import org.apache.dolphinscheduler.common.zk.ZookeeperOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; import java.util.*; @@ -40,35 +32,13 @@ import java.util.*; * A singleton of a task queue implemented with zookeeper * tasks queue implemention */ +@Service public class TaskQueueZkImpl implements ITaskQueue { private static final Logger logger = LoggerFactory.getLogger(TaskQueueZkImpl.class); - private static volatile TaskQueueZkImpl instance; - - private CuratorFramework zkClient; - - private ZookeeperConfig zookeeperConfig; - - private CuratorFramework getZkClient() { - return zkClient; - } - - private TaskQueueZkImpl(){ - init(); - } - - public static TaskQueueZkImpl getInstance(){ - if (null == instance) { - synchronized (TaskQueueZkImpl.class) { - if(null == instance) { - instance = new TaskQueueZkImpl(); - } - } - } - return instance; - } - + @Autowired + private ZookeeperOperator zookeeperOperator; /** * get all tasks from tasks queue @@ -78,14 +48,12 @@ public class TaskQueueZkImpl implements ITaskQueue { @Override public List getAllTasks(String key) { try { - List list = getZkClient().getChildren().forPath(getTasksPath(key)); - + List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); return list; } catch (Exception e) { logger.error("get all tasks from tasks queue exception",e); } - - return new ArrayList(); + return new ArrayList<>(); } /** @@ -99,22 +67,8 @@ public class TaskQueueZkImpl implements ITaskQueue { public boolean checkTaskExists(String key, String task) { String taskPath = getTasksPath(key) + Constants.SINGLE_SLASH + task; - try { - Stat stat = zkClient.checkExists().forPath(taskPath); - - if(null == stat){ - logger.info("check task:{} not exist in task queue",task); - return false; - }else{ - logger.info("check task {} exists in task queue ",task); - return true; - } + return zookeeperOperator.isExisted(taskPath); - } catch (Exception e) { - logger.info(String.format("task {} check exists in task queue exception ", task), e); - } - - return false; } @@ -128,9 +82,7 @@ public class TaskQueueZkImpl implements ITaskQueue { public boolean add(String key, String value){ try { String taskIdPath = getTasksPath(key) + Constants.SINGLE_SLASH + value; - String result = getZkClient().create().withMode(CreateMode.PERSISTENT).forPath(taskIdPath, Bytes.toBytes(value)); - - logger.info("add task : {} to tasks queue , result success",result); + zookeeperOperator.persist(taskIdPath, value); return true; } catch (Exception e) { logger.error("add task to tasks queue exception",e); @@ -153,8 +105,7 @@ public class TaskQueueZkImpl implements ITaskQueue { @Override public List poll(String key, int tasksNum) { try{ - CuratorFramework zk = getZkClient(); - List list = zk.getChildren().forPath(getTasksPath(key)); + List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); if(list != null && list.size() > 0){ @@ -277,15 +228,12 @@ public class TaskQueueZkImpl implements ITaskQueue { @Override public void removeNode(String key, String nodeValue){ - CuratorFramework zk = getZkClient(); String tasksQueuePath = getTasksPath(key) + Constants.SINGLE_SLASH; String taskIdPath = tasksQueuePath + nodeValue; - logger.info("consume task {}", taskIdPath); + logger.info("removeNode task {}", taskIdPath); try{ - Stat stat = zk.checkExists().forPath(taskIdPath); - if(stat != null){ - zk.delete().forPath(taskIdPath); - } + zookeeperOperator.remove(taskIdPath); + }catch(Exception e){ logger.error(String.format("delete task:%s from zookeeper fail, exception:" ,nodeValue) ,e); } @@ -307,13 +255,10 @@ public class TaskQueueZkImpl implements ITaskQueue { if(value != null && value.trim().length() > 0){ String path = getTasksPath(key) + Constants.SINGLE_SLASH; - CuratorFramework zk = getZkClient(); - Stat stat = zk.checkExists().forPath(path + value); - - if(null == stat){ - String result = zk.create().withMode(CreateMode.PERSISTENT).forPath(path + value,Bytes.toBytes(value)); - logger.info("add task:{} to tasks set result:{} ",value,result); - }else{ + if(!zookeeperOperator.isExisted(path + value)){ + zookeeperOperator.persist(path + value,value); + logger.info("add task:{} to tasks set ",value); + } else{ logger.info("task {} exists in tasks set ",value); } @@ -336,15 +281,7 @@ public class TaskQueueZkImpl implements ITaskQueue { public void srem(String key, String value) { try{ String path = getTasksPath(key) + Constants.SINGLE_SLASH; - CuratorFramework zk = getZkClient(); - Stat stat = zk.checkExists().forPath(path + value); - - if(null != stat){ - zk.delete().forPath(path + value); - logger.info("delete task:{} from tasks set ",value); - }else{ - logger.info("delete task:{} from tasks set fail, there is no this task",value); - } + zookeeperOperator.remove(path + value); }catch(Exception e){ logger.error(String.format("delete task:" + value + " exception"),e); @@ -363,7 +300,7 @@ public class TaskQueueZkImpl implements ITaskQueue { Set tasksSet = new HashSet<>(); try { - List list = getZkClient().getChildren().forPath(getTasksPath(key)); + List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); for (String task : list) { tasksSet.add(task); @@ -377,56 +314,6 @@ public class TaskQueueZkImpl implements ITaskQueue { return tasksSet; } - - - /** - * Init the task queue of zookeeper node - */ - private void init(){ - initZkClient(); - try { - String tasksQueuePath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); - String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL); - - for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){ - if(zkClient.checkExists().forPath(taskQueuePath) == null){ - // create a persistent parent node - zkClient.create().creatingParentContainersIfNeeded() - .withMode(CreateMode.PERSISTENT).forPath(taskQueuePath); - logger.info("create tasks queue parent node success : {} ",taskQueuePath); - } - } - - } catch (Exception e) { - logger.error("create zk node failure",e); - } - } - - private void initZkClient() { - - Configuration conf = null; - try { - conf = new PropertiesConfiguration(Constants.ZOOKEEPER_PROPERTIES_PATH); - } catch (ConfigurationException ex) { - logger.error("load zookeeper properties file failed, system exit"); - System.exit(-1); - } - - zkClient = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(conf.getString("zookeeper.quorum"))) - .retryPolicy(new ExponentialBackoffRetry(conf.getInt("zookeeper.retry.base.sleep"), conf.getInt("zookeeper.retry.maxtime"), conf.getInt("zookeeper.retry.max.sleep"))) - .sessionTimeoutMs(conf.getInt("zookeeper.session.timeout")) - .connectionTimeoutMs(conf.getInt("zookeeper.connection.timeout")) - .build(); - - zkClient.start(); - try { - zkClient.blockUntilConnected(); - } catch (final Exception ex) { - throw new RuntimeException(ex); - } - } - - /** * Clear the task queue of zookeeper node */ @@ -437,16 +324,12 @@ public class TaskQueueZkImpl implements ITaskQueue { String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL); for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){ - if(zkClient.checkExists().forPath(taskQueuePath) != null){ - - List list = zkClient.getChildren().forPath(taskQueuePath); - + if(zookeeperOperator.isExisted(taskQueuePath)){ + List list = zookeeperOperator.getChildrenKeys(taskQueuePath); for (String task : list) { - zkClient.delete().forPath(taskQueuePath + Constants.SINGLE_SLASH + task); + zookeeperOperator.remove(taskQueuePath + Constants.SINGLE_SLASH + task); logger.info("delete task from tasks queue : {}/{} ",taskQueuePath,task); - } - } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SpringApplicationContext.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java similarity index 96% rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SpringApplicationContext.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java index 96087e5a52..97618e1b39 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SpringApplicationContext.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.utils; +package org.apache.dolphinscheduler.common.utils; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java index f4d72f436e..5e3751b25d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java @@ -127,9 +127,6 @@ public class ZookeeperOperator implements InitializingBean { List values; try { values = zkClient.getChildren().forPath(key); - if (CollectionUtils.isEmpty(values)) { - logger.warn("getChildrenKeys key : {} is empty", key); - } return values; } catch (InterruptedException ex) { logger.error("getChildrenKeys key : {} InterruptedException", key); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java index b13f4f63c5..1b44673149 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java @@ -32,11 +32,9 @@ import static org.junit.Assert.*; /** * task queue test */ +@Ignore public class TaskQueueZKImplTest extends BaseTaskQueueTest { - - - @Before public void before(){ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java index eb97ad75b2..66948951f6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java @@ -105,7 +105,8 @@ public class ProcessDao { /** * task queue impl */ - protected ITaskQueue taskQueue = TaskQueueFactory.getTaskQueueInstance(); + @Autowired + private ITaskQueue taskQueue; /** * handle Command (construct ProcessInstance from Command) , wrapped in transaction * @param logger logger diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java index 75b79a4266..02df228f72 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java @@ -21,7 +21,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; @SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler.dao") +@ComponentScan("org.apache.dolphinscheduler") public class Application { public static void main(String[] args) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 8297cd0403..e8c8b6779e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -23,18 +23,17 @@ import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread; import org.apache.dolphinscheduler.server.quartz.ProcessScheduleJob; import org.apache.dolphinscheduler.server.quartz.QuartzExecutors; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.annotation.ComponentScan; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index 6b4b799ef9..5c96757072 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -18,13 +18,13 @@ package org.apache.dolphinscheduler.server.master.runner; import org.apache.dolphinscheduler.common.queue.ITaskQueue; import org.apache.dolphinscheduler.common.queue.TaskQueueFactory; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.utils.BeanContext; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 5446830780..3481b79caa 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -34,7 +34,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.utils.AlertManager; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java index 8d7d5a0add..5f594b3fa0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java @@ -22,12 +22,12 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 96f5ba0b5d..877d60a33b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -29,12 +29,12 @@ import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.FetchTaskThread; import org.apache.dolphinscheduler.server.zk.ZKWorkerClient; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java index 7429050605..ae67716da2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java @@ -25,12 +25,12 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.WorkerGroup; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.zk.ZKWorkerClient; import org.slf4j.Logger; @@ -155,6 +155,7 @@ public class FetchTaskThread implements Runnable{ //whether have tasks, if no tasks , no need lock //get all tasks List tasksQueueList = taskQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); if (CollectionUtils.isEmpty(tasksQueueList)){ + Thread.sleep(Constants.SLEEP_TIME_MILLIS); continue; } // creating distributed locks, lock path /dolphinscheduler/lock/worker diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java index b9b3ad6824..6846617408 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.server.worker.task; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java index 5dc25b8935..4be65ed49d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java @@ -23,10 +23,10 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.DependentItem; import org.apache.dolphinscheduler.common.utils.DependentUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java index b0bb4c6f4c..9af29e01dd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java @@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.DependentUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index 44eef65aba..993310f6ec 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -30,10 +30,10 @@ import org.apache.dolphinscheduler.common.task.http.HttpParameters; import org.apache.dolphinscheduler.common.utils.Bytes; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.http.HttpEntity; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index 59cf8a6e24..9b4952bbd2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -29,10 +29,10 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java index f6227b15a4..585d62f154 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java @@ -22,9 +22,9 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.python.PythonParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.PythonCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index 438d373775..789a0c5302 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -23,9 +23,9 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index ccfee2efec..08a90c62ce 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.common.task.sql.SqlType; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.DataSource; @@ -43,7 +44,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.utils.UDFUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 71bebe2990..1117fe0015 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -20,10 +20,10 @@ import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.LoggerUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java index 725f2835e9..7cf4b874d1 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java @@ -21,10 +21,10 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.LoggerUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; From 597d685955a3d3028039103edeb1b723077b3b7d Mon Sep 17 00:00:00 2001 From: samz406 Date: Fri, 27 Dec 2019 17:00:33 +0800 Subject: [PATCH 12/16] delAlertgroupById add entity exist check (#1602) * misspell words * modify common queue TaskQueueZKImplTest.java unit test * extends BaseTaskQueueTest get zkServer * modify zk config * add MonitorServiceTest * mobile phone need 11 number * delete file * delAlertgroupById add entity exist check --- .../dolphinscheduler/api/service/AlertGroupService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java index 40fc65b4dc..63f50f936f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java @@ -193,6 +193,12 @@ public class AlertGroupService extends BaseService{ if (checkAdmin(loginUser, result)){ return result; } + //check exist + AlertGroup alertGroup = alertGroupMapper.selectById(id); + if (alertGroup == null) { + putMsg(result, Status.ALERT_GROUP_NOT_EXIST); + return result; + } userAlertGroupMapper.deleteByAlertgroupId(id); alertGroupMapper.deleteById(id); From 63f396b61f77cc44e462d35928f46ef2710adfd2 Mon Sep 17 00:00:00 2001 From: "DK.Pino" Date: Fri, 27 Dec 2019 17:03:10 +0800 Subject: [PATCH 13/16] refactor zk tree cache (#1577) * refactor zk tree cache * refactor zk tree cache --- .../common/zk/AbstractListener.java | 35 ------- .../common/zk/ZookeeperCachedOperator.java | 48 ++++++---- .../common/zk/ZookeeperOperator.java | 8 +- .../server/zk/ZKMasterClient.java | 96 +++++++++---------- .../server/zk/ZKWorkerClient.java | 61 ++++++------ 5 files changed, 107 insertions(+), 141 deletions(-) delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java deleted file mode 100644 index d84b9f7e11..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.zk; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.TreeCacheEvent; -import org.apache.curator.framework.recipes.cache.TreeCacheListener; - -public abstract class AbstractListener implements TreeCacheListener { - - @Override - public final void childEvent(final CuratorFramework client, final TreeCacheEvent event) throws Exception { - String path = null == event.getData() ? "" : event.getData().getPath(); - if (path.isEmpty()) { - return; - } - dataChanged(client, event, path); - } - - protected abstract void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path); -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java index cf4980147e..daec765315 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java @@ -16,8 +16,10 @@ */ package org.apache.dolphinscheduler.common.zk; +import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.TreeCache; +import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.curator.framework.recipes.cache.TreeCacheListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,30 +36,37 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { private final Logger logger = LoggerFactory.getLogger(ZookeeperCachedOperator.class); - //kay is zk path, value is TreeCache - private ConcurrentHashMap allCaches = new ConcurrentHashMap<>(); + TreeCache treeCache; /** - * @param cachePath zk path - * @param listener operator + * register a unified listener of /${dsRoot}, */ - public void registerListener(final String cachePath, final TreeCacheListener listener) { - TreeCache newCache = new TreeCache(zkClient, cachePath); - logger.info("add listener to zk path: {}", cachePath); + @Override + protected void registerListener() { + treeCache = new TreeCache(zkClient, getZookeeperConfig().getDsRoot()); + logger.info("add listener to zk path: {}", getZookeeperConfig().getDsRoot()); try { - newCache.start(); + treeCache.start(); } catch (Exception e) { - logger.error("add listener to zk path: {} failed", cachePath); + logger.error("add listener to zk path: {} failed", getZookeeperConfig().getDsRoot()); throw new RuntimeException(e); } - newCache.getListenable().addListener(listener); + treeCache.getListenable().addListener((client, event) -> { + String path = null == event.getData() ? "" : event.getData().getPath(); + if (path.isEmpty()) { + return; + } + dataChanged(client, event, path); + }); - allCaches.put(cachePath, newCache); } + //for sub class + protected void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path){} + public String getFromCache(final String cachePath, final String key) { - ChildData resultInCache = allCaches.get(checkNotNull(cachePath)).getCurrentData(key); + ChildData resultInCache = treeCache.getCurrentData(key); if (null != resultInCache) { return null == resultInCache.getData() ? null : new String(resultInCache.getData(), StandardCharsets.UTF_8); } @@ -65,18 +74,15 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { } public TreeCache getTreeCache(final String cachePath) { - return allCaches.get(checkNotNull(cachePath)); + return treeCache; } public void close() { - - allCaches.forEach((path, cache) -> { - cache.close(); - try { - Thread.sleep(500); - } catch (InterruptedException ignore) { - } - }); + treeCache.close(); + try { + Thread.sleep(500); + } catch (InterruptedException ignore) { + } super.close(); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java index 5e3751b25d..c6faec2b78 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java @@ -57,11 +57,13 @@ public class ZookeeperOperator implements InitializingBean { public void afterPropertiesSet() throws Exception { this.zkClient = buildClient(); initStateLister(); - //init(); + registerListener(); } - //for subclass - //protected void init(){} + /** + * this method is for sub class, + */ + protected void registerListener(){} public void initStateLister() { checkNotNull(zkClient); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index 2aec6ecaf6..a26a217665 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.common.zk.AbstractListener; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; @@ -31,9 +30,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.utils.ThreadUtils; import org.slf4j.Logger; @@ -101,12 +97,6 @@ public class ZKMasterClient extends AbstractZKClient { // init system znode this.initSystemZNode(); - // monitor master - this.listenerMaster(); - - // monitor worker - this.listenerWorker(); - // register master this.registerMaster(); @@ -158,31 +148,22 @@ public class ZKMasterClient extends AbstractZKClient { } } - /** - * monitor master + * handle path events that this class cares about + * @param client zkClient + * @param event path event + * @param path zk path */ - public void listenerMaster(){ - registerListener(getZNodeParentPath(ZKNodeType.MASTER), new AbstractListener() { - @Override - protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - switch (event.getType()) { - case NODE_ADDED: - logger.info("master node added : {}", path); - break; - case NODE_REMOVED: - String serverHost = getHostByEventDataPath(path); - if (checkServerSelfDead(serverHost, ZKNodeType.MASTER)) { - return; - } - removeZKNodePath(path, ZKNodeType.MASTER, true); - break; - default: - break; - } - } - }); -} + @Override + protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { + if(path.startsWith(getZNodeParentPath(ZKNodeType.MASTER)+Constants.SINGLE_SLASH)){ //monitor master + handleMasterEvent(event,path); + + }else if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){ //monitor worker + handleWorkerEvent(event,path); + } + //other path event, ignore + } /** * remove zookeeper node path @@ -273,25 +254,40 @@ public class ZKMasterClient extends AbstractZKClient { } /** - * monitor worker + * monitor master */ - public void listenerWorker(){ - registerListener(getZNodeParentPath(ZKNodeType.WORKER), new AbstractListener() { - @Override - protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - switch (event.getType()) { - case NODE_ADDED: - logger.info("worker node added : {}", path); - break; - case NODE_REMOVED: - logger.info("worker node deleted : {}", path); - removeZKNodePath(path, ZKNodeType.WORKER, true); - break; - default: - break; + public void handleMasterEvent(TreeCacheEvent event, String path){ + switch (event.getType()) { + case NODE_ADDED: + logger.info("master node added : {}", path); + break; + case NODE_REMOVED: + String serverHost = getHostByEventDataPath(path); + if (checkServerSelfDead(serverHost, ZKNodeType.MASTER)) { + return; } - } - }); + removeZKNodePath(path, ZKNodeType.MASTER, true); + break; + default: + break; + } + } + + /** + * monitor worker + */ + public void handleWorkerEvent(TreeCacheEvent event, String path){ + switch (event.getType()) { + case NODE_ADDED: + logger.info("worker node added : {}", path); + break; + case NODE_REMOVED: + logger.info("worker node deleted : {}", path); + removeZKNodePath(path, ZKNodeType.WORKER, true); + break; + default: + break; + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java index 0dd1cf15be..2e063d50d5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java @@ -19,20 +19,13 @@ package org.apache.dolphinscheduler.server.zk; import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ZKNodeType; -import org.apache.dolphinscheduler.common.zk.AbstractListener; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; -import org.apache.curator.utils.ThreadUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import java.util.concurrent.ThreadFactory; - /** * zookeeper worker client @@ -61,9 +54,6 @@ public class ZKWorkerClient extends AbstractZKClient { // init system znode this.initSystemZNode(); - // monitor worker - this.listenerWorker(); - // register worker this.registWorker(); } @@ -83,31 +73,38 @@ public class ZKWorkerClient extends AbstractZKClient { System.exit(-1); } } - + /** - * monitor worker + * handle path events that this class cares about + * @param client zkClient + * @param event path event + * @param path zk path */ - private void listenerWorker(){ - registerListener(getZNodeParentPath(ZKNodeType.WORKER), new AbstractListener() { - @Override - protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - switch (event.getType()) { - case NODE_ADDED: - logger.info("worker node added : {}", path); - break; - case NODE_REMOVED: - //find myself dead - String serverHost = getHostByEventDataPath(path); - if(checkServerSelfDead(serverHost, ZKNodeType.WORKER)){ - return; - } - break; - default: - break; - } - } - }); + @Override + protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { + if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){ + handleWorkerEvent(event,path); + } + } + /** + * monitor worker + */ + public void handleWorkerEvent(TreeCacheEvent event, String path){ + switch (event.getType()) { + case NODE_ADDED: + logger.info("worker node added : {}", path); + break; + case NODE_REMOVED: + //find myself dead + String serverHost = getHostByEventDataPath(path); + if(checkServerSelfDead(serverHost, ZKNodeType.WORKER)){ + return; + } + break; + default: + break; + } } /** From b6cca46d8f39993b54f22ae362884537d9499eeb Mon Sep 17 00:00:00 2001 From: "DK.Pino" Date: Fri, 27 Dec 2019 17:04:25 +0800 Subject: [PATCH 14/16] remove lombok annocation (#1603) --- .../common/enums/AlertStatus.java | 10 +++- .../common/enums/CommandType.java | 10 +++- .../dolphinscheduler/common/enums/DbType.java | 60 ++++++++++--------- .../common/enums/ExecutionStatus.java | 8 ++- .../common/enums/SparkVersion.java | 10 +++- .../common/enums/UdfType.java | 10 +++- 6 files changed, 71 insertions(+), 37 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java index 7543dc48cd..42ea05f75d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * alert status */ -@Getter public enum AlertStatus { /** * 0 waiting executed; 1 execute successfully，2 execute failed @@ -40,4 +38,12 @@ public enum AlertStatus { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java index 352f6ef812..1ee79156dc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * command types */ -@Getter public enum CommandType { /** @@ -59,4 +57,12 @@ public enum CommandType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java index 4637771eda..5fb245afef 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java @@ -17,38 +17,44 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * data base types */ -@Getter public enum DbType { - /** - * 0 mysql - * 1 postgresql - * 2 hive - * 3 spark - * 4 clickhouse - * 5 oracle - * 6 sqlserver - * 7 db2 - */ - MYSQL(0, "mysql"), - POSTGRESQL(1, "postgresql"), - HIVE(2, "hive"), - SPARK(3, "spark"), - CLICKHOUSE(4, "clickhouse"), - ORACLE(5, "oracle"), - SQLSERVER(6, "sqlserver"), - DB2(7, "db2"); + /** + * 0 mysql + * 1 postgresql + * 2 hive + * 3 spark + * 4 clickhouse + * 5 oracle + * 6 sqlserver + * 7 db2 + */ + MYSQL(0, "mysql"), + POSTGRESQL(1, "postgresql"), + HIVE(2, "hive"), + SPARK(3, "spark"), + CLICKHOUSE(4, "clickhouse"), + ORACLE(5, "oracle"), + SQLSERVER(6, "sqlserver"), + DB2(7, "db2"); - DbType(int code, String descp){ - this.code = code; - this.descp = descp; - } + DbType(int code, String descp) { + this.code = code; + this.descp = descp; + } - @EnumValue - private final int code; - private final String descp; + @EnumValue + private final int code; + private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java index f29b8a54aa..12702527f0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java @@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * running status for workflow and task nodes * */ -@Getter public enum ExecutionStatus { /** @@ -123,5 +121,11 @@ public enum ExecutionStatus { return this == KILL || this == STOP ; } + public int getCode() { + return code; + } + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SparkVersion.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SparkVersion.java index e3f7c73797..867d063a64 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SparkVersion.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SparkVersion.java @@ -17,9 +17,7 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; -@Getter public enum SparkVersion { /** @@ -37,4 +35,12 @@ public enum SparkVersion { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java index a667c05878..22f6752689 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * UDF type */ -@Getter public enum UdfType { /** * 0 hive; 1 spark @@ -38,4 +36,12 @@ public enum UdfType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } From 7f27c8a86e8d10f30875a74edf0854c68283c873 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 27 Dec 2019 17:05:28 +0800 Subject: [PATCH 15/16] update assemble and rpmbuild because config/*.* remove script to dolphinscheduler-server/src/main/resources (#1609) --- dolphinscheduler-dist/pom.xml | 2 +- .../src/main/assembly/dolphinscheduler-binary.xml | 2 +- .../src/main/assembly/dolphinscheduler-nginx.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 2a7553e3a3..b43daff41b 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -271,6 +271,7 @@ **/*.properties **/*.xml **/*.json + config/*.* @@ -279,7 +280,6 @@ ${basedir}/../script - config/*.* env/*.* diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml index 2bf109698b..b4326c6795 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml @@ -94,6 +94,7 @@ **/*.properties **/*.xml **/*.json + config/*.* conf @@ -162,7 +163,6 @@ ${basedir}/../script - config/*.* env/*.* ./conf diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml index ebe5b2cb3f..f4e403e4b4 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml @@ -94,6 +94,7 @@ **/*.properties **/*.xml **/*.json + config/*.* conf @@ -185,7 +186,6 @@ ${basedir}/../script - config/*.* env/*.* ./conf From be5fc116effe456cb9b74b277a3fb5c3988e6870 Mon Sep 17 00:00:00 2001 From: samz406 Date: Fri, 27 Dec 2019 17:20:34 +0800 Subject: [PATCH 16/16] update resource may NPE (#1610) * misspell words * modify common queue TaskQueueZKImplTest.java unit test * extends BaseTaskQueueTest get zkServer * modify zk config * update resource may NPE --- .../apache/dolphinscheduler/api/service/ResourcesService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 97db9ee1d7..66bf214608 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -210,7 +210,6 @@ public class ResourcesService extends BaseService { } Resource resource = resourcesMapper.selectById(resourceId); - String originResourceName = resource.getAlias(); if (resource == null) { putMsg(result, Status.RESOURCE_NOT_EXIST); return result; @@ -236,6 +235,7 @@ public class ResourcesService extends BaseService { } //get the file suffix + String originResourceName = resource.getAlias(); String suffix = originResourceName.substring(originResourceName.lastIndexOf(".")); //if the name without suffix then add it ,else use the origin name