@@ -211,8 +219,10 @@
})
}
},
- created () {},
- mounted () {},
+ created () {
+ },
+ mounted () {
+ },
components: { mNoData }
}
@@ -221,10 +231,12 @@
.container {
width: 500px;
position: relative;
+
.title-box {
height: 61px;
border-bottom: 1px solid #DCDEDC;
position: relative;
+
.name {
position: absolute;
left: 24px;
@@ -232,6 +244,7 @@
font-size: 16px;
}
}
+
.bottom-box {
position: absolute;
bottom: 0;
@@ -242,10 +255,12 @@
line-height: 60px;
border-top: 1px solid #DCDEDC;
background: #fff;
+
.ans-page {
display: inline-block;
}
}
+
.table-box {
overflow-y: scroll;
height: calc(100vh - 61px);
From bc766aaab8c4d1c972f5208bb18d420aec7e9ce6 Mon Sep 17 00:00:00 2001
From: qingzhongli
Date: Sun, 30 Aug 2020 18:47:14 +0800
Subject: [PATCH 03/18] [Improvement][docs] Fix typo in CONTRIBUTING.md (#3624)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Co-authored-by: 李庆忠
---
CONTRIBUTING.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index e02ed113c4..f26b06e850 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -56,7 +56,7 @@ If remote branch has a new branch `DEV-1.0`, you need to synchronize this branch
```
git checkout -b dev-1.0 upstream/dev-1.0
-git push --set-upstream origin dev1.0
+git push --set-upstream origin dev-1.0
```
## Create your feature branch
From 908dba4ba67c8fb319890a731ecd12061af9fc58 Mon Sep 17 00:00:00 2001
From: zhuangchong <37063904+zhuangchong@users.noreply.github.com>
Date: Mon, 31 Aug 2020 16:54:45 +0800
Subject: [PATCH 04/18] [Improvement][install,start-all,stop-all] start- all.sh
and stop- all.sh After execution, you can view the running status of each
node and service. #3619 (#3627)
* start- all.sh and stop- all.sh After execution, you can view the running status of each node and service.
* ds server state check.
---
install.sh | 4 ---
script/dolphinscheduler-daemon.sh | 19 +++++++---
script/start-all.sh | 6 +++-
script/status-all.sh | 59 +++++++++++++++++++++++++------
script/stop-all.sh | 6 +++-
5 files changed, 73 insertions(+), 21 deletions(-)
diff --git a/install.sh b/install.sh
index ae16a1d199..da21085d14 100755
--- a/install.sh
+++ b/install.sh
@@ -98,7 +98,3 @@ sh ${workDir}/script/remove-zk-node.sh $zkRoot
# 6.startup
echo "6.startup"
sh ${workDir}/script/start-all.sh
-
-# 7.query status
-echo "7.query status"
-sh ${workDir}/script/status-all.sh
diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh
index 8c96764336..26c751f6e5 100644
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@@ -16,7 +16,7 @@
# limitations under the License.
#
-usage="Usage: dolphinscheduler-daemon.sh (start|stop) "
+usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) "
# if no args specified, show usage
if [ $# -le 1 ]; then
@@ -29,7 +29,6 @@ shift
command=$1
shift
-echo "Begin $startStop $command......"
BIN_DIR=`dirname $0`
BIN_DIR=`cd "$BIN_DIR"; pwd`
@@ -122,11 +121,23 @@ case $startStop in
fi
;;
+ (status)
+ # more details about the status can be added later
+ serverCount=`ps -ef |grep "$CLASS" |grep -v "grep" |wc -l`
+ state="STOP"
+ # font color - red
+ state="[ \033[1;31m $state \033[0m ]"
+ if [[ $serverCount -gt 0 ]];then
+ state="RUNNING"
+ # font color - green
+ state="[ \033[1;32m $state \033[0m ]"
+ fi
+ echo -e "$command $state"
+ ;;
+
(*)
echo $usage
exit 1
;;
esac
-
-echo "End $startStop $command."
diff --git a/script/start-all.sh b/script/start-all.sh
index 61b916483e..d0c0ab8188 100644
--- a/script/start-all.sh
+++ b/script/start-all.sh
@@ -54,4 +54,8 @@ for apiServer in ${apiServersHost[@]}
do
echo "$apiServer worker server is starting"
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;"
-done
\ No newline at end of file
+done
+
+# query server status
+echo "query server status"
+cd $installPath/; sh bin/status-all.sh
diff --git a/script/status-all.sh b/script/status-all.sh
index 64dfaa46b8..555a2c1ead 100644
--- a/script/status-all.sh
+++ b/script/status-all.sh
@@ -20,22 +20,59 @@ workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
source $workDir/../conf/config/install_config.conf
+# install_config.conf info
echo -e '\n'
-echo "====================== dolphinscheduler install config============================="
-echo -e "1.dolphinscheduler server node install hosts:[ \033[1;32m ${ips} \033[0m ]"
-echo -e "2.master server node install hosts:[ \033[1;32m ${masters} \033[0m ]"
-echo -e "3.worker server node install hosts:[ \033[1;32m ${workers} \033[0m ]"
-echo -e "4.alert server node install hosts:[ \033[1;32m ${alertServer} \033[0m ]"
-echo -e "5.api server node install hosts:[ \033[1;32m ${apiServers} \033[0m ]"
+echo "====================== dolphinscheduler server config ============================="
+echo -e "1.dolphinscheduler server node config hosts:[ \033[1;32m ${ips} \033[0m ]"
+echo -e "2.master server node config hosts:[ \033[1;32m ${masters} \033[0m ]"
+echo -e "3.worker server node config hosts:[ \033[1;32m ${workers} \033[0m ]"
+echo -e "4.alert server node config hosts:[ \033[1;32m ${alertServer} \033[0m ]"
+echo -e "5.api server node config hosts:[ \033[1;32m ${apiServers} \033[0m ]"
+# all server check state
echo -e '\n'
+echo "====================== dolphinscheduler server status ============================="
+firstColumn="node server state"
+echo $firstColumn
+echo -e '\n'
+
+declare -A workersGroupMap=()
+workersGroup=(${workers//,/ })
+for workerGroup in ${workersGroup[@]}
+do
+ worker=`echo $workerGroup|awk -F':' '{print $1}'`
+ groupName=`echo $workerGroup|awk -F':' '{print $2}'`
+ workersGroupMap+=([$worker]=$groupName)
+done
-ipsHost=(${ips//,/ })
-for ip in ${ipsHost[@]}
+StateRunning="Running"
+# 1.master server check state
+mastersHost=(${masters//,/ })
+for master in ${mastersHost[@]}
do
- echo -e "====================== [ \033[1;32m ${ip} \033[0m ] node all servers =========================="
- ssh -p $sshPort $ip "jps"
- echo -e '\n'
+ masterState=`ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status master-server;"`
+ echo "$master $masterState"
done
+# 2.worker server and logger-server check state
+for worker in ${!workersGroupMap[*]}
+do
+ workerState=`ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status worker-server;"`
+ echo "$worker $workerState"
+
+ masterState=`ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status logger-server;"`
+ echo "$worker $masterState"
+done
+
+# 3.alter server check state
+alertState=`ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status alert-server;"`
+echo "$alertServer $alertState"
+
+# 4.api server check state
+apiServersHost=(${apiServers//,/ })
+for apiServer in ${apiServersHost[@]}
+do
+ apiState=`ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status api-server;"`
+ echo "$apiServer $apiState"
+done
diff --git a/script/stop-all.sh b/script/stop-all.sh
index dc18dae586..20175e9a51 100644
--- a/script/stop-all.sh
+++ b/script/stop-all.sh
@@ -54,4 +54,8 @@ for apiServer in ${apiServersHost[@]}
do
echo "$apiServer worker server is stopping"
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
-done
\ No newline at end of file
+done
+
+# query server status
+echo "query server status"
+cd $installPath/; sh bin/status-all.sh
From fbeadfdd58d52fdf66d034296fff613a35daae32 Mon Sep 17 00:00:00 2001
From: geosmart
Date: Mon, 31 Aug 2020 16:58:56 +0800
Subject: [PATCH 05/18] [FIX][UI]add button disabeld when isDetails (#3611)
---
.../src/js/conf/home/pages/dag/_source/udp/udp.vue | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue
index 017a38f31d..18cf86a2b6 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue
@@ -82,7 +82,7 @@
{{$t('Cancel')}}
- {{$t('Add')}}
+ {{$t('Add')}}
From 632d52cd69506b5e307c14cc3a687af6a739a0b8 Mon Sep 17 00:00:00 2001
From: CalvinKirs
Date: Mon, 31 Aug 2020 20:35:45 +0800
Subject: [PATCH 06/18] [refactor][server-test] refactor
RoundRobinHostManagerTest (#3511)
* [refactor][server-test] refactor RoundRobinHostManagerTest
* add RoundRobinHostManagerTest
* format code
* format code
---
.../host/RoundRobinHostManagerTest.java | 46 +++++++------------
pom.xml | 1 +
2 files changed, 17 insertions(+), 30 deletions(-)
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
index 21780635ac..ab2cc12e20 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
@@ -14,67 +14,53 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.master.dispatch.host;
+import com.google.common.collect.Sets;
-import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
-import org.apache.dolphinscheduler.server.registry.DependencyConfig;
import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager;
-import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter;
import org.apache.dolphinscheduler.server.utils.ExecutionContextTestUtils;
-import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
-import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry;
-import org.apache.dolphinscheduler.server.zk.SpringZKServer;
-import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient;
-import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
-import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
+
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
/**
* round robin host manager test
*/
-@RunWith(SpringRunner.class)
-@ContextConfiguration(classes={DependencyConfig.class, SpringZKServer.class, WorkerRegistry.class, ZookeeperRegistryCenter.class, WorkerConfig.class,
- ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, CuratorZookeeperClient.class})
+@RunWith(MockitoJUnitRunner.class)
public class RoundRobinHostManagerTest {
- @Autowired
+ @Mock
private ZookeeperNodeManager zookeeperNodeManager;
- @Autowired
- private WorkerRegistry workerRegistry;
-
- @Autowired
- private WorkerConfig workerConfig;
+ @InjectMocks
+ RoundRobinHostManager roundRobinHostManager;
@Test
- public void testSelectWithEmptyResult(){
- RoundRobinHostManager roundRobinHostManager = new RoundRobinHostManager();
- roundRobinHostManager.setZookeeperNodeManager(zookeeperNodeManager);
+ public void testSelectWithEmptyResult() {
+ Mockito.when(zookeeperNodeManager.getWorkerGroupNodes("default")).thenReturn(null);
ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000);
Host emptyHost = roundRobinHostManager.select(context);
Assert.assertTrue(StringUtils.isEmpty(emptyHost.getAddress()));
}
@Test
- public void testSelectWithResult(){
- workerRegistry.registry();
- RoundRobinHostManager roundRobinHostManager = new RoundRobinHostManager();
- roundRobinHostManager.setZookeeperNodeManager(zookeeperNodeManager);
+ public void testSelectWithResult() {
+ Mockito.when(zookeeperNodeManager.getWorkerGroupNodes("default")).thenReturn(Sets.newHashSet("192.168.1.1:22:100"));
ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000);
Host host = roundRobinHostManager.select(context);
Assert.assertTrue(StringUtils.isNotEmpty(host.getAddress()));
- Assert.assertTrue(host.getAddress().equalsIgnoreCase(NetUtils.getHost() + ":" + workerConfig.getListenPort()));
- workerRegistry.unRegistry();
+ Assert.assertTrue(host.getAddress().equalsIgnoreCase("192.168.1.1:22"));
}
}
diff --git a/pom.xml b/pom.xml
index 0c20bd873f..32b2ccf16b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -821,6 +821,7 @@
**/server/master/dispatch/host/assign/RandomSelectorTest.java**/server/master/dispatch/host/assign/RoundRobinSelectorTest.java**/server/master/register/MasterRegistryTest.java
+ **/server/master/dispatch/host/assign/RoundRobinHostManagerTest.java**/server/master/AlertManagerTest.java**/server/master/MasterCommandTest.java**/server/master/DependentTaskTest.java
From ac4ed94061e91563268c3bc588c554182d22d277 Mon Sep 17 00:00:00 2001
From: Yichao Yang <1048262223@qq.com>
Date: Wed, 2 Sep 2020 15:59:47 +0800
Subject: [PATCH 07/18] [Improvement][common] Add UnsupportedOperationException
for utils construct (#3381)
* [Improvement][common] Add UnsupportedOperationException for utils construct
* Fix checkstyle
---
.../dolphinscheduler/common/Constants.java | 3 +-
.../common/utils/CollectionUtils.java | 32 +-
.../common/utils/CommonUtils.java | 205 ++---
.../common/utils/ConnectionUtils.java | 43 +-
.../common/utils/DateUtils.java | 4 +
.../common/utils/DependentUtils.java | 41 +-
.../common/utils/EncryptionUtils.java | 7 +-
.../common/utils/EnumUtils.java | 7 +-
.../common/utils/FileUtils.java | 96 +-
.../common/utils/HttpUtils.java | 224 ++---
.../common/utils/IOUtils.java | 10 +-
.../common/utils/IpUtils.java | 72 +-
.../common/utils/JSONUtils.java | 62 +-
.../common/utils/LoggerUtils.java | 32 +-
.../common/utils/NetUtils.java | 25 +-
.../common/utils/OSUtils.java | 818 +++++++++---------
.../common/utils/ParameterUtils.java | 406 +++++----
.../common/utils/Preconditions.java | 19 +-
.../common/utils/PropertyUtils.java | 48 +-
.../common/utils/RetryerUtils.java | 14 +-
.../common/utils/SchemaUtils.java | 222 ++---
.../common/utils/SensitiveLogUtils.java | 10 +-
.../common/utils/StreamUtils.java | 5 +-
.../common/utils/StringUtils.java | 12 +-
.../common/utils/TaskParametersUtils.java | 100 ++-
25 files changed, 1313 insertions(+), 1204 deletions(-)
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index cee83e73bc..3b12748888 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
@@ -27,7 +28,7 @@ import java.util.regex.Pattern;
public final class Constants {
private Constants() {
- throw new IllegalStateException("Constants class");
+ throw new UnsupportedOperationException("Construct Constants");
}
/**
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
index d900f0f6bf..bc7c93af3d 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
@@ -14,13 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.beanutils.BeanMap;
-import org.apache.commons.lang.StringUtils;
-
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
* Provides utility methods and decorators for {@link Collection} instances.
@@ -37,8 +43,9 @@ import java.util.*;
public class CollectionUtils {
private CollectionUtils() {
- throw new IllegalStateException("CollectionUtils class");
+ throw new UnsupportedOperationException("Construct CollectionUtils");
}
+
/**
* Returns a new {@link Collection} containing a minus a subset of
* b. Only the elements of b that satisfy the predicate
@@ -71,7 +78,7 @@ public class CollectionUtils {
/**
* String to map
*
- * @param str string
+ * @param str string
* @param separator separator
* @return string to map
*/
@@ -82,7 +89,7 @@ public class CollectionUtils {
/**
* String to map
*
- * @param str string
+ * @param str string
* @param separator separator
* @param keyPrefix prefix
* @return string to map
@@ -112,7 +119,6 @@ public class CollectionUtils {
return map;
}
-
/**
* Helper class to easily access cardinality properties of two collections.
*
@@ -137,8 +143,8 @@ public class CollectionUtils {
* @param b the second collection
*/
public CardinalityHelper(final Iterable extends O> a, final Iterable extends O> b) {
- cardinalityA = CollectionUtils.getCardinalityMap(a);
- cardinalityB = CollectionUtils.getCardinalityMap(b);
+ cardinalityA = CollectionUtils.getCardinalityMap(a);
+ cardinalityB = CollectionUtils.getCardinalityMap(b);
}
/**
@@ -227,7 +233,7 @@ public class CollectionUtils {
* Only those elements present in the collection will appear as
* keys in the map.
*
- * @param the type of object in the returned {@link Map}. This is a super type of O
+ * @param the type of object in the returned {@link Map}. This is a super type of O
* @param coll the collection to get the cardinality map for, must not be null
* @return the populated cardinality map
*/
@@ -239,9 +245,9 @@ public class CollectionUtils {
return count;
}
-
/**
* Removes certain attributes of each object in the list
+ *
* @param originList origin list
* @param exclusionSet exclusion set
* @param T
@@ -258,8 +264,8 @@ public class CollectionUtils {
Map instanceMap;
for (T instance : originList) {
Map dataMap = new BeanMap(instance);
- instanceMap = new LinkedHashMap<>(16,0.75f,true);
- for (Map.Entry entry: dataMap.entrySet()) {
+ instanceMap = new LinkedHashMap<>(16, 0.75f, true);
+ for (Map.Entry entry : dataMap.entrySet()) {
if (exclusionSet.contains(entry.getKey())) {
continue;
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
index 6722c23037..45c5aa2c93 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
@@ -14,124 +14,129 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils;
-import org.apache.commons.codec.binary.Base64;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
+
+import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* common utils
*/
public class CommonUtils {
- private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
-
- private static final Base64 BASE64 = new Base64();
-
- private CommonUtils() {
- throw new IllegalStateException("CommonUtils class");
- }
-
- /**
- * @return get the path of system environment variables
- */
- public static String getSystemEnvPath() {
- String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH);
- if (StringUtils.isEmpty(envPath)) {
- URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH);
-
- if (envDefaultPath != null){
- envPath = envDefaultPath.getPath();
- logger.debug("env path :{}", envPath);
- }else{
- envPath = "/etc/profile";
- }
+ private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
+
+ private static final Base64 BASE64 = new Base64();
+
+ private CommonUtils() {
+ throw new UnsupportedOperationException("Construct CommonUtils");
}
- return envPath;
- }
-
- /**
- *
- * @return is develop mode
- */
- public static boolean isDevelopMode() {
- return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true);
- }
-
-
-
- /**
- * if upload resource is HDFS and kerberos startup is true , else false
- * @return true if upload resource is HDFS and kerberos startup
- */
- public static boolean getKerberosStartupState(){
- String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
- ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
- Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false);
- return resUploadType == ResUploadType.HDFS && kerberosStartupState;
- }
-
- /**
- * load kerberos configuration
- * @throws Exception errors
- */
- public static void loadKerberosConf()throws Exception{
- if (CommonUtils.getKerberosStartupState()) {
- System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
- Configuration configuration = new Configuration();
- configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS);
- UserGroupInformation.setConfiguration(configuration);
- UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
- PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
+ /**
+ * @return get the path of system environment variables
+ */
+ public static String getSystemEnvPath() {
+ String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH);
+ if (StringUtils.isEmpty(envPath)) {
+ URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH);
+
+ if (envDefaultPath != null) {
+ envPath = envDefaultPath.getPath();
+ logger.debug("env path :{}", envPath);
+ } else {
+ envPath = "/etc/profile";
+ }
+ }
+
+ return envPath;
}
- }
-
- /**
- * encode password
- * @param password
- * @return
- */
- public static String encodePassword(String password) {
- if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; }
- //if encryption is not turned on, return directly
- boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
- if ( !encryptionEnable){ return password; }
-
- // Using Base64 + salt to process password
- String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
- String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))) ;
- return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
- }
-
- /**
- * decode password
- * @param password
- * @return
- */
- public static String decodePassword(String password) {
- if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; }
-
- //if encryption is not turned on, return directly
- boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
- if ( !encryptionEnable){ return password; }
-
- // Using Base64 + salt to process password
- String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
- String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ;
- if(!passwordWithSalt.startsWith(salt)){
- logger.warn("There is a password and salt mismatch: {} ",password);
- return password;
+
+ /**
+ * @return is develop mode
+ */
+ public static boolean isDevelopMode() {
+ return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true);
}
- return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ;
- }
+ /**
+ * if upload resource is HDFS and kerberos startup is true , else false
+ *
+ * @return true if upload resource is HDFS and kerberos startup
+ */
+ public static boolean getKerberosStartupState() {
+ String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
+ ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
+ Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
+ return resUploadType == ResUploadType.HDFS && kerberosStartupState;
+ }
+
+ /**
+ * load kerberos configuration
+ *
+ * @throws Exception errors
+ */
+ public static void loadKerberosConf() throws Exception {
+ if (CommonUtils.getKerberosStartupState()) {
+ System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
+ Configuration configuration = new Configuration();
+ configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS);
+ UserGroupInformation.setConfiguration(configuration);
+ UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
+ PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
+ }
+ }
+
+ /**
+ * encode password
+ */
+ public static String encodePassword(String password) {
+ if (StringUtils.isEmpty(password)) {
+ return StringUtils.EMPTY;
+ }
+ //if encryption is not turned on, return directly
+ boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false);
+ if (!encryptionEnable) {
+ return password;
+ }
+
+ // Using Base64 + salt to process password
+ String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
+ String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8)));
+ return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
+ }
+
+ /**
+ * decode password
+ */
+ public static String decodePassword(String password) {
+ if (StringUtils.isEmpty(password)) {
+ return StringUtils.EMPTY;
+ }
+
+ //if encryption is not turned on, return directly
+ boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false);
+ if (!encryptionEnable) {
+ return password;
+ }
+
+ // Using Base64 + salt to process password
+ String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
+ String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8);
+ if (!passwordWithSalt.startsWith(salt)) {
+ logger.warn("There is a password and salt mismatch: {} ", password);
+ return password;
+ }
+ return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8);
+ }
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java
index f8ea0e7188..f0cd8f2fe0 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java
@@ -14,37 +14,40 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils;
import java.util.Arrays;
import java.util.Objects;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ConnectionUtils {
- public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class);
+ public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class);
- private ConnectionUtils() {
- throw new IllegalStateException("ConnectionUtils class");
- }
+ private ConnectionUtils() {
+ throw new UnsupportedOperationException("Construct ConnectionUtils");
+ }
- /**
- * release resource
- * @param resources resources
- */
- public static void releaseResource(AutoCloseable... resources) {
+ /**
+ * release resource
+ *
+ * @param resources resources
+ */
+ public static void releaseResource(AutoCloseable... resources) {
- if (resources == null || resources.length == 0) {
- return;
+ if (resources == null || resources.length == 0) {
+ return;
+ }
+ Arrays.stream(resources).filter(Objects::nonNull)
+ .forEach(resource -> {
+ try {
+ resource.close();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ });
}
- Arrays.stream(resources).filter(Objects::nonNull)
- .forEach(resource -> {
- try {
- resource.close();
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- }
- });
- }
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
index 6cd1d5867e..283b4e7f80 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
@@ -36,6 +36,10 @@ public class DateUtils {
private static final Logger logger = LoggerFactory.getLogger(DateUtils.class);
+ private DateUtils() {
+ throw new UnsupportedOperationException("Construct DateUtils");
+ }
+
/**
* date to local datetime
*
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java
index 591c16db39..b8b6c1d13e 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java
@@ -20,8 +20,6 @@ import org.apache.dolphinscheduler.common.enums.DependResult;
import org.apache.dolphinscheduler.common.enums.DependentRelation;
import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Date;
@@ -29,32 +27,35 @@ import java.util.List;
public class DependentUtils {
- private static final Logger logger = LoggerFactory.getLogger(DependentUtils.class);
+ private DependentUtils() {
+ throw new UnsupportedOperationException("Construct DependentUtils");
+ }
public static DependResult getDependResultForRelation(DependentRelation relation,
- List dependResultList){
+ List dependResultList) {
DependResult dependResult = DependResult.SUCCESS;
- switch (relation){
+ switch (relation) {
case AND:
- if(dependResultList.contains(DependResult.FAILED)){
+ if (dependResultList.contains(DependResult.FAILED)) {
dependResult = DependResult.FAILED;
- } if(dependResultList.contains(DependResult.WAITING)){
+ }
+ if (dependResultList.contains(DependResult.WAITING)) {
dependResult = DependResult.WAITING;
}
break;
case OR:
- if(dependResultList.contains(DependResult.SUCCESS)){
+ if (dependResultList.contains(DependResult.SUCCESS)) {
dependResult = DependResult.SUCCESS;
- }else if(dependResultList.contains(DependResult.WAITING)){
+ } else if (dependResultList.contains(DependResult.WAITING)) {
dependResult = DependResult.WAITING;
- }else{
+ } else {
dependResult = DependResult.FAILED;
}
break;
default:
- break;
+ break;
}
return dependResult;
}
@@ -62,36 +63,37 @@ public class DependentUtils {
/**
* get date interval list by business date and date value.
+ *
* @param businessDate business date
* @param dateValue date value
* @return date interval list by business date and date value.
*/
- public static List getDateIntervalList(Date businessDate, String dateValue){
+ public static List getDateIntervalList(Date businessDate, String dateValue) {
List result = new ArrayList<>();
- switch (dateValue){
+ switch (dateValue) {
case "currentHour":
result = DependentDateUtils.getLastHoursInterval(businessDate, 0);
break;
case "last1Hour":
- result = DependentDateUtils.getLastHoursInterval(businessDate, 1);
+ result = DependentDateUtils.getLastHoursInterval(businessDate, 1);
break;
case "last2Hours":
- result = DependentDateUtils.getLastHoursInterval(businessDate, 2);
+ result = DependentDateUtils.getLastHoursInterval(businessDate, 2);
break;
case "last3Hours":
- result = DependentDateUtils.getLastHoursInterval(businessDate, 3);
+ result = DependentDateUtils.getLastHoursInterval(businessDate, 3);
break;
case "last24Hours":
result = DependentDateUtils.getSpecialLastDayInterval(businessDate);
break;
case "today":
- result = DependentDateUtils.getTodayInterval(businessDate);
+ result = DependentDateUtils.getTodayInterval(businessDate);
break;
case "last1Days":
- result = DependentDateUtils.getLastDayInterval(businessDate, 1);
+ result = DependentDateUtils.getLastDayInterval(businessDate, 1);
break;
case "last2Days":
- result = DependentDateUtils.getLastDayInterval(businessDate, 2);
+ result = DependentDateUtils.getLastDayInterval(businessDate, 2);
break;
case "last3Days":
result = DependentDateUtils.getLastDayInterval(businessDate, 3);
@@ -144,5 +146,4 @@ public class DependentUtils {
return result;
}
-
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java
index c153ec817a..5d9d540d96 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.digest.DigestUtils;
@@ -23,14 +24,16 @@ import org.apache.commons.codec.digest.DigestUtils;
*/
public class EncryptionUtils {
+ private EncryptionUtils() {
+ throw new UnsupportedOperationException("Construct EncryptionUtils");
+ }
/**
- *
* @param rawStr raw string
* @return md5(rawStr)
*/
public static String getMd5(String rawStr) {
- return DigestUtils.md5Hex(null == rawStr ? StringUtils.EMPTY : rawStr);
+ return DigestUtils.md5Hex(null == rawStr ? StringUtils.EMPTY : rawStr);
}
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java
index 924e8ff719..10963b486a 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java
@@ -14,12 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.utils;
-
+package org.apache.dolphinscheduler.common.utils;
public class EnumUtils {
+ private EnumUtils() {
+ throw new UnsupportedOperationException("Construct EnumUtils");
+ }
+
public static > E getEnum(final Class enumClass, final String enumName) {
if (enumName == null) {
return null;
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
index de3d42974a..e3e33566e9 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH;
@@ -21,6 +22,9 @@ import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS
import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;
import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
+import org.apache.commons.io.Charsets;
+import org.apache.commons.io.IOUtils;
+
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
@@ -36,9 +40,6 @@ import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException;
import java.util.Optional;
-import org.apache.commons.io.Charsets;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,12 +47,17 @@ import org.slf4j.LoggerFactory;
* file utils
*/
public class FileUtils {
+
public static final Logger logger = LoggerFactory.getLogger(FileUtils.class);
- public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH,"/tmp/dolphinscheduler");
+ public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH, "/tmp/dolphinscheduler");
public static final ThreadLocal taskLoggerThreadLocal = new ThreadLocal<>();
+ private FileUtils() {
+ throw new UnsupportedOperationException("Construct FileUtils");
+ }
+
/**
* get file suffix
*
@@ -80,7 +86,7 @@ public class FileUtils {
String fileName = String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename);
File file = new File(fileName);
- if (!file.getParentFile().exists()){
+ if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
@@ -97,7 +103,7 @@ public class FileUtils {
public static String getUploadFilename(String tenantCode, String filename) {
String fileName = String.format("%s/%s/resources/%s", DATA_BASEDIR, tenantCode, filename);
File file = new File(fileName);
- if (!file.getParentFile().exists()){
+ if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
@@ -106,6 +112,7 @@ public class FileUtils {
/**
* directory of process execution
+ *
* @param projectId project id
* @param processDefineId process definition id
* @param processInstanceId process instance id
@@ -114,9 +121,9 @@ public class FileUtils {
*/
public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId, int taskInstanceId) {
String fileName = String.format("%s/exec/process/%s/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId),
- Integer.toString(processDefineId), Integer.toString(processInstanceId),Integer.toString(taskInstanceId));
+ Integer.toString(processDefineId), Integer.toString(processInstanceId), Integer.toString(taskInstanceId));
File file = new File(fileName);
- if (!file.getParentFile().exists()){
+ if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
@@ -125,6 +132,7 @@ public class FileUtils {
/**
* directory of process instances
+ *
* @param projectId project id
* @param processDefineId process definition id
* @param processInstanceId process instance id
@@ -150,6 +158,7 @@ public class FileUtils {
/**
* create directory and user
+ *
* @param execLocalPath execute local path
* @param userName user name
* @throws IOException errors
@@ -190,12 +199,11 @@ public class FileUtils {
OSUtils.taskLoggerThreadLocal.remove();
}
-
/**
* write content to file ,if parent path not exists, it will do one's utmost to mkdir
*
- * @param content content
- * @param filePath target file path
+ * @param content content
+ * @param filePath target file path
* @return true if write success
*/
public static boolean writeContent2File(String content, String filePath) {
@@ -231,13 +239,13 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist.
- *
+ *
* NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist.
*
- * @param file the file to write
- * @param data the content to write to the file
- * @param encoding the encoding to use, {@code null} means platform default
+ * @param file the file to write
+ * @param data the content to write to the file
+ * @param encoding the encoding to use, {@code null} means platform default
* @throws IOException in case of an I/O error
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
* @since 2.4
@@ -248,13 +256,13 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist.
- *
+ *
* NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist.
*
- * @param file the file to write
- * @param data the content to write to the file
- * @param encoding the encoding to use, {@code null} means platform default
+ * @param file the file to write
+ * @param data the content to write to the file
+ * @param encoding the encoding to use, {@code null} means platform default
* @throws IOException in case of an I/O error
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
*/
@@ -265,9 +273,9 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist.
*
- * @param file the file to write
- * @param data the content to write to the file
- * @param encoding the encoding to use, {@code null} means platform default
+ * @param file the file to write
+ * @param data the content to write to the file
+ * @param encoding the encoding to use, {@code null} means platform default
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
@@ -287,15 +295,14 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist.
*
- * @param file the file to write
- * @param data the content to write to the file
- * @param encoding the encoding to use, {@code null} means platform default
+ * @param file the file to write
+ * @param data the content to write to the file
+ * @param encoding the encoding to use, {@code null} means platform default
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
- * @throws UnsupportedCharsetException
- * thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not
- * supported by the VM
+ * @throws UnsupportedCharsetException thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not
+ * supported by the VM
* @since 2.1
*/
public static void writeStringToFile(File file, String data, String encoding, boolean append) throws IOException {
@@ -305,8 +312,8 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM.
*
- * @param file the file to write
- * @param data the content to write to the file
+ * @param file the file to write
+ * @param data the content to write to the file
* @throws IOException in case of an I/O error
*/
public static void writeStringToFile(File file, String data) throws IOException {
@@ -316,8 +323,8 @@ public class FileUtils {
/**
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM.
*
- * @param file the file to write
- * @param data the content to write to the file
+ * @param file the file to write
+ * @param data the content to write to the file
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
@@ -340,7 +347,7 @@ public class FileUtils {
* An exception is thrown if the file exists but cannot be written to.
* An exception is thrown if the parent directory cannot be created.
*
- * @param file the file to open for output, must not be {@code null}
+ * @param file the file to open for output, must not be {@code null}
* @return a new {@link FileOutputStream} for the specified file
* @throws IOException if the file object is a directory
* @throws IOException if the file cannot be written to
@@ -364,7 +371,7 @@ public class FileUtils {
* An exception is thrown if the file exists but cannot be written to.
* An exception is thrown if the parent directory cannot be created.
*
- * @param file the file to open for output, must not be {@code null}
+ * @param file the file to open for output, must not be {@code null}
* @param append if {@code true}, then bytes will be added to the
* end of the file rather than overwriting
* @return a new {@link FileOutputStream} for the specified file
@@ -384,15 +391,15 @@ public class FileUtils {
} else {
File parent = file.getParentFile();
if (parent != null && !parent.mkdirs() && !parent.isDirectory()) {
- throw new IOException("Directory '" + parent + "' could not be created");
+ throw new IOException("Directory '" + parent + "' could not be created");
}
}
return new FileOutputStream(file, append);
}
-
/**
* deletes a directory recursively
+ *
* @param dir directory
* @throws IOException in case deletion is unsuccessful
*/
@@ -420,17 +427,18 @@ public class FileUtils {
/**
* Gets all the parent subdirectories of the parentDir directory
+ *
* @param parentDir parent dir
* @return all dirs
*/
- public static File[] getAllDir(String parentDir){
- if(parentDir == null || "".equals(parentDir)) {
+ public static File[] getAllDir(String parentDir) {
+ if (parentDir == null || "".equals(parentDir)) {
throw new RuntimeException("parentDir can not be empty");
}
File file = new File(parentDir);
- if(!file.exists() || !file.isDirectory()) {
- throw new RuntimeException("parentDir not exist, or is not a directory:"+parentDir);
+ if (!file.exists() || !file.isDirectory()) {
+ throw new RuntimeException("parentDir not exist, or is not a directory:" + parentDir);
}
return file.listFiles(File::isDirectory);
@@ -438,6 +446,7 @@ public class FileUtils {
/**
* Get Content
+ *
* @param inputStream input stream
* @return string of input stream
*/
@@ -447,15 +456,14 @@ public class FileUtils {
ByteArrayOutputStream output = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int length;
- while ((length= inputStream.read(buffer)) != -1) {
- output.write(buffer,0,length);
+ while ((length = inputStream.read(buffer)) != -1) {
+ output.write(buffer, 0, length);
}
return output.toString();
} catch (Exception e) {
- logger.error(e.getMessage(),e);
+ logger.error(e.getMessage(), e);
throw new RuntimeException(e);
}
}
-
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
index 7d1e0a523a..36b437f312 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
@@ -14,9 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
+
import org.apache.http.HttpEntity;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
@@ -30,148 +32,148 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
import java.util.Arrays;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* http utils
*/
public class HttpUtils {
+ public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
- public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
-
- private HttpUtils() {
-
- }
-
- public static CloseableHttpClient getInstance(){
- return HttpClientInstance.httpClient;
- }
-
- private static class HttpClientInstance{
- private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
- }
-
-
- private static PoolingHttpClientConnectionManager cm;
-
- private static SSLContext ctx = null;
-
- private static SSLConnectionSocketFactory socketFactory;
-
- private static RequestConfig requestConfig;
-
- private static Registry socketFactoryRegistry;
-
- private static X509TrustManager xtm = new X509TrustManager() {
- @Override
- public void checkClientTrusted(X509Certificate[] chain, String authType) {
- }
-
- @Override
- public void checkServerTrusted(X509Certificate[] chain, String authType) {
- }
+ private HttpUtils() {
+ throw new UnsupportedOperationException("Construct HttpUtils");
+ }
- @Override
- public X509Certificate[] getAcceptedIssuers() {
- return null;
- }
- };
+ public static CloseableHttpClient getInstance() {
+ return HttpClientInstance.httpClient;
+ }
- static {
- try {
- ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
- ctx.init(null, new TrustManager[] { xtm }, null);
- } catch (NoSuchAlgorithmException e) {
- logger.error("SSLContext init with NoSuchAlgorithmException", e);
- } catch (KeyManagementException e) {
- logger.error("SSLContext init with KeyManagementException", e);
- }
- socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
- /** set timeout、request time、socket timeout */
- requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
- .setExpectContinueEnabled(Boolean.TRUE)
- .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
- .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
- .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
- .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
- .build();
- socketFactoryRegistry = RegistryBuilder.create()
- .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
- cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
- cm.setDefaultMaxPerRoute(60);
- cm.setMaxTotal(100);
+ private static class HttpClientInstance {
+ private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
+ }
- }
+ private static PoolingHttpClientConnectionManager cm;
+
+ private static SSLContext ctx = null;
+
+ private static SSLConnectionSocketFactory socketFactory;
+
+ private static RequestConfig requestConfig;
+
+ private static Registry socketFactoryRegistry;
+
+ private static X509TrustManager xtm = new X509TrustManager() {
+ @Override
+ public void checkClientTrusted(X509Certificate[] chain, String authType) {
+ }
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] chain, String authType) {
+ }
+
+ @Override
+ public X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ };
+
+ static {
+ try {
+ ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
+ ctx.init(null, new TrustManager[]{xtm}, null);
+ } catch (NoSuchAlgorithmException e) {
+ logger.error("SSLContext init with NoSuchAlgorithmException", e);
+ } catch (KeyManagementException e) {
+ logger.error("SSLContext init with KeyManagementException", e);
+ }
+ socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
+ /** set timeout、request time、socket timeout */
+ requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
+ .setExpectContinueEnabled(Boolean.TRUE)
+ .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
+ .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
+ .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
+ .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
+ .build();
+ socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
+ cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ cm.setDefaultMaxPerRoute(60);
+ cm.setMaxTotal(100);
+ }
- /**
- * get http request content
- * @param url url
- * @return http get request response content
- */
- public static String get(String url){
- CloseableHttpClient httpclient = HttpUtils.getInstance();
+ /**
+ * get http request content
+ *
+ * @param url url
+ * @return http get request response content
+ */
+ public static String get(String url) {
+ CloseableHttpClient httpclient = HttpUtils.getInstance();
- HttpGet httpget = new HttpGet(url);
- return getResponseContentString(httpget,httpclient);
+ HttpGet httpget = new HttpGet(url);
+ return getResponseContentString(httpget, httpclient);
}
/**
* get http response content
*
- * @param httpget httpget
+ * @param httpget httpget
* @param httpClient httpClient
* @return http get request response content
*/
public static String getResponseContentString(HttpGet httpget, CloseableHttpClient httpClient) {
- String responseContent = null;
- CloseableHttpResponse response = null;
- try {
- response = httpClient.execute(httpget);
- // check response status is 200
- if (response.getStatusLine().getStatusCode() == 200) {
- HttpEntity entity = response.getEntity();
- if (entity != null) {
- responseContent = EntityUtils.toString(entity, Constants.UTF_8);
- } else {
- logger.warn("http entity is null");
- }
- } else {
- logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
- }
- } catch (IOException ioe) {
- logger.error(ioe.getMessage(), ioe);
- } finally {
- try {
- if (response != null) {
- EntityUtils.consume(response.getEntity());
- response.close();
- }
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
- }
- if (!httpget.isAborted()) {
- httpget.releaseConnection();
- httpget.abort();
- }
-
- }
- return responseContent;
+ String responseContent = null;
+ CloseableHttpResponse response = null;
+ try {
+ response = httpClient.execute(httpget);
+ // check response status is 200
+ if (response.getStatusLine().getStatusCode() == 200) {
+ HttpEntity entity = response.getEntity();
+ if (entity != null) {
+ responseContent = EntityUtils.toString(entity, Constants.UTF_8);
+ } else {
+ logger.warn("http entity is null");
+ }
+ } else {
+ logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
+ }
+ } catch (IOException ioe) {
+ logger.error(ioe.getMessage(), ioe);
+ } finally {
+ try {
+ if (response != null) {
+ EntityUtils.consume(response.getEntity());
+ response.close();
+ }
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
+ if (!httpget.isAborted()) {
+ httpget.releaseConnection();
+ httpget.abort();
+ }
+
+ }
+ return responseContent;
}
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java
index ce551d8405..96366d539f 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java
@@ -1,4 +1,3 @@
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -18,14 +17,17 @@
package org.apache.dolphinscheduler.common.utils;
-
import java.io.Closeable;
import java.io.IOException;
public class IOUtils {
- public static void closeQuietly(Closeable closeable){
- if(closeable != null){
+ private IOUtils() {
+ throw new UnsupportedOperationException("Construct IOUtils");
+ }
+
+ public static void closeQuietly(Closeable closeable) {
+ if (closeable != null) {
try {
closeable.close();
} catch (IOException ignore) {
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java
index 858e5b44b5..63d43e7b69 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java
@@ -14,46 +14,50 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.utils;
+package org.apache.dolphinscheduler.common.utils;
/**
* http utils
*/
public class IpUtils {
- public static final String DOT = ".";
-
- /**
- * ip str to long
- *
- * @param ipStr ip string
- * @return ip to long
- */
- public static Long ipToLong(String ipStr) {
- String[] ipSet = ipStr.split("\\" + DOT);
-
- return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]);
- }
-
- /**
- * long to ip
- * @param ipLong the long number converted from IP
- * @return String
- */
- public static String longToIp(long ipLong) {
- long[] ipNumbers = new long[4];
- long tmp = 0xFF;
- ipNumbers[0] = ipLong >> 24 & tmp;
- ipNumbers[1] = ipLong >> 16 & tmp;
- ipNumbers[2] = ipLong >> 8 & tmp;
- ipNumbers[3] = ipLong & tmp;
-
- String sb = ipNumbers[0] + DOT +
- ipNumbers[1] + DOT +
- ipNumbers[2] + DOT +
- ipNumbers[3];
- return sb;
- }
+ private IpUtils() {
+ throw new UnsupportedOperationException("Construct IpUtils");
+ }
+
+ public static final String DOT = ".";
+
+ /**
+ * ip str to long
+ *
+ * @param ipStr ip string
+ * @return ip to long
+ */
+ public static Long ipToLong(String ipStr) {
+ String[] ipSet = ipStr.split("\\" + DOT);
+
+ return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]);
+ }
+
+ /**
+ * long to ip
+ *
+ * @param ipLong the long number converted from IP
+ * @return String
+ */
+ public static String longToIp(long ipLong) {
+ long[] ipNumbers = new long[4];
+ long tmp = 0xFF;
+ ipNumbers[0] = ipLong >> 24 & tmp;
+ ipNumbers[1] = ipLong >> 16 & tmp;
+ ipNumbers[2] = ipLong >> 8 & tmp;
+ ipNumbers[3] = ipLong & tmp;
+
+ return ipNumbers[0] + DOT
+ + ipNumbers[1] + DOT
+ + ipNumbers[2] + DOT
+ + ipNumbers[3];
+ }
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
index 3e88a84ec2..56ef74d6ee 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
@@ -14,25 +14,39 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils;
+import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT;
+import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
+import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
+import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.*;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import com.fasterxml.jackson.databind.type.CollectionType;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.*;
-
-import static com.fasterxml.jackson.databind.DeserializationFeature.*;
-import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
/**
* json utils
@@ -49,13 +63,12 @@ public class JSONUtils {
.configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true)
.configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
.configure(REQUIRE_SETTERS_FOR_GETTERS, true)
- .setTimeZone(TimeZone.getDefault())
- ;
+ .setTimeZone(TimeZone.getDefault());
private JSONUtils() {
+ throw new UnsupportedOperationException("Construct JSONUtils");
}
-
public static ArrayNode createArrayNode() {
return objectMapper.createArrayNode();
}
@@ -94,9 +107,9 @@ public class JSONUtils {
* the fields of the specified object are generics, just the object itself should not be a
* generic type.
*
- * @param json the string from which the object is to be deserialized
+ * @param json the string from which the object is to be deserialized
* @param clazz the class of T
- * @param T
+ * @param T
* @return an object of type T from the string
* classOfT
*/
@@ -116,9 +129,9 @@ public class JSONUtils {
/**
* json to list
*
- * @param json json string
+ * @param json json string
* @param clazz class
- * @param T
+ * @param T
* @return list
*/
public static List toList(String json, Class clazz) {
@@ -137,7 +150,6 @@ public class JSONUtils {
return Collections.emptyList();
}
-
/**
* check json object valid
*
@@ -160,13 +172,12 @@ public class JSONUtils {
return false;
}
-
/**
* Method for finding a JSON Object field with specified name in this
* node or its child nodes, and returning value it has.
* If no matching field is found in this node or its descendants, returns null.
*
- * @param jsonNode json node
+ * @param jsonNode json node
* @param fieldName Name of field to look for
* @return Value of first matching node found, if any; null if none
*/
@@ -180,7 +191,6 @@ public class JSONUtils {
return node.toString();
}
-
/**
* json to map
*
@@ -195,7 +205,8 @@ public class JSONUtils {
}
try {
- return objectMapper.readValue(json, new TypeReference