Browse Source

[Improvement][common] Add UnsupportedOperationException for utils construct (#3381)

* [Improvement][common] Add UnsupportedOperationException for utils construct

* Fix checkstyle
pull/3/MERGE
Yichao Yang 4 years ago committed by GitHub
parent
commit
ac4ed94061
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  2. 26
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
  3. 65
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
  4. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java
  5. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
  6. 27
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java
  7. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java
  8. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java
  9. 52
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
  10. 32
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
  11. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java
  12. 16
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java
  13. 46
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
  14. 16
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
  15. 25
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java
  16. 62
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
  17. 100
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
  18. 11
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java
  19. 36
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
  20. 14
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java
  21. 50
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java
  22. 8
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java
  23. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java
  24. 12
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  25. 12
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common; package org.apache.dolphinscheduler.common;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
@ -27,7 +28,7 @@ import java.util.regex.Pattern;
public final class Constants { public final class Constants {
private Constants() { private Constants() {
throw new IllegalStateException("Constants class"); throw new UnsupportedOperationException("Construct Constants");
} }
/** /**

26
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java

@ -14,13 +14,19 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.beanutils.BeanMap; import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.lang.StringUtils;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** /**
* Provides utility methods and decorators for {@link Collection} instances. * Provides utility methods and decorators for {@link Collection} instances.
@ -37,8 +43,9 @@ import java.util.*;
public class CollectionUtils { public class CollectionUtils {
private CollectionUtils() { private CollectionUtils() {
throw new IllegalStateException("CollectionUtils class"); throw new UnsupportedOperationException("Construct CollectionUtils");
} }
/** /**
* Returns a new {@link Collection} containing <i>a</i> minus a subset of * Returns a new {@link Collection} containing <i>a</i> minus a subset of
* <i>b</i>. Only the elements of <i>b</i> that satisfy the predicate * <i>b</i>. Only the elements of <i>b</i> that satisfy the predicate
@ -112,7 +119,6 @@ public class CollectionUtils {
return map; return map;
} }
/** /**
* Helper class to easily access cardinality properties of two collections. * Helper class to easily access cardinality properties of two collections.
* *
@ -137,8 +143,8 @@ public class CollectionUtils {
* @param b the second collection * @param b the second collection
*/ */
public CardinalityHelper(final Iterable<? extends O> a, final Iterable<? extends O> b) { public CardinalityHelper(final Iterable<? extends O> a, final Iterable<? extends O> b) {
cardinalityA = CollectionUtils.<O>getCardinalityMap(a); cardinalityA = CollectionUtils.getCardinalityMap(a);
cardinalityB = CollectionUtils.<O>getCardinalityMap(b); cardinalityB = CollectionUtils.getCardinalityMap(b);
} }
/** /**
@ -239,9 +245,9 @@ public class CollectionUtils {
return count; return count;
} }
/** /**
* Removes certain attributes of each object in the list * Removes certain attributes of each object in the list
*
* @param originList origin list * @param originList origin list
* @param exclusionSet exclusion set * @param exclusionSet exclusion set
* @param <T> T * @param <T> T
@ -258,8 +264,8 @@ public class CollectionUtils {
Map<String, Object> instanceMap; Map<String, Object> instanceMap;
for (T instance : originList) { for (T instance : originList) {
Map<String, Object> dataMap = new BeanMap(instance); Map<String, Object> dataMap = new BeanMap(instance);
instanceMap = new LinkedHashMap<>(16,0.75f,true); instanceMap = new LinkedHashMap<>(16, 0.75f, true);
for (Map.Entry<String, Object> entry: dataMap.entrySet()) { for (Map.Entry<String, Object> entry : dataMap.entrySet()) {
if (exclusionSet.contains(entry.getKey())) { if (exclusionSet.contains(entry.getKey())) {
continue; continue;
} }

65
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java

@ -14,19 +14,22 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.binary.Base64;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* common utils * common utils
*/ */
@ -36,7 +39,7 @@ public class CommonUtils {
private static final Base64 BASE64 = new Base64(); private static final Base64 BASE64 = new Base64();
private CommonUtils() { private CommonUtils() {
throw new IllegalStateException("CommonUtils class"); throw new UnsupportedOperationException("Construct CommonUtils");
} }
/** /**
@ -47,10 +50,10 @@ public class CommonUtils {
if (StringUtils.isEmpty(envPath)) { if (StringUtils.isEmpty(envPath)) {
URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH); URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH);
if (envDefaultPath != null){ if (envDefaultPath != null) {
envPath = envDefaultPath.getPath(); envPath = envDefaultPath.getPath();
logger.debug("env path :{}", envPath); logger.debug("env path :{}", envPath);
}else{ } else {
envPath = "/etc/profile"; envPath = "/etc/profile";
} }
} }
@ -59,31 +62,30 @@ public class CommonUtils {
} }
/** /**
*
* @return is develop mode * @return is develop mode
*/ */
public static boolean isDevelopMode() { public static boolean isDevelopMode() {
return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true); return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true);
} }
/** /**
* if upload resource is HDFS and kerberos startup is true , else false * if upload resource is HDFS and kerberos startup is true , else false
*
* @return true if upload resource is HDFS and kerberos startup * @return true if upload resource is HDFS and kerberos startup
*/ */
public static boolean getKerberosStartupState(){ public static boolean getKerberosStartupState() {
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false); Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
return resUploadType == ResUploadType.HDFS && kerberosStartupState; return resUploadType == ResUploadType.HDFS && kerberosStartupState;
} }
/** /**
* load kerberos configuration * load kerberos configuration
*
* @throws Exception errors * @throws Exception errors
*/ */
public static void loadKerberosConf()throws Exception{ public static void loadKerberosConf() throws Exception {
if (CommonUtils.getKerberosStartupState()) { if (CommonUtils.getKerberosStartupState()) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
Configuration configuration = new Configuration(); Configuration configuration = new Configuration();
@ -96,42 +98,45 @@ public class CommonUtils {
/** /**
* encode password * encode password
* @param password
* @return
*/ */
public static String encodePassword(String password) { public static String encodePassword(String password) {
if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; } if (StringUtils.isEmpty(password)) {
return StringUtils.EMPTY;
}
//if encryption is not turned on, return directly //if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false); boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false);
if ( !encryptionEnable){ return password; } if (!encryptionEnable) {
return password;
}
// Using Base64 + salt to process password // Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))) ; String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8)));
return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8))); return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
} }
/** /**
* decode password * decode password
* @param password
* @return
*/ */
public static String decodePassword(String password) { public static String decodePassword(String password) {
if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; } if (StringUtils.isEmpty(password)) {
return StringUtils.EMPTY;
}
//if encryption is not turned on, return directly //if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false); boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false);
if ( !encryptionEnable){ return password; } if (!encryptionEnable) {
return password;
}
// Using Base64 + salt to process password // Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ; String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8);
if(!passwordWithSalt.startsWith(salt)){ if (!passwordWithSalt.startsWith(salt)) {
logger.warn("There is a password and salt mismatch: {} ",password); logger.warn("There is a password and salt mismatch: {} ", password);
return password; return password;
} }
return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ; return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8);
} }
} }

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java

@ -14,10 +14,12 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import java.util.Arrays; import java.util.Arrays;
import java.util.Objects; import java.util.Objects;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -26,11 +28,12 @@ public class ConnectionUtils {
public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class); public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class);
private ConnectionUtils() { private ConnectionUtils() {
throw new IllegalStateException("ConnectionUtils class"); throw new UnsupportedOperationException("Construct ConnectionUtils");
} }
/** /**
* release resource * release resource
*
* @param resources resources * @param resources resources
*/ */
public static void releaseResource(AutoCloseable... resources) { public static void releaseResource(AutoCloseable... resources) {

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java

@ -36,6 +36,10 @@ public class DateUtils {
private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); private static final Logger logger = LoggerFactory.getLogger(DateUtils.class);
private DateUtils() {
throw new UnsupportedOperationException("Construct DateUtils");
}
/** /**
* date to local datetime * date to local datetime
* *

27
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java

@ -20,8 +20,6 @@ import org.apache.dolphinscheduler.common.enums.DependResult;
import org.apache.dolphinscheduler.common.enums.DependentRelation; import org.apache.dolphinscheduler.common.enums.DependentRelation;
import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils; import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
@ -29,27 +27,30 @@ import java.util.List;
public class DependentUtils { public class DependentUtils {
private static final Logger logger = LoggerFactory.getLogger(DependentUtils.class); private DependentUtils() {
throw new UnsupportedOperationException("Construct DependentUtils");
}
public static DependResult getDependResultForRelation(DependentRelation relation, public static DependResult getDependResultForRelation(DependentRelation relation,
List<DependResult> dependResultList){ List<DependResult> dependResultList) {
DependResult dependResult = DependResult.SUCCESS; DependResult dependResult = DependResult.SUCCESS;
switch (relation){ switch (relation) {
case AND: case AND:
if(dependResultList.contains(DependResult.FAILED)){ if (dependResultList.contains(DependResult.FAILED)) {
dependResult = DependResult.FAILED; dependResult = DependResult.FAILED;
} if(dependResultList.contains(DependResult.WAITING)){ }
if (dependResultList.contains(DependResult.WAITING)) {
dependResult = DependResult.WAITING; dependResult = DependResult.WAITING;
} }
break; break;
case OR: case OR:
if(dependResultList.contains(DependResult.SUCCESS)){ if (dependResultList.contains(DependResult.SUCCESS)) {
dependResult = DependResult.SUCCESS; dependResult = DependResult.SUCCESS;
}else if(dependResultList.contains(DependResult.WAITING)){ } else if (dependResultList.contains(DependResult.WAITING)) {
dependResult = DependResult.WAITING; dependResult = DependResult.WAITING;
}else{ } else {
dependResult = DependResult.FAILED; dependResult = DependResult.FAILED;
} }
break; break;
@ -62,13 +63,14 @@ public class DependentUtils {
/** /**
* get date interval list by business date and date value. * get date interval list by business date and date value.
*
* @param businessDate business date * @param businessDate business date
* @param dateValue date value * @param dateValue date value
* @return date interval list by business date and date value. * @return date interval list by business date and date value.
*/ */
public static List<DateInterval> getDateIntervalList(Date businessDate, String dateValue){ public static List<DateInterval> getDateIntervalList(Date businessDate, String dateValue) {
List<DateInterval> result = new ArrayList<>(); List<DateInterval> result = new ArrayList<>();
switch (dateValue){ switch (dateValue) {
case "currentHour": case "currentHour":
result = DependentDateUtils.getLastHoursInterval(businessDate, 0); result = DependentDateUtils.getLastHoursInterval(businessDate, 0);
break; break;
@ -144,5 +146,4 @@ public class DependentUtils {
return result; return result;
} }
} }

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
@ -23,9 +24,11 @@ import org.apache.commons.codec.digest.DigestUtils;
*/ */
public class EncryptionUtils { public class EncryptionUtils {
private EncryptionUtils() {
throw new UnsupportedOperationException("Construct EncryptionUtils");
}
/** /**
*
* @param rawStr raw string * @param rawStr raw string
* @return md5(rawStr) * @return md5(rawStr)
*/ */

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java

@ -14,12 +14,15 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils;
package org.apache.dolphinscheduler.common.utils;
public class EnumUtils { public class EnumUtils {
private EnumUtils() {
throw new UnsupportedOperationException("Construct EnumUtils");
}
public static <E extends Enum<E>> E getEnum(final Class<E> enumClass, final String enumName) { public static <E extends Enum<E>> E getEnum(final Class<E> enumClass, final String enumName) {
if (enumName == null) { if (enumName == null) {
return null; return null;

52
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH; import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH;
@ -21,6 +22,9 @@ import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS
import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE; import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;
import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
@ -36,9 +40,6 @@ import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException; import java.nio.charset.UnsupportedCharsetException;
import java.util.Optional; import java.util.Optional;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -46,12 +47,17 @@ import org.slf4j.LoggerFactory;
* file utils * file utils
*/ */
public class FileUtils { public class FileUtils {
public static final Logger logger = LoggerFactory.getLogger(FileUtils.class); public static final Logger logger = LoggerFactory.getLogger(FileUtils.class);
public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH,"/tmp/dolphinscheduler"); public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH, "/tmp/dolphinscheduler");
public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>(); public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();
private FileUtils() {
throw new UnsupportedOperationException("Construct FileUtils");
}
/** /**
* get file suffix * get file suffix
* *
@ -80,7 +86,7 @@ public class FileUtils {
String fileName = String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename); String fileName = String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename);
File file = new File(fileName); File file = new File(fileName);
if (!file.getParentFile().exists()){ if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs(); file.getParentFile().mkdirs();
} }
@ -97,7 +103,7 @@ public class FileUtils {
public static String getUploadFilename(String tenantCode, String filename) { public static String getUploadFilename(String tenantCode, String filename) {
String fileName = String.format("%s/%s/resources/%s", DATA_BASEDIR, tenantCode, filename); String fileName = String.format("%s/%s/resources/%s", DATA_BASEDIR, tenantCode, filename);
File file = new File(fileName); File file = new File(fileName);
if (!file.getParentFile().exists()){ if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs(); file.getParentFile().mkdirs();
} }
@ -106,6 +112,7 @@ public class FileUtils {
/** /**
* directory of process execution * directory of process execution
*
* @param projectId project id * @param projectId project id
* @param processDefineId process definition id * @param processDefineId process definition id
* @param processInstanceId process instance id * @param processInstanceId process instance id
@ -114,9 +121,9 @@ public class FileUtils {
*/ */
public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId, int taskInstanceId) { public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId, int taskInstanceId) {
String fileName = String.format("%s/exec/process/%s/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId), String fileName = String.format("%s/exec/process/%s/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId),
Integer.toString(processDefineId), Integer.toString(processInstanceId),Integer.toString(taskInstanceId)); Integer.toString(processDefineId), Integer.toString(processInstanceId), Integer.toString(taskInstanceId));
File file = new File(fileName); File file = new File(fileName);
if (!file.getParentFile().exists()){ if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs(); file.getParentFile().mkdirs();
} }
@ -125,6 +132,7 @@ public class FileUtils {
/** /**
* directory of process instances * directory of process instances
*
* @param projectId project id * @param projectId project id
* @param processDefineId process definition id * @param processDefineId process definition id
* @param processInstanceId process instance id * @param processInstanceId process instance id
@ -150,6 +158,7 @@ public class FileUtils {
/** /**
* create directory and user * create directory and user
*
* @param execLocalPath execute local path * @param execLocalPath execute local path
* @param userName user name * @param userName user name
* @throws IOException errors * @throws IOException errors
@ -190,7 +199,6 @@ public class FileUtils {
OSUtils.taskLoggerThreadLocal.remove(); OSUtils.taskLoggerThreadLocal.remove();
} }
/** /**
* write content to file ,if parent path not exists, it will do one's utmost to mkdir * write content to file ,if parent path not exists, it will do one's utmost to mkdir
* *
@ -231,7 +239,7 @@ public class FileUtils {
/** /**
* Writes a String to a file creating the file if it does not exist. * Writes a String to a file creating the file if it does not exist.
* * <p>
* NOTE: As from v1.3, the parent directories of the file will be created * NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist. * if they do not exist.
* *
@ -248,7 +256,7 @@ public class FileUtils {
/** /**
* Writes a String to a file creating the file if it does not exist. * Writes a String to a file creating the file if it does not exist.
* * <p>
* NOTE: As from v1.3, the parent directories of the file will be created * NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist. * if they do not exist.
* *
@ -293,8 +301,7 @@ public class FileUtils {
* @param append if {@code true}, then the String will be added to the * @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting * end of the file rather than overwriting
* @throws IOException in case of an I/O error * @throws IOException in case of an I/O error
* @throws UnsupportedCharsetException * @throws UnsupportedCharsetException thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not
* thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not
* supported by the VM * supported by the VM
* @since 2.1 * @since 2.1
*/ */
@ -390,9 +397,9 @@ public class FileUtils {
return new FileOutputStream(file, append); return new FileOutputStream(file, append);
} }
/** /**
* deletes a directory recursively * deletes a directory recursively
*
* @param dir directory * @param dir directory
* @throws IOException in case deletion is unsuccessful * @throws IOException in case deletion is unsuccessful
*/ */
@ -420,17 +427,18 @@ public class FileUtils {
/** /**
* Gets all the parent subdirectories of the parentDir directory * Gets all the parent subdirectories of the parentDir directory
*
* @param parentDir parent dir * @param parentDir parent dir
* @return all dirs * @return all dirs
*/ */
public static File[] getAllDir(String parentDir){ public static File[] getAllDir(String parentDir) {
if(parentDir == null || "".equals(parentDir)) { if (parentDir == null || "".equals(parentDir)) {
throw new RuntimeException("parentDir can not be empty"); throw new RuntimeException("parentDir can not be empty");
} }
File file = new File(parentDir); File file = new File(parentDir);
if(!file.exists() || !file.isDirectory()) { if (!file.exists() || !file.isDirectory()) {
throw new RuntimeException("parentDir not exist, or is not a directory:"+parentDir); throw new RuntimeException("parentDir not exist, or is not a directory:" + parentDir);
} }
return file.listFiles(File::isDirectory); return file.listFiles(File::isDirectory);
@ -438,6 +446,7 @@ public class FileUtils {
/** /**
* Get Content * Get Content
*
* @param inputStream input stream * @param inputStream input stream
* @return string of input stream * @return string of input stream
*/ */
@ -447,15 +456,14 @@ public class FileUtils {
ByteArrayOutputStream output = new ByteArrayOutputStream(); ByteArrayOutputStream output = new ByteArrayOutputStream();
byte[] buffer = new byte[1024]; byte[] buffer = new byte[1024];
int length; int length;
while ((length= inputStream.read(buffer)) != -1) { while ((length = inputStream.read(buffer)) != -1) {
output.write(buffer,0,length); output.write(buffer, 0, length);
} }
return output.toString(); return output.toString();
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(),e); logger.error(e.getMessage(), e);
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
} }

32
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

@ -14,9 +14,11 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.client.config.AuthSchemes; import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.CookieSpecs;
@ -30,38 +32,39 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException; import java.io.IOException;
import java.security.KeyManagementException; import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate; import java.security.cert.X509Certificate;
import java.util.Arrays; import java.util.Arrays;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* http utils * http utils
*/ */
public class HttpUtils { public class HttpUtils {
public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
private HttpUtils() { private HttpUtils() {
throw new UnsupportedOperationException("Construct HttpUtils");
} }
public static CloseableHttpClient getInstance(){ public static CloseableHttpClient getInstance() {
return HttpClientInstance.httpClient; return HttpClientInstance.httpClient;
} }
private static class HttpClientInstance{ private static class HttpClientInstance {
private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build(); private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
} }
@ -94,7 +97,7 @@ public class HttpUtils {
static { static {
try { try {
ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS); ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
ctx.init(null, new TrustManager[] { xtm }, null); ctx.init(null, new TrustManager[]{xtm}, null);
} catch (NoSuchAlgorithmException e) { } catch (NoSuchAlgorithmException e) {
logger.error("SSLContext init with NoSuchAlgorithmException", e); logger.error("SSLContext init with NoSuchAlgorithmException", e);
} catch (KeyManagementException e) { } catch (KeyManagementException e) {
@ -117,18 +120,17 @@ public class HttpUtils {
} }
/** /**
* get http request content * get http request content
*
* @param url url * @param url url
* @return http get request response content * @return http get request response content
*/ */
public static String get(String url){ public static String get(String url) {
CloseableHttpClient httpclient = HttpUtils.getInstance(); CloseableHttpClient httpclient = HttpUtils.getInstance();
HttpGet httpget = new HttpGet(url); HttpGet httpget = new HttpGet(url);
return getResponseContentString(httpget,httpclient); return getResponseContentString(httpget, httpclient);
} }
/** /**

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java

@ -1,4 +1,3 @@
/* /*
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
@ -18,14 +17,17 @@
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
public class IOUtils { public class IOUtils {
public static void closeQuietly(Closeable closeable){ private IOUtils() {
if(closeable != null){ throw new UnsupportedOperationException("Construct IOUtils");
}
public static void closeQuietly(Closeable closeable) {
if (closeable != null) {
try { try {
closeable.close(); closeable.close();
} catch (IOException ignore) { } catch (IOException ignore) {

16
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java

@ -14,14 +14,18 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils;
package org.apache.dolphinscheduler.common.utils;
/** /**
* http utils * http utils
*/ */
public class IpUtils { public class IpUtils {
private IpUtils() {
throw new UnsupportedOperationException("Construct IpUtils");
}
public static final String DOT = "."; public static final String DOT = ".";
/** /**
@ -38,6 +42,7 @@ public class IpUtils {
/** /**
* long to ip * long to ip
*
* @param ipLong the long number converted from IP * @param ipLong the long number converted from IP
* @return String * @return String
*/ */
@ -49,11 +54,10 @@ public class IpUtils {
ipNumbers[2] = ipLong >> 8 & tmp; ipNumbers[2] = ipLong >> 8 & tmp;
ipNumbers[3] = ipLong & tmp; ipNumbers[3] = ipLong & tmp;
String sb = ipNumbers[0] + DOT + return ipNumbers[0] + DOT
ipNumbers[1] + DOT + + ipNumbers[1] + DOT
ipNumbers[2] + DOT + + ipNumbers[2] + DOT
ipNumbers[3]; + ipNumbers[3];
return sb;
} }
} }

46
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java

@ -14,25 +14,39 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT;
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode; import com.fasterxml.jackson.databind.node.TextNode;
import com.fasterxml.jackson.databind.type.CollectionType; import com.fasterxml.jackson.databind.type.CollectionType;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
import static com.fasterxml.jackson.databind.DeserializationFeature.*;
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
/** /**
* json utils * json utils
@ -49,13 +63,12 @@ public class JSONUtils {
.configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true)
.configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
.configure(REQUIRE_SETTERS_FOR_GETTERS, true) .configure(REQUIRE_SETTERS_FOR_GETTERS, true)
.setTimeZone(TimeZone.getDefault()) .setTimeZone(TimeZone.getDefault());
;
private JSONUtils() { private JSONUtils() {
throw new UnsupportedOperationException("Construct JSONUtils");
} }
public static ArrayNode createArrayNode() { public static ArrayNode createArrayNode() {
return objectMapper.createArrayNode(); return objectMapper.createArrayNode();
} }
@ -137,7 +150,6 @@ public class JSONUtils {
return Collections.emptyList(); return Collections.emptyList();
} }
/** /**
* check json object valid * check json object valid
* *
@ -160,7 +172,6 @@ public class JSONUtils {
return false; return false;
} }
/** /**
* Method for finding a JSON Object field with specified name in this * Method for finding a JSON Object field with specified name in this
* node or its child nodes, and returning value it has. * node or its child nodes, and returning value it has.
@ -180,7 +191,6 @@ public class JSONUtils {
return node.toString(); return node.toString();
} }
/** /**
* json to map * json to map
* <p> * <p>
@ -195,7 +205,8 @@ public class JSONUtils {
} }
try { try {
return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {}); return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {
});
} catch (Exception e) { } catch (Exception e) {
logger.error("json to map exception!", e); logger.error("json to map exception!", e);
} }
@ -258,7 +269,6 @@ public class JSONUtils {
} }
} }
/** /**
* json serializer * json serializer
*/ */

16
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java

@ -14,15 +14,17 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.dolphinscheduler.common.Constants;
import org.slf4j.Logger; import org.slf4j.Logger;
/** /**
@ -30,6 +32,10 @@ import org.slf4j.Logger;
*/ */
public class LoggerUtils { public class LoggerUtils {
private LoggerUtils() {
throw new UnsupportedOperationException("Construct LoggerUtils");
}
/** /**
* rules for extracting application ID * rules for extracting application ID
*/ */
@ -62,18 +68,18 @@ public class LoggerUtils {
public static String buildTaskId(String affix, public static String buildTaskId(String affix,
int processDefId, int processDefId,
int processInstId, int processInstId,
int taskId){ int taskId) {
// - [taskAppId=TASK_79_4084_15210] // - [taskAppId=TASK_79_4084_15210]
return String.format(" - %s%s-%s-%s-%s]",TASK_APPID_LOG_FORMAT,affix, return String.format(" - %s%s-%s-%s-%s]", TASK_APPID_LOG_FORMAT, affix,
processDefId, processDefId,
processInstId, processInstId,
taskId); taskId);
} }
/** /**
* processing log * processing log
* get yarn application id list * get yarn application id list
*
* @param log log content * @param log log content
* @param logger logger * @param logger logger
* @return app id list * @return app id list
@ -87,7 +93,7 @@ public class LoggerUtils {
// analyse logs to get all submit yarn application id // analyse logs to get all submit yarn application id
while (matcher.find()) { while (matcher.find()) {
String appId = matcher.group(); String appId = matcher.group();
if(!appIds.contains(appId)){ if (!appIds.contains(appId)) {
logger.info("find app id: {}", appId); logger.info("find app id: {}", appId);
appIds.add(appId); appIds.add(appId);
} }

25
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java

@ -14,27 +14,36 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.slf4j.Logger; import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE;
import org.slf4j.LoggerFactory;
import static java.util.Collections.emptyList;
import java.io.IOException; import java.io.IOException;
import java.net.*; import java.net.Inet6Address;
import java.util.*; import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static java.util.Collections.emptyList; import org.slf4j.Logger;
import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE; import org.slf4j.LoggerFactory;
/** /**
* NetUtils * NetUtils
*/ */
public class NetUtils { public class NetUtils {
private NetUtils() { private NetUtils() {
throw new IllegalStateException("Utility class"); throw new UnsupportedOperationException("Construct NetUtils");
} }
private static Logger logger = LoggerFactory.getLogger(NetUtils.class); private static Logger logger = LoggerFactory.getLogger(NetUtils.class);

62
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@ -14,8 +14,14 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.shell.ShellExecutor;
import org.apache.commons.configuration.Configuration;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
@ -32,9 +38,6 @@ import java.util.Optional;
import java.util.StringTokenizer; import java.util.StringTokenizer;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.configuration.Configuration;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.shell.ShellExecutor;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -45,7 +48,6 @@ import oshi.hardware.HardwareAbstractionLayer;
/** /**
* os utils * os utils
*
*/ */
public class OSUtils { public class OSUtils {
@ -64,7 +66,9 @@ public class OSUtils {
private static HardwareAbstractionLayer hal = SI.getHardware(); private static HardwareAbstractionLayer hal = SI.getHardware();
private OSUtils() {} private OSUtils() {
throw new UnsupportedOperationException("Construct OSUtils");
}
/** /**
* Initialization regularization, solve the problem of pre-compilation performance, * Initialization regularization, solve the problem of pre-compilation performance,
@ -72,10 +76,10 @@ public class OSUtils {
*/ */
private static final Pattern PATTERN = Pattern.compile("\\s+"); private static final Pattern PATTERN = Pattern.compile("\\s+");
/** /**
* get memory usage * get memory usage
* Keep 2 decimal * Keep 2 decimal
*
* @return percent % * @return percent %
*/ */
public static double memoryUsage() { public static double memoryUsage() {
@ -87,16 +91,16 @@ public class OSUtils {
return Double.parseDouble(df.format(memoryUsage)); return Double.parseDouble(df.format(memoryUsage));
} }
/** /**
* get available physical memory size * get available physical memory size
* * <p>
* Keep 2 decimal * Keep 2 decimal
*
* @return available Physical Memory Size, unit: G * @return available Physical Memory Size, unit: G
*/ */
public static double availablePhysicalMemorySize() { public static double availablePhysicalMemorySize() {
GlobalMemory memory = hal.getMemory(); GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) /1024.0/1024/1024; double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL); DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP); df.setRoundingMode(RoundingMode.HALF_UP);
@ -106,20 +110,20 @@ public class OSUtils {
/** /**
* get total physical memory size * get total physical memory size
* * <p>
* Keep 2 decimal * Keep 2 decimal
*
* @return available Physical Memory Size, unit: G * @return available Physical Memory Size, unit: G
*/ */
public static double totalMemorySize() { public static double totalMemorySize() {
GlobalMemory memory = hal.getMemory(); GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = memory.getTotal() /1024.0/1024/1024; double availablePhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL); DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP); df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(availablePhysicalMemorySize)); return Double.parseDouble(df.format(availablePhysicalMemorySize));
} }
/** /**
* load average * load average
* *
@ -194,12 +198,13 @@ public class OSUtils {
/** /**
* get user list from mac * get user list from mac
*
* @return user list * @return user list
*/ */
private static List<String> getUserListFromMac() throws IOException { private static List<String> getUserListFromMac() throws IOException {
String result = exeCmd("dscl . list /users"); String result = exeCmd("dscl . list /users");
if (StringUtils.isNotEmpty(result)) { if (StringUtils.isNotEmpty(result)) {
return Arrays.asList(result.split( "\n")); return Arrays.asList(result.split("\n"));
} }
return Collections.emptyList(); return Collections.emptyList();
@ -207,8 +212,8 @@ public class OSUtils {
/** /**
* get user list from windows * get user list from windows
*
* @return user list * @return user list
* @throws IOException
*/ */
private static List<String> getUserListFromWindows() throws IOException { private static List<String> getUserListFromWindows() throws IOException {
String result = exeCmd("net user"); String result = exeCmd("net user");
@ -247,6 +252,7 @@ public class OSUtils {
/** /**
* create user * create user
*
* @param userName user name * @param userName user name
* @return true if creation was successful, otherwise false * @return true if creation was successful, otherwise false
*/ */
@ -277,6 +283,7 @@ public class OSUtils {
/** /**
* create linux user * create linux user
*
* @param userName user name * @param userName user name
* @param userGroup user group * @param userGroup user group
* @throws IOException in case of an I/O error * @throws IOException in case of an I/O error
@ -295,6 +302,7 @@ public class OSUtils {
/** /**
* create mac user (Supports Mac OSX 10.10+) * create mac user (Supports Mac OSX 10.10+)
*
* @param userName user name * @param userName user name
* @param userGroup user group * @param userGroup user group
* @throws IOException in case of an I/O error * @throws IOException in case of an I/O error
@ -323,6 +331,7 @@ public class OSUtils {
/** /**
* create windows user * create windows user
*
* @param userName user name * @param userName user name
* @param userGroup user group * @param userGroup user group
* @throws IOException in case of an I/O error * @throws IOException in case of an I/O error
@ -347,6 +356,7 @@ public class OSUtils {
/** /**
* get system group information * get system group information
*
* @return system group info * @return system group info
* @throws IOException errors * @throws IOException errors
*/ */
@ -390,6 +400,7 @@ public class OSUtils {
/** /**
* Execute the shell * Execute the shell
*
* @param command command * @param command command
* @return result of execute the shell * @return result of execute the shell
* @throws IOException errors * @throws IOException errors
@ -400,6 +411,7 @@ public class OSUtils {
/** /**
* get process id * get process id
*
* @return process id * @return process id
*/ */
public static int getProcessID() { public static int getProcessID() {
@ -409,15 +421,16 @@ public class OSUtils {
/** /**
* whether is macOS * whether is macOS
*
* @return true if mac * @return true if mac
*/ */
public static boolean isMacOS() { public static boolean isMacOS() {
return getOSName().startsWith("Mac"); return getOSName().startsWith("Mac");
} }
/** /**
* whether is windows * whether is windows
*
* @return true if windows * @return true if windows
*/ */
public static boolean isWindows() { public static boolean isWindows() {
@ -426,6 +439,7 @@ public class OSUtils {
/** /**
* get current OS name * get current OS name
*
* @return current OS name * @return current OS name
*/ */
public static String getOSName() { public static String getOSName() {
@ -434,42 +448,44 @@ public class OSUtils {
/** /**
* check memory and cpu usage * check memory and cpu usage
*
* @param systemCpuLoad systemCpuLoad * @param systemCpuLoad systemCpuLoad
* @param systemReservedMemory systemReservedMemory * @param systemReservedMemory systemReservedMemory
* @return check memory and cpu usage * @return check memory and cpu usage
*/ */
public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory){ public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory) {
// system load average // system load average
double loadAverage = OSUtils.loadAverage(); double loadAverage = OSUtils.loadAverage();
// system available physical memory // system available physical memory
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
if(loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory){ if (loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory) {
logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize, loadAverage);
return false; return false;
}else{ } else {
return true; return true;
} }
} }
/** /**
* check memory and cpu usage * check memory and cpu usage
*
* @param conf conf * @param conf conf
* @param isMaster is master * @param isMaster is master
* @return check memory and cpu usage * @return check memory and cpu usage
*/ */
public static Boolean checkResource(Configuration conf, Boolean isMaster){ public static Boolean checkResource(Configuration conf, Boolean isMaster) {
double systemCpuLoad; double systemCpuLoad;
double systemReservedMemory; double systemReservedMemory;
if(Boolean.TRUE.equals(isMaster)){ if (Boolean.TRUE.equals(isMaster)) {
systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD); systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD);
systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY); systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY);
}else{ } else {
systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD); systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD);
systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY); systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY);
} }
return checkResource(systemCpuLoad,systemReservedMemory); return checkResource(systemCpuLoad, systemReservedMemory);
} }
} }

100
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java

@ -14,10 +14,9 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.DataType;
@ -25,12 +24,17 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.text.ParseException; import java.util.Date;
import java.util.*; import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* parameter parse utils * parameter parse utils
@ -39,6 +43,10 @@ public class ParameterUtils {
private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class); private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class);
private ParameterUtils() {
throw new UnsupportedOperationException("Construct ParameterUtils");
}
/** /**
* convert parameters place holders * convert parameters place holders
* *
@ -57,11 +65,7 @@ public class ParameterUtils {
Date cronTime = null; Date cronTime = null;
if (StringUtils.isNotEmpty(cronTimeStr)) { if (StringUtils.isNotEmpty(cronTimeStr)) {
try { cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME});
} catch (ParseException e) {
logger.error("parse {} exception", cronTimeStr, e);
}
} else { } else {
cronTime = new Date(); cronTime = new Date();
} }
@ -94,12 +98,8 @@ public class ParameterUtils {
Date cronTime = null; Date cronTime = null;
if (StringUtils.isNotEmpty(cronTimeStr)) { if (StringUtils.isNotEmpty(cronTimeStr)) {
try { cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME});
} catch (ParseException e) {
logger.error(String.format("parse %s exception", cronTimeStr), e);
}
} else { } else {
cronTime = new Date(); cronTime = new Date();
} }
@ -115,34 +115,34 @@ public class ParameterUtils {
return parameterString; return parameterString;
} }
/** /**
* set in parameter * set in parameter
*
* @param index index * @param index index
* @param stmt preparedstatement * @param stmt preparedstatement
* @param dataType data type * @param dataType data type
* @param value value * @param value value
* @throws Exception errors * @throws Exception errors
*/ */
public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value)throws Exception{ public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value) throws Exception {
if (dataType.equals(DataType.VARCHAR)){ if (dataType.equals(DataType.VARCHAR)) {
stmt.setString(index,value); stmt.setString(index, value);
}else if (dataType.equals(DataType.INTEGER)){ } else if (dataType.equals(DataType.INTEGER)) {
stmt.setInt(index, Integer.parseInt(value)); stmt.setInt(index, Integer.parseInt(value));
}else if (dataType.equals(DataType.LONG)){ } else if (dataType.equals(DataType.LONG)) {
stmt.setLong(index, Long.parseLong(value)); stmt.setLong(index, Long.parseLong(value));
}else if (dataType.equals(DataType.FLOAT)){ } else if (dataType.equals(DataType.FLOAT)) {
stmt.setFloat(index, Float.parseFloat(value)); stmt.setFloat(index, Float.parseFloat(value));
}else if (dataType.equals(DataType.DOUBLE)){ } else if (dataType.equals(DataType.DOUBLE)) {
stmt.setDouble(index, Double.parseDouble(value)); stmt.setDouble(index, Double.parseDouble(value));
}else if (dataType.equals(DataType.DATE)){ } else if (dataType.equals(DataType.DATE)) {
stmt.setDate(index, java.sql.Date.valueOf(value)); stmt.setDate(index, java.sql.Date.valueOf(value));
}else if (dataType.equals(DataType.TIME)){ } else if (dataType.equals(DataType.TIME)) {
stmt.setString(index, value); stmt.setString(index, value);
}else if (dataType.equals(DataType.TIMESTAMP)){ } else if (dataType.equals(DataType.TIMESTAMP)) {
stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value)); stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value));
}else if (dataType.equals(DataType.BOOLEAN)){ } else if (dataType.equals(DataType.BOOLEAN)) {
stmt.setBoolean(index,Boolean.parseBoolean(value)); stmt.setBoolean(index, Boolean.parseBoolean(value));
} }
} }
@ -155,20 +155,20 @@ public class ParameterUtils {
* @param scheduleTime schedule time * @param scheduleTime schedule time
* @return curing user define parameters * @return curing user define parameters
*/ */
public static String curingGlobalParams(Map<String,String> globalParamMap, List<Property> globalParamList, public static String curingGlobalParams(Map<String, String> globalParamMap, List<Property> globalParamList,
CommandType commandType, Date scheduleTime){ CommandType commandType, Date scheduleTime) {
if (globalParamList == null || globalParamList.isEmpty()) { if (globalParamList == null || globalParamList.isEmpty()) {
return null; return null;
} }
Map<String, String> globalMap = new HashMap<>(); Map<String, String> globalMap = new HashMap<>();
if (globalParamMap!= null){ if (globalParamMap != null) {
globalMap.putAll(globalParamMap); globalMap.putAll(globalParamMap);
} }
Map<String,String> allParamMap = new HashMap<>(); Map<String, String> allParamMap = new HashMap<>();
//If it is a complement, a complement time needs to be passed in, according to the task type //If it is a complement, a complement time needs to be passed in, according to the task type
Map<String,String> timeParams = BusinessTimeUtils Map<String, String> timeParams = BusinessTimeUtils
.getBusinessTime(commandType, scheduleTime); .getBusinessTime(commandType, scheduleTime);
if (timeParams != null) { if (timeParams != null) {
@ -179,45 +179,41 @@ public class ParameterUtils {
Set<Map.Entry<String, String>> entries = allParamMap.entrySet(); Set<Map.Entry<String, String>> entries = allParamMap.entrySet();
Map<String,String> resolveMap = new HashMap<>(); Map<String, String> resolveMap = new HashMap<>();
for (Map.Entry<String,String> entry : entries){ for (Map.Entry<String, String> entry : entries) {
String val = entry.getValue(); String val = entry.getValue();
if (val.startsWith("$")){ if (val.startsWith("$")) {
String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap); String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap);
resolveMap.put(entry.getKey(),str); resolveMap.put(entry.getKey(), str);
} }
} }
globalMap.putAll(resolveMap); globalMap.putAll(resolveMap);
for (Property property : globalParamList){ for (Property property : globalParamList) {
String val = globalMap.get(property.getProp()); String val = globalMap.get(property.getProp());
if (val != null){ if (val != null) {
property.setValue(val); property.setValue(val);
} }
} }
return JSONUtils.toJsonString(globalParamList); return JSONUtils.toJsonString(globalParamList);
} }
/** /**
* handle escapes * handle escapes
*
* @param inputString input string * @param inputString input string
* @return string filter escapes * @return string filter escapes
*/ */
public static String handleEscapes(String inputString){ public static String handleEscapes(String inputString) {
if(StringUtils.isNotEmpty(inputString)){ if (StringUtils.isNotEmpty(inputString)) {
return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_"); return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_");
} }
return inputString; return inputString;
} }
/** /**
* $[yyyyMMdd] replace schedule time * $[yyyyMMdd] replace schedule time
* @param text
* @param scheduleTime
* @return
*/ */
public static String replaceScheduleTime(String text, Date scheduleTime) { public static String replaceScheduleTime(String text, Date scheduleTime) {
Map<String, Property> paramsMap = new HashMap<>(); Map<String, Property> paramsMap = new HashMap<>();
@ -236,19 +232,19 @@ public class ParameterUtils {
return text; return text;
} }
/** /**
* format convert * format convert
*
* @param paramsMap params map * @param paramsMap params map
* @return Map of converted * @return Map of converted
* see org.apache.dolphinscheduler.server.utils.ParamUtils.convert * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert
*/ */
public static Map<String,String> convert(Map<String,Property> paramsMap){ public static Map<String, String> convert(Map<String, Property> paramsMap) {
Map<String,String> map = new HashMap<>(); Map<String, String> map = new HashMap<>();
Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator(); Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator();
while (iter.hasNext()){ while (iter.hasNext()) {
Map.Entry<String, Property> en = iter.next(); Map.Entry<String, Property> en = iter.next();
map.put(en.getKey(),en.getValue().getValue()); map.put(en.getKey(), en.getValue().getValue());
} }
return map; return map;
} }

11
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java

@ -14,16 +14,17 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils;
package org.apache.dolphinscheduler.common.utils;
/** /**
* utility methods for validating input * utility methods for validating input
*
*/ */
public final class Preconditions { public final class Preconditions {
private Preconditions() {} private Preconditions() {
throw new UnsupportedOperationException("Construct Preconditions");
}
/** /**
* if obj is null will throw NPE * if obj is null will throw NPE
@ -41,6 +42,7 @@ public final class Preconditions {
/** /**
* if obj is null will throw NullPointerException with error message * if obj is null will throw NullPointerException with error message
*
* @param obj obj * @param obj obj
* @param errorMsg error message * @param errorMsg error message
* @param <T> T * @param <T> T
@ -53,13 +55,11 @@ public final class Preconditions {
return obj; return obj;
} }
/** /**
* if condition is false will throw an IllegalArgumentException with the given message * if condition is false will throw an IllegalArgumentException with the given message
* *
* @param condition condition * @param condition condition
* @param errorMsg error message * @param errorMsg error message
*
* @throws IllegalArgumentException Thrown, if the condition is violated. * @throws IllegalArgumentException Thrown, if the condition is violated.
*/ */
public static void checkArgument(boolean condition, Object errorMsg) { public static void checkArgument(boolean condition, Object errorMsg) {
@ -68,5 +68,4 @@ public final class Preconditions {
} }
} }
} }

36
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@ -14,13 +14,15 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
@ -28,7 +30,8 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* property utils * property utils
@ -44,7 +47,7 @@ public class PropertyUtils {
private static final Properties properties = new Properties(); private static final Properties properties = new Properties();
private PropertyUtils() { private PropertyUtils() {
throw new IllegalStateException("PropertyUtils class"); throw new UnsupportedOperationException("Construct PropertyUtils");
} }
static { static {
@ -68,10 +71,9 @@ public class PropertyUtils {
} }
/** /**
*
* @return judge whether resource upload startup * @return judge whether resource upload startup
*/ */
public static Boolean getResUploadStartupState(){ public static Boolean getResUploadStartupState() {
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3; return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3;
@ -120,7 +122,6 @@ public class PropertyUtils {
} }
/** /**
*
* @param key key * @param key key
* @param defaultValue default value * @param defaultValue default value
* @return property value * @return property value
@ -134,7 +135,7 @@ public class PropertyUtils {
try { try {
return Integer.parseInt(value); return Integer.parseInt(value);
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
logger.info(e.getMessage(),e); logger.info(e.getMessage(), e);
} }
return defaultValue; return defaultValue;
} }
@ -147,7 +148,7 @@ public class PropertyUtils {
*/ */
public static boolean getBoolean(String key) { public static boolean getBoolean(String key) {
String value = properties.getProperty(key.trim()); String value = properties.getProperty(key.trim());
if(null != value){ if (null != value) {
return Boolean.parseBoolean(value); return Boolean.parseBoolean(value);
} }
@ -163,7 +164,7 @@ public class PropertyUtils {
*/ */
public static Boolean getBoolean(String key, boolean defaultValue) { public static Boolean getBoolean(String key, boolean defaultValue) {
String value = properties.getProperty(key.trim()); String value = properties.getProperty(key.trim());
if(null != value){ if (null != value) {
return Boolean.parseBoolean(value); return Boolean.parseBoolean(value);
} }
@ -172,6 +173,7 @@ public class PropertyUtils {
/** /**
* get property long value * get property long value
*
* @param key key * @param key key
* @param defaultVal default value * @param defaultVal default value
* @return property value * @return property value
@ -182,16 +184,14 @@ public class PropertyUtils {
} }
/** /**
*
* @param key key * @param key key
* @return property value * @return property value
*/ */
public static long getLong(String key) { public static long getLong(String key) {
return getLong(key,-1); return getLong(key, -1);
} }
/** /**
*
* @param key key * @param key key
* @param defaultVal default value * @param defaultVal default value
* @return property value * @return property value
@ -201,9 +201,9 @@ public class PropertyUtils {
return val == null ? defaultVal : Double.parseDouble(val); return val == null ? defaultVal : Double.parseDouble(val);
} }
/** /**
* get array * get array
*
* @param key property name * @param key property name
* @param splitStr separator * @param splitStr separator
* @return property value through array * @return property value through array
@ -217,13 +217,12 @@ public class PropertyUtils {
String[] propertyArray = value.split(splitStr); String[] propertyArray = value.split(splitStr);
return propertyArray; return propertyArray;
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
logger.info(e.getMessage(),e); logger.info(e.getMessage(), e);
} }
return new String[0]; return new String[0];
} }
/** /**
*
* @param key key * @param key key
* @param type type * @param type type
* @param defaultValue default value * @param defaultValue default value
@ -238,6 +237,7 @@ public class PropertyUtils {
/** /**
* get all properties with specified prefix, like: fs. * get all properties with specified prefix, like: fs.
*
* @param prefix prefix to search * @param prefix prefix to search
* @return all properties with specified prefix * @return all properties with specified prefix
*/ */
@ -253,11 +253,9 @@ public class PropertyUtils {
/** /**
* *
* @param key
* @param value
*/ */
public static void setValue(String key, String value) { public static void setValue(String key, String value) {
properties.setProperty(key,value); properties.setProperty(key, value);
} }
} }

14
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java

@ -17,15 +17,21 @@
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import com.github.rholder.retry.*;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.Retryer;
import com.github.rholder.retry.RetryerBuilder;
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
/** /**
* The Retryer util. * The Retryer util.
*/ */
@ -35,7 +41,7 @@ public class RetryerUtils {
private static Retryer<Boolean> defaultRetryerResultNoCheck; private static Retryer<Boolean> defaultRetryerResultNoCheck;
private RetryerUtils() { private RetryerUtils() {
throw new UnsupportedOperationException("Construct RetryerUtils");
} }
private static Retryer<Boolean> getDefaultRetryerResultNoCheck() { private static Retryer<Boolean> getDefaultRetryerResultNoCheck() {

50
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java

@ -14,10 +14,8 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils;
import org.slf4j.Logger; package org.apache.dolphinscheduler.common.utils;
import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
@ -29,51 +27,58 @@ import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* Metadata related common classes * Metadata related common classes
*
*/ */
public class SchemaUtils { public class SchemaUtils {
private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class);
private static Pattern p = Pattern.compile("\\s*|\t|\r|\n"); private static Pattern p = Pattern.compile("\\s*|\t|\r|\n");
private SchemaUtils() {
throw new UnsupportedOperationException("Construct SchemaUtils");
}
/** /**
* Gets upgradable schemas for all upgrade directories * Gets upgradable schemas for all upgrade directories
*
* @return all schema list * @return all schema list
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public static List<String> getAllSchemaList() { public static List<String> getAllSchemaList() {
List<String> schemaDirList = new ArrayList<>(); List<String> schemaDirList = new ArrayList<>();
File[] schemaDirArr = FileUtils.getAllDir("sql/upgrade"); File[] schemaDirArr = FileUtils.getAllDir("sql/upgrade");
if(schemaDirArr == null || schemaDirArr.length == 0) { if (schemaDirArr == null || schemaDirArr.length == 0) {
return null; return null;
} }
for(File file : schemaDirArr) { for (File file : schemaDirArr) {
schemaDirList.add(file.getName()); schemaDirList.add(file.getName());
} }
Collections.sort(schemaDirList , new Comparator() { Collections.sort(schemaDirList, new Comparator() {
@Override @Override
public int compare(Object o1 , Object o2){ public int compare(Object o1, Object o2) {
try { try {
String dir1 = String.valueOf(o1); String dir1 = String.valueOf(o1);
String dir2 = String.valueOf(o2); String dir2 = String.valueOf(o2);
String version1 = dir1.split("_")[0]; String version1 = dir1.split("_")[0];
String version2 = dir2.split("_")[0]; String version2 = dir2.split("_")[0];
if(version1.equals(version2)) { if (version1.equals(version2)) {
return 0; return 0;
} }
if(SchemaUtils.isAGreatVersion(version1, version2)) { if (SchemaUtils.isAGreatVersion(version1, version2)) {
return 1; return 1;
} }
return -1; return -1;
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(),e); logger.error(e.getMessage(), e);
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
@ -84,22 +89,23 @@ public class SchemaUtils {
/** /**
* Determine whether schemaVersion is higher than version * Determine whether schemaVersion is higher than version
*
* @param schemaVersion schema version * @param schemaVersion schema version
* @param version version * @param version version
* @return Determine whether schemaVersion is higher than version * @return Determine whether schemaVersion is higher than version
*/ */
public static boolean isAGreatVersion(String schemaVersion, String version) { public static boolean isAGreatVersion(String schemaVersion, String version) {
if(StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) { if (StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) {
throw new RuntimeException("schemaVersion or version is empty"); throw new RuntimeException("schemaVersion or version is empty");
} }
String[] schemaVersionArr = schemaVersion.split("\\."); String[] schemaVersionArr = schemaVersion.split("\\.");
String[] versionArr = version.split("\\."); String[] versionArr = version.split("\\.");
int arrLength = Math.min(schemaVersionArr.length, versionArr.length); int arrLength = Math.min(schemaVersionArr.length, versionArr.length);
for(int i = 0 ; i < arrLength ; i++) { for (int i = 0; i < arrLength; i++) {
if(Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) { if (Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) {
return true; return true;
}else if(Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) { } else if (Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) {
return false; return false;
} }
} }
@ -110,28 +116,30 @@ public class SchemaUtils {
/** /**
* Gets the current software version number of the system * Gets the current software version number of the system
*
* @return current software version * @return current software version
*/ */
public static String getSoftVersion() { public static String getSoftVersion() {
String soft_version; String softVersion;
try { try {
soft_version = FileUtils.readFile2Str(new FileInputStream(new File("sql/soft_version"))); softVersion = FileUtils.readFile2Str(new FileInputStream(new File("sql/soft_version")));
soft_version = replaceBlank(soft_version); softVersion = replaceBlank(softVersion);
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
logger.error(e.getMessage(),e); logger.error(e.getMessage(), e);
throw new RuntimeException("Failed to get the product version description file. The file could not be found", e); throw new RuntimeException("Failed to get the product version description file. The file could not be found", e);
} }
return soft_version; return softVersion;
} }
/** /**
* Strips the string of space carriage returns and tabs * Strips the string of space carriage returns and tabs
*
* @param str string * @param str string
* @return string removed blank * @return string removed blank
*/ */
public static String replaceBlank(String str) { public static String replaceBlank(String str) {
String dest = ""; String dest = "";
if (str!=null) { if (str != null) {
Matcher m = p.matcher(str); Matcher m = p.matcher(str);
dest = m.replaceAll(""); dest = m.replaceAll("");

8
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java

@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
/** /**
@ -24,11 +24,15 @@ import org.apache.dolphinscheduler.common.Constants;
*/ */
public class SensitiveLogUtils { public class SensitiveLogUtils {
private SensitiveLogUtils() {
throw new UnsupportedOperationException("Construct SensitiveLogUtils");
}
/** /**
* @param dataSourcePwd data source password * @param dataSourcePwd data source password
* @return String * @return String
*/ */
public static String maskDataSourcePwd(String dataSourcePwd){ public static String maskDataSourcePwd(String dataSourcePwd) {
if (StringUtils.isNotEmpty(dataSourcePwd)) { if (StringUtils.isNotEmpty(dataSourcePwd)) {
dataSourcePwd = Constants.PASSWORD_DEFAULT; dataSourcePwd = Constants.PASSWORD_DEFAULT;

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import java.util.Iterator; import java.util.Iterator;
@ -22,7 +23,9 @@ import java.util.stream.StreamSupport;
public class StreamUtils { public class StreamUtils {
private StreamUtils() { } private StreamUtils() {
throw new UnsupportedOperationException("Construct StreamUtils");
}
public static <T> Stream<T> asStream(Iterator<T> sourceIterator) { public static <T> Stream<T> asStream(Iterator<T> sourceIterator) {
return asStream(sourceIterator, false); return asStream(sourceIterator, false);

12
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@ -14,11 +14,17 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
public class StringUtils { public class StringUtils {
public static final String EMPTY = ""; public static final String EMPTY = "";
private StringUtils() {
throw new UnsupportedOperationException("Construct StringUtils");
}
public static boolean isEmpty(final CharSequence cs) { public static boolean isEmpty(final CharSequence cs) {
return cs == null || cs.length() == 0; return cs == null || cs.length() == 0;
} }
@ -27,18 +33,18 @@ public class StringUtils {
return !isEmpty(cs); return !isEmpty(cs);
} }
public static boolean isBlank(String s){ public static boolean isBlank(String s) {
if (isEmpty(s)) { if (isEmpty(s)) {
return true; return true;
} }
return s.trim().length() == 0; return s.trim().length() == 0;
} }
public static boolean isNotBlank(String s){ public static boolean isNotBlank(String s) {
return !isBlank(s); return !isBlank(s);
} }
public static String replaceNRTtoUnderline(String src){ public static String replaceNRTtoUnderline(String src) {
return src.replaceAll("[\n|\r|\t]", "_"); return src.replaceAll("[\n|\r|\t]", "_");
} }
} }

12
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java

@ -14,13 +14,14 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters;
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters;
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.apache.dolphinscheduler.common.task.http.HttpParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters;
import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters; import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters;
@ -31,10 +32,10 @@ import org.apache.dolphinscheduler.common.task.spark.SparkParameters;
import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
/** /**
* task parameters utils * task parameters utils
*/ */
@ -42,15 +43,20 @@ public class TaskParametersUtils {
private static Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class); private static Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class);
private TaskParametersUtils() {
throw new UnsupportedOperationException("Construct TaskParametersUtils");
}
/** /**
* get task parameters * get task parameters
*
* @param taskType task type * @param taskType task type
* @param parameter parameter * @param parameter parameter
* @return task parameters * @return task parameters
*/ */
public static AbstractParameters getParameters(String taskType, String parameter) { public static AbstractParameters getParameters(String taskType, String parameter) {
try { try {
switch (EnumUtils.getEnum(TaskType.class,taskType)) { switch (EnumUtils.getEnum(TaskType.class, taskType)) {
case SUB_PROCESS: case SUB_PROCESS:
return JSONUtils.parseObject(parameter, SubProcessParameters.class); return JSONUtils.parseObject(parameter, SubProcessParameters.class);
case WATERDROP: case WATERDROP:

Loading…
Cancel
Save