Browse Source

Merge pull request #6 from apache/dev

merge
pull/2/head
Tboy 5 years ago committed by GitHub
parent
commit
4b5f60d96e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 9
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java
  2. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java
  3. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
  4. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java
  5. 6
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  6. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java
  7. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java
  8. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java
  9. 60
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java
  10. 8
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java
  11. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/FailureStrategy.java
  12. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java
  13. 9
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java
  14. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java
  15. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceType.java
  16. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java
  17. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ShowType.java
  18. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SparkVersion.java
  19. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java
  20. 9
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java
  21. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java
  22. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java
  23. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java
  24. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java
  25. 48
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java
  26. 8
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java
  27. 130
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java
  28. 327
      dolphinscheduler-dist/pom.xml
  29. 21
      dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml
  30. 236
      dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml
  31. 28
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtil.java
  32. 92
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverter.java
  33. 96
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java
  34. 61
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java
  35. 4
      dolphinscheduler-server/src/main/resources/worker_logback.xml
  36. 37
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtilTest.java
  37. 92
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverterTest.java
  38. 165
      dolphinscheduler-ui/pom.xml
  39. 17
      pom.xml

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java

@ -154,8 +154,13 @@ public class AccessTokenService extends BaseService {
*/
public Map<String, Object> updateToken(int id,int userId, String expireTime, String token) {
Map<String, Object> result = new HashMap<>(5);
AccessToken accessToken = new AccessToken();
accessToken.setId(id);
AccessToken accessToken = accessTokenMapper.selectById(id);
if (accessToken == null) {
logger.error("access token not exist, access token id {}", id);
putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST);
return result;
}
accessToken.setUserId(userId);
accessToken.setExpireTime(DateUtils.stringToDate(expireTime));
accessToken.setToken(token);

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java

@ -193,6 +193,12 @@ public class AlertGroupService extends BaseService{
if (checkAdmin(loginUser, result)){
return result;
}
//check exist
AlertGroup alertGroup = alertGroupMapper.selectById(id);
if (alertGroup == null) {
putMsg(result, Status.ALERT_GROUP_NOT_EXIST);
return result;
}
userAlertGroupMapper.deleteByAlertgroupId(id);
alertGroupMapper.deleteById(id);

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@ -210,7 +210,6 @@ public class ResourcesService extends BaseService {
}
Resource resource = resourcesMapper.selectById(resourceId);
String originResourceName = resource.getAlias();
if (resource == null) {
putMsg(result, Status.RESOURCE_NOT_EXIST);
return result;
@ -236,6 +235,7 @@ public class ResourcesService extends BaseService {
}
//get the file suffix
String originResourceName = resource.getAlias();
String suffix = originResourceName.substring(originResourceName.lastIndexOf("."));
//if the name without suffix then add it ,else use the origin name

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java

@ -104,7 +104,7 @@ public class CheckUtils {
* @return true if phone regex valid, otherwise return false
*/
public static boolean checkPhone(String phone) {
return StringUtils.isEmpty(phone) || phone.length() <= 11;
return StringUtils.isEmpty(phone) || phone.length() == 11;
}

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -36,7 +36,6 @@ public final class Constants {
*/
public static final String HADOOP_PROPERTIES_PATH = "/common/hadoop/hadoop.properties";
/**
* common properties path
*/
@ -1007,4 +1006,9 @@ public final class Constants {
public static final String RECEIVERS = "receivers";
public static final String RECEIVERS_CC = "receiversCc";
/**
* dataSource sensitive param
*/
public static final String DATASOURCE_PASSWORD_REGEX = "(?<=(\"password\":\")).*?(?=(\"))";
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* alert status
*/
@Getter
public enum AlertStatus {
/**
* 0 waiting executed; 1 execute successfully2 execute failed
@ -40,4 +38,12 @@ public enum AlertStatus {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* warning message notification method
*/
@Getter
public enum AlertType {
/**
* 0 email; 1 SMS
@ -39,4 +37,12 @@ public enum AlertType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* command types
*/
@Getter
public enum CommandType {
/**
@ -59,4 +57,12 @@ public enum CommandType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

60
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java

@ -17,38 +17,44 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* data base types
*/
@Getter
public enum DbType {
/**
* 0 mysql
* 1 postgresql
* 2 hive
* 3 spark
* 4 clickhouse
* 5 oracle
* 6 sqlserver
* 7 db2
*/
MYSQL(0, "mysql"),
POSTGRESQL(1, "postgresql"),
HIVE(2, "hive"),
SPARK(3, "spark"),
CLICKHOUSE(4, "clickhouse"),
ORACLE(5, "oracle"),
SQLSERVER(6, "sqlserver"),
DB2(7, "db2");
/**
* 0 mysql
* 1 postgresql
* 2 hive
* 3 spark
* 4 clickhouse
* 5 oracle
* 6 sqlserver
* 7 db2
*/
MYSQL(0, "mysql"),
POSTGRESQL(1, "postgresql"),
HIVE(2, "hive"),
SPARK(3, "spark"),
CLICKHOUSE(4, "clickhouse"),
ORACLE(5, "oracle"),
SQLSERVER(6, "sqlserver"),
DB2(7, "db2");
DbType(int code, String descp){
this.code = code;
this.descp = descp;
}
DbType(int code, String descp) {
this.code = code;
this.descp = descp;
}
@EnumValue
private final int code;
private final String descp;
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

8
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java

@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* running status for workflow and task nodes
*
*/
@Getter
public enum ExecutionStatus {
/**
@ -123,5 +121,11 @@ public enum ExecutionStatus {
return this == KILL || this == STOP ;
}
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/FailureStrategy.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* failure policy when some task node failed.
*/
@Getter
public enum FailureStrategy {
/**
@ -40,4 +38,12 @@ public enum FailureStrategy {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* have_script
@ -27,7 +26,6 @@ import lombok.Getter;
* have_map_variables
* have_alert
*/
@Getter
public enum Flag {
/**
* 0 no
@ -45,4 +43,12 @@ public enum Flag {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

9
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* define process and task priority
*/
@Getter
public enum Priority {
/**
* 0 highest priority
@ -46,4 +44,11 @@ public enum Priority {
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* process define release state
*/
@Getter
public enum ReleaseState {
/**
@ -50,4 +48,12 @@ public enum ReleaseState {
//For values out of enum scope
return null;
}
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceType.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* resource type
*/
@Getter
public enum ResourceType {
/**
* 0 file, 1 udf
@ -39,4 +37,12 @@ public enum ResourceType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* complement data run mode
*/
@Getter
public enum RunMode {
/**
* 0 serial run
@ -39,4 +37,12 @@ public enum RunMode {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ShowType.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* show type for email
*/
@Getter
public enum ShowType {
/**
* 0 TABLE;
@ -44,4 +42,12 @@ public enum ShowType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SparkVersion.java

@ -17,9 +17,7 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
@Getter
public enum SparkVersion {
/**
@ -37,4 +35,12 @@ public enum SparkVersion {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* task node depend type
*/
@Getter
public enum TaskDependType {
/**
* 0 run current tasks only
@ -41,4 +39,12 @@ public enum TaskDependType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

9
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* task node type
*/
@Getter
public enum TaskType {
/**
* 0 SHELL
@ -61,4 +59,11 @@ public enum TaskType {
return !(taskType == TaskType.SUB_PROCESS || taskType == TaskType.DEPENDENT);
}
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* UDF type
*/
@Getter
public enum UdfType {
/**
* 0 hive; 1 spark
@ -38,4 +36,12 @@ public enum UdfType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* user type
*/
@Getter
public enum UserType {
/**
* 0 admin user; 1 general user
@ -39,5 +37,13 @@ public enum UserType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* types for whether to send warning when process ending;
*/
@Getter
public enum WarningType {
/**
* 0 do not send warning;
@ -44,4 +42,12 @@ public enum WarningType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java

@ -461,7 +461,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
if (serverHost.equals(OSUtils.getHost())) {
logger.error("{} server({}) of myself dead , stopping...",
zkNodeType.toString(), serverHost);
stoppable.stop(String.format(" {} server {} of myself dead , stopping...",
stoppable.stop(String.format(" %s server %s of myself dead , stopping...",
zkNodeType.toString(), serverHost));
return true;
}

48
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java

@ -16,8 +16,10 @@
*/
package org.apache.dolphinscheduler.common.zk;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.TreeCache;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
import org.apache.curator.framework.recipes.cache.TreeCacheListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -34,30 +36,37 @@ public class ZookeeperCachedOperator extends ZookeeperOperator {
private final Logger logger = LoggerFactory.getLogger(ZookeeperCachedOperator.class);
//kay is zk path, value is TreeCache
private ConcurrentHashMap<String, TreeCache> allCaches = new ConcurrentHashMap<>();
TreeCache treeCache;
/**
* @param cachePath zk path
* @param listener operator
* register a unified listener of /${dsRoot},
*/
public void registerListener(final String cachePath, final TreeCacheListener listener) {
TreeCache newCache = new TreeCache(zkClient, cachePath);
logger.info("add listener to zk path: {}", cachePath);
@Override
protected void registerListener() {
treeCache = new TreeCache(zkClient, getZookeeperConfig().getDsRoot());
logger.info("add listener to zk path: {}", getZookeeperConfig().getDsRoot());
try {
newCache.start();
treeCache.start();
} catch (Exception e) {
logger.error("add listener to zk path: {} failed", cachePath);
logger.error("add listener to zk path: {} failed", getZookeeperConfig().getDsRoot());
throw new RuntimeException(e);
}
newCache.getListenable().addListener(listener);
treeCache.getListenable().addListener((client, event) -> {
String path = null == event.getData() ? "" : event.getData().getPath();
if (path.isEmpty()) {
return;
}
dataChanged(client, event, path);
});
allCaches.put(cachePath, newCache);
}
//for sub class
protected void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path){}
public String getFromCache(final String cachePath, final String key) {
ChildData resultInCache = allCaches.get(checkNotNull(cachePath)).getCurrentData(key);
ChildData resultInCache = treeCache.getCurrentData(key);
if (null != resultInCache) {
return null == resultInCache.getData() ? null : new String(resultInCache.getData(), StandardCharsets.UTF_8);
}
@ -65,18 +74,15 @@ public class ZookeeperCachedOperator extends ZookeeperOperator {
}
public TreeCache getTreeCache(final String cachePath) {
return allCaches.get(checkNotNull(cachePath));
return treeCache;
}
public void close() {
allCaches.forEach((path, cache) -> {
cache.close();
try {
Thread.sleep(500);
} catch (InterruptedException ignore) {
}
});
treeCache.close();
try {
Thread.sleep(500);
} catch (InterruptedException ignore) {
}
super.close();
}
}

8
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java

@ -57,11 +57,13 @@ public class ZookeeperOperator implements InitializingBean {
public void afterPropertiesSet() throws Exception {
this.zkClient = buildClient();
initStateLister();
//init();
registerListener();
}
//for subclass
//protected void init(){}
/**
* this method is for sub class,
*/
protected void registerListener(){}
public void initStateLister() {
checkNotNull(zkClient);

130
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java

@ -0,0 +1,130 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang.time.DateUtils;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DataType;
import org.apache.dolphinscheduler.common.enums.Direct;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_TIME;
import static org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils.replacePlaceholders;
public class ParameterUtilsTest {
public static final Logger logger = LoggerFactory.getLogger(ParameterUtilsTest.class);
/**
* Test convertParameterPlaceholders
*/
@Test
public void testConvertParameterPlaceholders() throws Exception {
// parameterString,parameterMap is null
Assert.assertNull(ParameterUtils.convertParameterPlaceholders(null, null));
// parameterString is null,parameterMap is not null
Map<String, String> parameterMap = new HashMap<String,String>();
parameterMap.put("testParameter","testParameter");
Assert.assertNull(ParameterUtils.convertParameterPlaceholders(null, parameterMap));
// parameterString、parameterMap is not null
String parameterString = "test_parameter";
Assert.assertEquals(parameterString, ParameterUtils.convertParameterPlaceholders(parameterString, parameterMap));
//replace variable ${} form
parameterMap.put("testParameter2","${testParameter}");
Assert.assertEquals(parameterString,PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true));
// replace time $[...] form, eg. $[yyyyMMdd]
Date cronTime = new Date();
Assert.assertEquals(parameterString, replacePlaceholders(parameterString, cronTime, true));
// replace time $[...] form, eg. $[yyyyMMdd]
Date cronTimeStr = DateUtils.parseDate("20191220145900", new String[]{PARAMETER_FORMAT_TIME});
Assert.assertEquals(parameterString, replacePlaceholders(parameterString, cronTimeStr, true));
}
/**
* Test curingGlobalParams
*/
@Test
public void testCuringGlobalParams() throws Exception {
//define globalMap
Map<String, String> globalParamMap = new HashMap<>();
globalParamMap.put("globalParams1","Params1");
//define globalParamList
List<Property> globalParamList = new ArrayList<>();
//define scheduleTime
Date scheduleTime = DateUtils.parseDate("20191220145900", new String[]{PARAMETER_FORMAT_TIME});
//test globalParamList is null
String result = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
Assert.assertNull(result);
Assert.assertNull(ParameterUtils.curingGlobalParams(null,null,CommandType.START_CURRENT_TASK_PROCESS,null));
Assert.assertNull(ParameterUtils.curingGlobalParams(globalParamMap,null,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime));
//test globalParamList is not null
Property property=new Property("testGlobalParam", Direct.IN, DataType.VARCHAR,"testGlobalParam");
globalParamList.add(property);
String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime);
Assert.assertEquals(result2, JSONObject.toJSONString(globalParamList));
String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null);
Assert.assertEquals(result3, JSONObject.toJSONString(globalParamList));
String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
Assert.assertEquals(result4, JSONObject.toJSONString(globalParamList));
//test var $ startsWith
globalParamMap.put("bizDate","${system.biz.date}");
globalParamMap.put("b1zCurdate","${system.biz.curdate}");
Property property2=new Property("testParamList1", Direct.IN, DataType.VARCHAR,"testParamList");
Property property3=new Property("testParamList2", Direct.IN, DataType.VARCHAR,"{testParamList1}");
Property property4=new Property("testParamList3", Direct.IN, DataType.VARCHAR,"${b1zCurdate}");
globalParamList.add(property2);
globalParamList.add(property3);
globalParamList.add(property4);
String result5 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
Assert.assertEquals(result5,JSONUtils.toJsonString(globalParamList));
}
/**
* Test handleEscapes
*/
@Test
public void testHandleEscapes() throws Exception {
Assert.assertNull(ParameterUtils.handleEscapes(null));
Assert.assertEquals("",ParameterUtils.handleEscapes(""));
Assert.assertEquals("test Parameter",ParameterUtils.handleEscapes("test Parameter"));
Assert.assertEquals("////%test////%Parameter",ParameterUtils.handleEscapes("%test%Parameter"));
}
}

327
dolphinscheduler-dist/pom.xml vendored

@ -101,6 +101,333 @@
</plugins>
</build>
</profile>
<profile>
<id>nginx</id>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>dolphinscheduler-nginx</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-nginx.xml</descriptor>
</descriptors>
<appendAssemblyId>true</appendAssemblyId>
</configuration>
</execution>
<execution>
<id>src</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-src.xml</descriptor>
</descriptors>
<appendAssemblyId>true</appendAssemblyId>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<phase>verify</phase>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>rpmbuild</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<excludeScope>provided</excludeScope>
</configuration>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<extensions>true</extensions>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>attached-rpm</goal>
</goals>
</execution>
</executions>
<configuration>
<name>apache-dolphinscheduler-incubating</name>
<release>1</release>
<distribution>apache dolphinscheduler incubating rpm</distribution>
<group>apache</group>
<packager>dolphinscheduler</packager>
<!-- <version>${project.version}</version> -->
<prefix>/opt/soft</prefix>
<defineStatements>
<!-- disable compile python when rpm build -->
<defineStatement>__os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')</defineStatement>
</defineStatements>
<mappings>
<mapping>
<directory>/opt/soft/${project.build.finalName}/conf</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../dolphinscheduler-alert/src/main/resources
</location>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.ftl</include>
</includes>
</source>
<source>
<location>
${basedir}/../dolphinscheduler-common/src/main/resources
</location>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
</source>
<source>
<location>
${basedir}/../dolphinscheduler-dao/src/main/resources
</location>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.yml</include>
</includes>
</source>
<source>
<location>
${basedir}/../dolphinscheduler-api/src/main/resources
</location>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
</source>
<source>
<location>
${basedir}/../dolphinscheduler-server/src/main/resources
</location>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>config/*.*</include>
</includes>
</source>
<source>
<location>
${basedir}/../script
</location>
<includes>
<include>env/*.*</include>
</includes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}/lib</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../dolphinscheduler-dist/target/lib
</location>
<includes>
<include>*.*</include>
</includes>
<excludes>
<exclude>servlet-api-*.jar</exclude>
<exclude>slf4j-log4j12-${slf4j.log4j12.version}.jar</exclude>
</excludes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}/bin</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../script
</location>
<includes>
<include>start-all.sh</include>
<include>stop-all.sh</include>
<include>dolphinscheduler-daemon.sh</include>
</includes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../
</location>
<includes>
<include>*.sh</include>
<include>*.py</include>
<include>DISCLAIMER</include>
</includes>
</source>
<source>
<location>
${basedir}/../dolphinscheduler-ui
</location>
<includes>
<include>install-dolphinscheduler-ui.sh</include>
</includes>
</source>
<source>
<location>
${basedir}/release-docs
</location>
<includes>
<include>**/*</include>
</includes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}/dist</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../dolphinscheduler-ui/dist
</location>
<includes>
<include>**/*.*</include>
</includes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}/sql</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../sql
</location>
<includes>
<include>**/*.*</include>
</includes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}/script</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../script
</location>
<includes>
<include>**/*.*</include>
</includes>
</source>
</sources>
</mapping>
</mappings>
<preinstallScriptlet>
<script>mkdir -p /opt/soft</script>
</preinstallScriptlet>
<postremoveScriptlet>
<script>rm -rf /opt/soft/apache-dolphinscheduler-incubating-${project.version}</script>
</postremoveScriptlet>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>

21
dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml vendored

@ -94,26 +94,12 @@
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>config/*.*</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources/bin</directory>
<includes>
<include>*.*</include>
</includes>
<directoryMode>755</directoryMode>
<outputDirectory>bin</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-dao/src/main/resources</directory>
<includes>
@ -177,7 +163,6 @@
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>config/*.*</include>
<include>env/*.*</include>
</includes>
<outputDirectory>./conf</outputDirectory>

236
dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml vendored

@ -0,0 +1,236 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>dolphinscheduler-nginx</id>
<formats>
<format>tar.gz</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<baseDirectory>${project.build.finalName}-dolphinscheduler-bin</baseDirectory>
<fileSets>
<!--alert start-->
<fileSet>
<directory>${basedir}/../dolphinscheduler-alert/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.ftl</include>
</includes>
<outputDirectory>./conf</outputDirectory>
</fileSet>
<!--alert end-->
<!--api start-->
<fileSet>
<directory>src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources/bin</directory>
<includes>
<include>*.*</include>
</includes>
<directoryMode>755</directoryMode>
<outputDirectory>bin</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-dao/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-api/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<!--api end-->
<!--server start-->
<fileSet>
<directory>${basedir}/../dolphinscheduler-server/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>config/*.*</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources/bin</directory>
<includes>
<include>*.*</include>
</includes>
<directoryMode>755</directoryMode>
<outputDirectory>bin</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-dao/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.yml</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<!--server end-->
<fileSet>
<directory>${basedir}/../dolphinscheduler-server/target/dolphinscheduler-server-${project.version}</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-api/target/dolphinscheduler-api-${project.version}</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-alert/target/dolphinscheduler-alert-${project.version}</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-ui/dist</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>./ui/dist</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-ui</directory>
<includes>
<include>install-dolphinscheduler-ui.sh</include>
</includes>
<outputDirectory>./ui</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../sql</directory>
<includes>
<include>**/*</include>
</includes>
<outputDirectory>./sql</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>*.*</include>
</includes>
<outputDirectory>./script</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>env/*.*</include>
</includes>
<outputDirectory>./conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>start-all.sh</include>
<include>stop-all.sh</include>
<include>dolphinscheduler-daemon.sh</include>
</includes>
<outputDirectory>./bin</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/.././</directory>
<includes>
<include>*.sh</include>
<include>*.py</include>
<include>DISCLAIMER</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/release-docs</directory>
<useDefaultExcludes>true</useDefaultExcludes>
<includes>
<include>**/*</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<excludes>
<exclude>javax.servlet:servlet-api</exclude>
<exclude>org.eclipse.jetty.aggregate:jetty-all</exclude>
<exclude>org.slf4j:slf4j-log4j12</exclude>
</excludes>
</dependencySet>
</dependencySets>
</assembly>

28
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java → dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtil.java

@ -14,22 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.zk;
package org.apache.dolphinscheduler.server.utils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
import org.apache.curator.framework.recipes.cache.TreeCacheListener;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
public abstract class AbstractListener implements TreeCacheListener {
/**
* sensitive log Util
*/
public class SensitiveLogUtil {
/**
* @param dataSourcePwd data source password
* @return String
*/
public static String maskDataSourcePwd(String dataSourcePwd){
@Override
public final void childEvent(final CuratorFramework client, final TreeCacheEvent event) throws Exception {
String path = null == event.getData() ? "" : event.getData().getPath();
if (path.isEmpty()) {
return;
if (StringUtils.isNotEmpty(dataSourcePwd)) {
dataSourcePwd = Constants.PASSWORD_DEFAULT;
}
dataChanged(client, event, path);
return dataSourcePwd;
}
protected abstract void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path);
}

92
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverter.java

@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.worker.log;
import ch.qos.logback.classic.pattern.MessageConverter;
import ch.qos.logback.classic.spi.ILoggingEvent;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.server.utils.SensitiveLogUtil;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* sensitive data log converter
*/
@Slf4j
public class SensitiveDataConverter extends MessageConverter {
/**
* password pattern
*/
private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX);
@Override
public String convert(ILoggingEvent event) {
// get original log
String requestLogMsg = event.getFormattedMessage();
// desensitization log
return convertMsg(requestLogMsg);
}
/**
* deal with sensitive log
*
* @param oriLogMsg original log
*/
private String convertMsg(final String oriLogMsg) {
String tempLogMsg = oriLogMsg;
if (StringUtils.isNotEmpty(tempLogMsg)) {
tempLogMsg = passwordHandler(pwdPattern, tempLogMsg);
}
return tempLogMsg;
}
/**
* password regex
*
* @param logMsg original log
*/
private String passwordHandler(Pattern pwdPattern, String logMsg) {
Matcher matcher = pwdPattern.matcher(logMsg);
StringBuffer sb = new StringBuffer(logMsg.length());
while (matcher.find()) {
String password = matcher.group();
String maskPassword = SensitiveLogUtil.maskDataSourcePwd(password);
matcher.appendReplacement(sb, maskPassword);
}
matcher.appendTail(sb);
return sb.toString();
}
}

96
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.ZKNodeType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.zk.AbstractListener;
import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.DaoFactory;
@ -31,9 +30,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.ProcessUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.utils.ThreadUtils;
import org.slf4j.Logger;
@ -101,12 +97,6 @@ public class ZKMasterClient extends AbstractZKClient {
// init system znode
this.initSystemZNode();
// monitor master
this.listenerMaster();
// monitor worker
this.listenerWorker();
// register master
this.registerMaster();
@ -158,31 +148,22 @@ public class ZKMasterClient extends AbstractZKClient {
}
}
/**
* monitor master
* handle path events that this class cares about
* @param client zkClient
* @param event path event
* @param path zk path
*/
public void listenerMaster(){
registerListener(getZNodeParentPath(ZKNodeType.MASTER), new AbstractListener() {
@Override
protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) {
switch (event.getType()) {
case NODE_ADDED:
logger.info("master node added : {}", path);
break;
case NODE_REMOVED:
String serverHost = getHostByEventDataPath(path);
if (checkServerSelfDead(serverHost, ZKNodeType.MASTER)) {
return;
}
removeZKNodePath(path, ZKNodeType.MASTER, true);
break;
default:
break;
}
}
});
}
@Override
protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) {
if(path.startsWith(getZNodeParentPath(ZKNodeType.MASTER)+Constants.SINGLE_SLASH)){ //monitor master
handleMasterEvent(event,path);
}else if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){ //monitor worker
handleWorkerEvent(event,path);
}
//other path event, ignore
}
/**
* remove zookeeper node path
@ -273,25 +254,40 @@ public class ZKMasterClient extends AbstractZKClient {
}
/**
* monitor worker
* monitor master
*/
public void listenerWorker(){
registerListener(getZNodeParentPath(ZKNodeType.WORKER), new AbstractListener() {
@Override
protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) {
switch (event.getType()) {
case NODE_ADDED:
logger.info("worker node added : {}", path);
break;
case NODE_REMOVED:
logger.info("worker node deleted : {}", path);
removeZKNodePath(path, ZKNodeType.WORKER, true);
break;
default:
break;
public void handleMasterEvent(TreeCacheEvent event, String path){
switch (event.getType()) {
case NODE_ADDED:
logger.info("master node added : {}", path);
break;
case NODE_REMOVED:
String serverHost = getHostByEventDataPath(path);
if (checkServerSelfDead(serverHost, ZKNodeType.MASTER)) {
return;
}
}
});
removeZKNodePath(path, ZKNodeType.MASTER, true);
break;
default:
break;
}
}
/**
* monitor worker
*/
public void handleWorkerEvent(TreeCacheEvent event, String path){
switch (event.getType()) {
case NODE_ADDED:
logger.info("worker node added : {}", path);
break;
case NODE_REMOVED:
logger.info("worker node deleted : {}", path);
removeZKNodePath(path, ZKNodeType.WORKER, true);
break;
default:
break;
}
}

61
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java

@ -19,20 +19,13 @@ package org.apache.dolphinscheduler.server.zk;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ZKNodeType;
import org.apache.dolphinscheduler.common.zk.AbstractListener;
import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.curator.utils.ThreadUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.util.concurrent.ThreadFactory;
/**
* zookeeper worker client
@ -61,9 +54,6 @@ public class ZKWorkerClient extends AbstractZKClient {
// init system znode
this.initSystemZNode();
// monitor worker
this.listenerWorker();
// register worker
this.registWorker();
}
@ -83,31 +73,38 @@ public class ZKWorkerClient extends AbstractZKClient {
System.exit(-1);
}
}
/**
* monitor worker
* handle path events that this class cares about
* @param client zkClient
* @param event path event
* @param path zk path
*/
private void listenerWorker(){
registerListener(getZNodeParentPath(ZKNodeType.WORKER), new AbstractListener() {
@Override
protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) {
switch (event.getType()) {
case NODE_ADDED:
logger.info("worker node added : {}", path);
break;
case NODE_REMOVED:
//find myself dead
String serverHost = getHostByEventDataPath(path);
if(checkServerSelfDead(serverHost, ZKNodeType.WORKER)){
return;
}
break;
default:
break;
}
}
});
@Override
protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) {
if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){
handleWorkerEvent(event,path);
}
}
/**
* monitor worker
*/
public void handleWorkerEvent(TreeCacheEvent event, String path){
switch (event.getType()) {
case NODE_ADDED:
logger.info("worker node added : {}", path);
break;
case NODE_REMOVED:
//find myself dead
String serverHost = getHostByEventDataPath(path);
if(checkServerSelfDead(serverHost, ZKNodeType.WORKER)){
return;
}
break;
default:
break;
}
}
/**

4
dolphinscheduler-server/src/main/resources/worker_logback.xml

@ -18,6 +18,8 @@
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds">
<conversionRule conversionWord="msg"
converterClass="org.apache.dolphinscheduler.server.worker.log.SensitiveDataConverter"/>
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
@ -31,7 +33,7 @@
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
<filter class="org.apache.dolphinscheduler.server.worker.log.TaskLogFilter"></filter>
<filter class="org.apache.dolphinscheduler.server.worker.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.server.worker.log.TaskLogDiscriminator">
<key>taskAppId</key>
<logBase>${log.base}</logBase>

37
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtilTest.java

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.junit.Assert;
import org.junit.Test;
public class SensitiveLogUtilTest {
@Test
public void testMaskDataSourcePwd() {
String password = "123456";
String emptyPassword = "";
Assert.assertEquals(Constants.PASSWORD_DEFAULT, SensitiveLogUtil.maskDataSourcePwd(password));
Assert.assertEquals("", SensitiveLogUtil.maskDataSourcePwd(emptyPassword));
}
}

92
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverterTest.java

@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.worker.log;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.server.utils.SensitiveLogUtil;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class SensitiveDataConverterTest {
private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class);
/**
* password pattern
*/
private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX);
/**
* mask sensitive logMsg - sql task datasource password
*/
@Test
public void testPwdLogMsgConverter() {
String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," +
"\"database\":\"carbond\"," +
"\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," +
"\"user\":\"view\"," +
"\"password\":\"view1\"}";
String maskLogMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," +
"\"database\":\"carbond\"," +
"\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," +
"\"user\":\"view\"," +
"\"password\":\"******\"}";
logger.info("parameter : {}", logMsg);
logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg));
Assert.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg));
Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
}
/**
* password regex test
*
* @param logMsg original log
*/
private static String passwordHandler(Pattern pattern, String logMsg) {
Matcher matcher = pattern.matcher(logMsg);
StringBuffer sb = new StringBuffer(logMsg.length());
while (matcher.find()) {
String password = matcher.group();
String maskPassword = SensitiveLogUtil.maskDataSourcePwd(password);
matcher.appendReplacement(sb, maskPassword);
}
matcher.appendTail(sb);
return sb.toString();
}
}

165
dolphinscheduler-ui/pom.xml

@ -32,55 +32,120 @@
<node.version>v12.12.0</node.version>
<npm.version>6.11.3</npm.version>
</properties>
<profiles>
<profile>
<id>release</id>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>${frontend-maven-plugin.version}</version>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>${node.version}</nodeVersion>
<npmVersion>${npm.version}</npmVersion>
</configuration>
</execution>
<execution>
<id>npm install node-sass --unsafe-perm</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install node-sass --unsafe-perm</arguments>
</configuration>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>npm run build:release</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>run build:release</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>nginx</id>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>${frontend-maven-plugin.version}</version>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>${node.version}</nodeVersion>
<npmVersion>${npm.version}</npmVersion>
</configuration>
</execution>
<execution>
<id>npm install node-sass --unsafe-perm</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install node-sass --unsafe-perm</arguments>
</configuration>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>npm run build</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>run build</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>${frontend-maven-plugin.version}</version>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>${node.version}</nodeVersion>
<npmVersion>${npm.version}</npmVersion>
</configuration>
</execution>
<execution>
<id>npm install node-sass --unsafe-perm</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install node-sass --unsafe-perm</arguments>
</configuration>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>npm run build:release</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>run build:release</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

17
pom.xml

@ -108,6 +108,8 @@
<maven-javadoc-plugin.version>2.10.3</maven-javadoc-plugin.version>
<maven-source-plugin.version>2.4</maven-source-plugin.version>
<maven-surefire-plugin.version>2.18.1</maven-surefire-plugin.version>
<maven-dependency-plugin.version>3.1.1</maven-dependency-plugin.version>
<rpm-maven-plugion.version>2.2.0</rpm-maven-plugion.version>
<jacoco.version>0.8.4</jacoco.version>
<jcip.version>1.0</jcip.version>
<maven.deploy.skip>false</maven.deploy.skip>
@ -524,6 +526,14 @@
<pluginManagement>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<version>${rpm-maven-plugion.version}</version>
<inherited>false</inherited>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
@ -568,6 +578,12 @@
<version>${maven-source-plugin.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
</plugin>
</plugins>
</pluginManagement>
@ -657,6 +673,7 @@
<include>**/server/utils/SparkArgsUtilsTest.java</include>
<include>**/server/utils/FlinkArgsUtilsTest.java</include>
<include>**/dao/mapper/AccessTokenMapperTest.java</include>
<include>**/dao/mapper/AlertGroupMapperTest.java</include>
</includes>
<!-- <skip>true</skip> -->
</configuration>

Loading…
Cancel
Save