github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

change escheduler to dolphinscheduler (#961) 

* rename from DatasourceUserMapper to DataSourceUserMapper

* add unit test in UserMapper and WorkerGroupMapper

* change cn.escheduler to org.apache.dolphinscheduler

* add unit test in UdfFuncMapperTest

* add unit test in UdfFuncMapperTest

* remove DatabaseConfiguration

* add ConnectionFactoryTest

* cal duration in processInstancesList

* change desc to description

* change table name in mysql ddl

* change table name in mysql ddl

* change escheduler to dolphinscheduler

* change escheduler to dolphinscheduler
pull/2/head
lgcareer 5 years ago committed by qiaozhanwei
parent
be59cda0da
commit
31f9fbf335
21 changed files with 180 additions and 181 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 218
      .gitignore
  2. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java
  3. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
  4. 6
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/HttpClientTest.java
  5. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java
  6. 2
      dolphinscheduler-api/src/test/resources/dao/data_source.properties
  7. 34
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  8. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java
  9. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java
  10. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
  11. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  12. 14
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java
  13. 14
      dolphinscheduler-common/src/main/resources/common/common.properties
  14. 14
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueImplTest.java
  15. 5
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java
  16. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
  17. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java
  18. 8
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
  19. 12
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java
  20. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java
  21. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java

218
.gitignore vendored
Unescape View File

@ -35,112 +35,112 @@ config.gypi
test/coverage
/docs/zh_CN/介绍
/docs/zh_CN/贡献代码.md
/escheduler-common/src/main/resources/zookeeper.properties
escheduler-alert/logs/
escheduler-alert/src/main/resources/alert.properties_bak
escheduler-alert/src/main/resources/logback.xml
escheduler-server/src/main/resources/logback.xml
escheduler-ui/dist/css/common.16ac5d9.css
escheduler-ui/dist/css/home/index.b444b91.css
escheduler-ui/dist/css/login/index.5866c64.css
escheduler-ui/dist/js/0.ac94e5d.js
escheduler-ui/dist/js/0.ac94e5d.js.map
escheduler-ui/dist/js/1.0b043a3.js
escheduler-ui/dist/js/1.0b043a3.js.map
escheduler-ui/dist/js/10.1bce3dc.js
escheduler-ui/dist/js/10.1bce3dc.js.map
escheduler-ui/dist/js/11.79f04d8.js
escheduler-ui/dist/js/11.79f04d8.js.map
escheduler-ui/dist/js/12.420daa5.js
escheduler-ui/dist/js/12.420daa5.js.map
escheduler-ui/dist/js/13.e5bae1c.js
escheduler-ui/dist/js/13.e5bae1c.js.map
escheduler-ui/dist/js/14.f2a0dca.js
escheduler-ui/dist/js/14.f2a0dca.js.map
escheduler-ui/dist/js/15.45373e8.js
escheduler-ui/dist/js/15.45373e8.js.map
escheduler-ui/dist/js/16.fecb0fc.js
escheduler-ui/dist/js/16.fecb0fc.js.map
escheduler-ui/dist/js/17.84be279.js
escheduler-ui/dist/js/17.84be279.js.map
escheduler-ui/dist/js/18.307ea70.js
escheduler-ui/dist/js/18.307ea70.js.map
escheduler-ui/dist/js/19.144db9c.js
escheduler-ui/dist/js/19.144db9c.js.map
escheduler-ui/dist/js/2.8b4ef29.js
escheduler-ui/dist/js/2.8b4ef29.js.map
escheduler-ui/dist/js/20.4c527e9.js
escheduler-ui/dist/js/20.4c527e9.js.map
escheduler-ui/dist/js/21.831b2a2.js
escheduler-ui/dist/js/21.831b2a2.js.map
escheduler-ui/dist/js/22.2b4bb2a.js
escheduler-ui/dist/js/22.2b4bb2a.js.map
escheduler-ui/dist/js/23.81467ef.js
escheduler-ui/dist/js/23.81467ef.js.map
escheduler-ui/dist/js/24.54a00e4.js
escheduler-ui/dist/js/24.54a00e4.js.map
escheduler-ui/dist/js/25.8d7bd36.js
escheduler-ui/dist/js/25.8d7bd36.js.map
escheduler-ui/dist/js/26.2ec5e78.js
escheduler-ui/dist/js/26.2ec5e78.js.map
escheduler-ui/dist/js/27.3ab48c2.js
escheduler-ui/dist/js/27.3ab48c2.js.map
escheduler-ui/dist/js/28.363088a.js
escheduler-ui/dist/js/28.363088a.js.map
escheduler-ui/dist/js/29.6c5853a.js
escheduler-ui/dist/js/29.6c5853a.js.map
escheduler-ui/dist/js/3.a0edb5b.js
escheduler-ui/dist/js/3.a0edb5b.js.map
escheduler-ui/dist/js/30.940fdd3.js
escheduler-ui/dist/js/30.940fdd3.js.map
escheduler-ui/dist/js/31.168a460.js
escheduler-ui/dist/js/31.168a460.js.map
escheduler-ui/dist/js/32.8df6594.js
escheduler-ui/dist/js/32.8df6594.js.map
escheduler-ui/dist/js/33.4480bbe.js
escheduler-ui/dist/js/33.4480bbe.js.map
escheduler-ui/dist/js/34.b407fe1.js
escheduler-ui/dist/js/34.b407fe1.js.map
escheduler-ui/dist/js/35.f340b0a.js
escheduler-ui/dist/js/35.f340b0a.js.map
escheduler-ui/dist/js/36.8880c2d.js
escheduler-ui/dist/js/36.8880c2d.js.map
escheduler-ui/dist/js/37.ea2a25d.js
escheduler-ui/dist/js/37.ea2a25d.js.map
escheduler-ui/dist/js/38.98a59ee.js
escheduler-ui/dist/js/38.98a59ee.js.map
escheduler-ui/dist/js/39.a5e958a.js
escheduler-ui/dist/js/39.a5e958a.js.map
escheduler-ui/dist/js/4.4ca44db.js
escheduler-ui/dist/js/4.4ca44db.js.map
escheduler-ui/dist/js/40.e187b1e.js
escheduler-ui/dist/js/40.e187b1e.js.map
escheduler-ui/dist/js/41.0e89182.js
escheduler-ui/dist/js/41.0e89182.js.map
escheduler-ui/dist/js/42.341047c.js
escheduler-ui/dist/js/42.341047c.js.map
escheduler-ui/dist/js/43.27b8228.js
escheduler-ui/dist/js/43.27b8228.js.map
escheduler-ui/dist/js/44.e8869bc.js
escheduler-ui/dist/js/44.e8869bc.js.map
escheduler-ui/dist/js/45.8d54901.js
escheduler-ui/dist/js/45.8d54901.js.map
escheduler-ui/dist/js/5.e1ed7f3.js
escheduler-ui/dist/js/5.e1ed7f3.js.map
escheduler-ui/dist/js/6.241ba07.js
escheduler-ui/dist/js/6.241ba07.js.map
escheduler-ui/dist/js/7.ab2e297.js
escheduler-ui/dist/js/7.ab2e297.js.map
escheduler-ui/dist/js/8.83ff814.js
escheduler-ui/dist/js/8.83ff814.js.map
escheduler-ui/dist/js/9.39cb29f.js
escheduler-ui/dist/js/9.39cb29f.js.map
escheduler-ui/dist/js/common.733e342.js
escheduler-ui/dist/js/common.733e342.js.map
escheduler-ui/dist/js/home/index.78a5d12.js
escheduler-ui/dist/js/home/index.78a5d12.js.map
escheduler-ui/dist/js/login/index.291b8e3.js
escheduler-ui/dist/js/login/index.291b8e3.js.map
escheduler-ui/dist/lib/external/
escheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue
/escheduler-dao/src/main/resources/dao/data_source.properties
/dolphinscheduler-common/src/main/resources/zookeeper.properties
dolphinscheduler-alert/logs/
dolphinscheduler-alert/src/main/resources/alert.properties_bak
dolphinscheduler-alert/src/main/resources/logback.xml
dolphinscheduler-server/src/main/resources/logback.xml
dolphinscheduler-ui/dist/css/common.16ac5d9.css
dolphinscheduler-ui/dist/css/home/index.b444b91.css
dolphinscheduler-ui/dist/css/login/index.5866c64.css
dolphinscheduler-ui/dist/js/0.ac94e5d.js
dolphinscheduler-ui/dist/js/0.ac94e5d.js.map
dolphinscheduler-ui/dist/js/1.0b043a3.js
dolphinscheduler-ui/dist/js/1.0b043a3.js.map
dolphinscheduler-ui/dist/js/10.1bce3dc.js
dolphinscheduler-ui/dist/js/10.1bce3dc.js.map
dolphinscheduler-ui/dist/js/11.79f04d8.js
dolphinscheduler-ui/dist/js/11.79f04d8.js.map
dolphinscheduler-ui/dist/js/12.420daa5.js
dolphinscheduler-ui/dist/js/12.420daa5.js.map
dolphinscheduler-ui/dist/js/13.e5bae1c.js
dolphinscheduler-ui/dist/js/13.e5bae1c.js.map
dolphinscheduler-ui/dist/js/14.f2a0dca.js
dolphinscheduler-ui/dist/js/14.f2a0dca.js.map
dolphinscheduler-ui/dist/js/15.45373e8.js
dolphinscheduler-ui/dist/js/15.45373e8.js.map
dolphinscheduler-ui/dist/js/16.fecb0fc.js
dolphinscheduler-ui/dist/js/16.fecb0fc.js.map
dolphinscheduler-ui/dist/js/17.84be279.js
dolphinscheduler-ui/dist/js/17.84be279.js.map
dolphinscheduler-ui/dist/js/18.307ea70.js
dolphinscheduler-ui/dist/js/18.307ea70.js.map
dolphinscheduler-ui/dist/js/19.144db9c.js
dolphinscheduler-ui/dist/js/19.144db9c.js.map
dolphinscheduler-ui/dist/js/2.8b4ef29.js
dolphinscheduler-ui/dist/js/2.8b4ef29.js.map
dolphinscheduler-ui/dist/js/20.4c527e9.js
dolphinscheduler-ui/dist/js/20.4c527e9.js.map
dolphinscheduler-ui/dist/js/21.831b2a2.js
dolphinscheduler-ui/dist/js/21.831b2a2.js.map
dolphinscheduler-ui/dist/js/22.2b4bb2a.js
dolphinscheduler-ui/dist/js/22.2b4bb2a.js.map
dolphinscheduler-ui/dist/js/23.81467ef.js
dolphinscheduler-ui/dist/js/23.81467ef.js.map
dolphinscheduler-ui/dist/js/24.54a00e4.js
dolphinscheduler-ui/dist/js/24.54a00e4.js.map
dolphinscheduler-ui/dist/js/25.8d7bd36.js
dolphinscheduler-ui/dist/js/25.8d7bd36.js.map
dolphinscheduler-ui/dist/js/26.2ec5e78.js
dolphinscheduler-ui/dist/js/26.2ec5e78.js.map
dolphinscheduler-ui/dist/js/27.3ab48c2.js
dolphinscheduler-ui/dist/js/27.3ab48c2.js.map
dolphinscheduler-ui/dist/js/28.363088a.js
dolphinscheduler-ui/dist/js/28.363088a.js.map
dolphinscheduler-ui/dist/js/29.6c5853a.js
dolphinscheduler-ui/dist/js/29.6c5853a.js.map
dolphinscheduler-ui/dist/js/3.a0edb5b.js
dolphinscheduler-ui/dist/js/3.a0edb5b.js.map
dolphinscheduler-ui/dist/js/30.940fdd3.js
dolphinscheduler-ui/dist/js/30.940fdd3.js.map
dolphinscheduler-ui/dist/js/31.168a460.js
dolphinscheduler-ui/dist/js/31.168a460.js.map
dolphinscheduler-ui/dist/js/32.8df6594.js
dolphinscheduler-ui/dist/js/32.8df6594.js.map
dolphinscheduler-ui/dist/js/33.4480bbe.js
dolphinscheduler-ui/dist/js/33.4480bbe.js.map
dolphinscheduler-ui/dist/js/34.b407fe1.js
dolphinscheduler-ui/dist/js/34.b407fe1.js.map
dolphinscheduler-ui/dist/js/35.f340b0a.js
dolphinscheduler-ui/dist/js/35.f340b0a.js.map
dolphinscheduler-ui/dist/js/36.8880c2d.js
dolphinscheduler-ui/dist/js/36.8880c2d.js.map
dolphinscheduler-ui/dist/js/37.ea2a25d.js
dolphinscheduler-ui/dist/js/37.ea2a25d.js.map
dolphinscheduler-ui/dist/js/38.98a59ee.js
dolphinscheduler-ui/dist/js/38.98a59ee.js.map
dolphinscheduler-ui/dist/js/39.a5e958a.js
dolphinscheduler-ui/dist/js/39.a5e958a.js.map
dolphinscheduler-ui/dist/js/4.4ca44db.js
dolphinscheduler-ui/dist/js/4.4ca44db.js.map
dolphinscheduler-ui/dist/js/40.e187b1e.js
dolphinscheduler-ui/dist/js/40.e187b1e.js.map
dolphinscheduler-ui/dist/js/41.0e89182.js
dolphinscheduler-ui/dist/js/41.0e89182.js.map
dolphinscheduler-ui/dist/js/42.341047c.js
dolphinscheduler-ui/dist/js/42.341047c.js.map
dolphinscheduler-ui/dist/js/43.27b8228.js
dolphinscheduler-ui/dist/js/43.27b8228.js.map
dolphinscheduler-ui/dist/js/44.e8869bc.js
dolphinscheduler-ui/dist/js/44.e8869bc.js.map
dolphinscheduler-ui/dist/js/45.8d54901.js
dolphinscheduler-ui/dist/js/45.8d54901.js.map
dolphinscheduler-ui/dist/js/5.e1ed7f3.js
dolphinscheduler-ui/dist/js/5.e1ed7f3.js.map
dolphinscheduler-ui/dist/js/6.241ba07.js
dolphinscheduler-ui/dist/js/6.241ba07.js.map
dolphinscheduler-ui/dist/js/7.ab2e297.js
dolphinscheduler-ui/dist/js/7.ab2e297.js.map
dolphinscheduler-ui/dist/js/8.83ff814.js
dolphinscheduler-ui/dist/js/8.83ff814.js.map
dolphinscheduler-ui/dist/js/9.39cb29f.js
dolphinscheduler-ui/dist/js/9.39cb29f.js.map
dolphinscheduler-ui/dist/js/common.733e342.js
dolphinscheduler-ui/dist/js/common.733e342.js.map
dolphinscheduler-ui/dist/js/home/index.78a5d12.js
dolphinscheduler-ui/dist/js/home/index.78a5d12.js.map
dolphinscheduler-ui/dist/js/login/index.291b8e3.js
dolphinscheduler-ui/dist/js/login/index.291b8e3.js.map
dolphinscheduler-ui/dist/lib/external/
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue
/dolphinscheduler-dao/src/main/resources/dao/data_source.properties

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java
Unescape View File

@ -347,8 +347,8 @@ public class DataAnalysisService {
}
ITaskQueue tasksQueue = TaskQueueFactory.getTaskQueueInstance();
List<String> tasksQueueList = tasksQueue.getAllTasks(org.apache.dolphinscheduler.common.Constants.SCHEDULER_TASKS_QUEUE);
List<String> tasksKillList = tasksQueue.getAllTasks(org.apache.dolphinscheduler.common.Constants.SCHEDULER_TASKS_KILL);
List<String> tasksQueueList = tasksQueue.getAllTasks(org.apache.dolphinscheduler.common.Constants.DOLPHINSCHEDULER_TASKS_QUEUE);
List<String> tasksKillList = tasksQueue.getAllTasks(org.apache.dolphinscheduler.common.Constants.DOLPHINSCHEDULER_TASKS_KILL);
Map<String,Integer> dataMap = new HashMap<>();
if (loginUser.getUserType() == UserType.ADMIN_USER){

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
Unescape View File

@ -536,7 +536,7 @@ public class ProcessInstanceService extends BaseDAGService {
try {
logger.info("delete task queue node : {}",nodeValueSb.toString());
tasksQueue.removeNode(org.apache.dolphinscheduler.common.Constants.SCHEDULER_TASKS_QUEUE, nodeValueSb.toString());
tasksQueue.removeNode(org.apache.dolphinscheduler.common.Constants.DOLPHINSCHEDULER_TASKS_QUEUE, nodeValueSb.toString());
}catch (Exception e){
logger.error("delete task queue node : {}", nodeValueSb.toString());
}

6
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/HttpClientTest.java
Unescape View File

@ -44,7 +44,7 @@ public class HttpClientTest {
CloseableHttpClient httpclient = HttpClients.createDefault();
// create http post request
HttpPost httpPost = new HttpPost("http://localhost:12345/escheduler/projects/create");
HttpPost httpPost = new HttpPost("http://localhost:12345/dolphinscheduler/projects/create");
httpPost.setHeader("token", "123");
// set parameters
List<NameValuePair> parameters = new ArrayList<NameValuePair>();
@ -85,7 +85,7 @@ public class HttpClientTest {
// parameters.add(new BasicNameValuePair("pageSize", "10"));
// define the parameters of the request
URI uri = new URIBuilder("http://localhost:12345/escheduler/projects/%E5%85%A8%E9%83%A8%E6%B5%81%E7%A8%8B%E6%B5%8B%E8%AF%95/process/list")
URI uri = new URIBuilder("http://localhost:12345/dolphinscheduler/projects/%E5%85%A8%E9%83%A8%E6%B5%81%E7%A8%8B%E6%B5%8B%E8%AF%95/process/list")
.build();
// create http GET request
@ -127,7 +127,7 @@ public class HttpClientTest {
parameters.add(new BasicNameValuePair("projectId", "0"));
// define the parameters of the request
URI uri = new URIBuilder("http://localhost:12345/escheduler/projects/analysis/queue-count")
URI uri = new URIBuilder("http://localhost:12345/dolphinscheduler/projects/analysis/queue-count")
.setParameters(parameters)
.build();

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java
Unescape View File

@ -45,7 +45,7 @@ public class LoginControllerTest extends AbstractControllerTest{
public void login() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("userName","admin");
paramsMap.add("userPassword","escheduler123");
paramsMap.add("userPassword","dolphinscheduler123");
MvcResult mvcResult = mockMvc.perform(post("/login")
.params(paramsMap))

2
dolphinscheduler-api/src/test/resources/dao/data_source.properties
Unescape View File

@ -1,7 +1,7 @@
# base spring data source configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/escheduler?characterEncoding=UTF-8
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/dolphinscheduler?characterEncoding=UTF-8
spring.datasource.username=root
spring.datasource.password=root@123

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
Unescape View File

@ -115,9 +115,9 @@ public final class Constants {
public static final String PROCESS_EXEC_BASEPATH = "process.exec.basepath";
/**
* escheduler.env.path
* dolphinscheduler.env.path
*/
public static final String ESCHEDULER_ENV_PATH = "escheduler.env.path";
public static final String DOLPHINSCHEDULER_ENV_PATH = "dolphinscheduler.env.path";
/**
@ -148,47 +148,47 @@ public final class Constants {
/**
* MasterServer directory registered in zookeeper
*/
public static final String ZOOKEEPER_ESCHEDULER_MASTERS = "zookeeper.escheduler.masters";
public static final String ZOOKEEPER_DOLPHINSCHEDULER_MASTERS = "zookeeper.dolphinscheduler.masters";
/**
* WorkerServer directory registered in zookeeper
*/
public static final String ZOOKEEPER_ESCHEDULER_WORKERS = "zookeeper.escheduler.workers";
public static final String ZOOKEEPER_DOLPHINSCHEDULER_WORKERS = "zookeeper.dolphinscheduler.workers";
/**
* all servers directory registered in zookeeper
*/
public static final String ZOOKEEPER_ESCHEDULER_DEAD_SERVERS = "zookeeper.escheduler.dead.servers";
public static final String ZOOKEEPER_DOLPHINSCHEDULER_DEAD_SERVERS = "zookeeper.dolphinscheduler.dead.servers";
/**
* MasterServer lock directory registered in zookeeper
*/
public static final String ZOOKEEPER_ESCHEDULER_LOCK_MASTERS = "zookeeper.escheduler.lock.masters";
public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_MASTERS = "zookeeper.dolphinscheduler.lock.masters";
/**
* WorkerServer lock directory registered in zookeeper
*/
public static final String ZOOKEEPER_ESCHEDULER_LOCK_WORKERS = "zookeeper.escheduler.lock.workers";
public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_WORKERS = "zookeeper.dolphinscheduler.lock.workers";
/**
* MasterServer failover directory registered in zookeeper
*/
public static final String ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_MASTERS = "zookeeper.escheduler.lock.failover.masters";
public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS = "zookeeper.dolphinscheduler.lock.failover.masters";
/**
* WorkerServer failover directory registered in zookeeper
*/
public static final String ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS = "zookeeper.escheduler.lock.failover.workers";
public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS = "zookeeper.dolphinscheduler.lock.failover.workers";
/**
* MasterServer startup failover runing and fault tolerance process
*/
public static final String ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS = "zookeeper.escheduler.lock.failover.startup.masters";
public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS = "zookeeper.dolphinscheduler.lock.failover.startup.masters";
/**
* need send warn times when master server or worker server failover
*/
public static final int ESCHEDULER_WARN_TIMES_FAILOVER = 3;
public static final int DOLPHINSCHEDULER_WARN_TIMES_FAILOVER = 3;
/**
* comma ,
@ -260,18 +260,18 @@ public final class Constants {
/**
* escheduler tasks queue
* dolphinscheduler tasks queue
*/
public static final String SCHEDULER_TASKS_QUEUE = "tasks_queue";
public static final String DOLPHINSCHEDULER_TASKS_QUEUE = "tasks_queue";
/**
* escheduler need kill tasks queue
* dolphinscheduler need kill tasks queue
*/
public static final String SCHEDULER_TASKS_KILL = "tasks_kill";
public static final String DOLPHINSCHEDULER_TASKS_KILL = "tasks_kill";
public static final String ZOOKEEPER_SCHEDULER_ROOT = "zookeeper.escheduler.root";
public static final String ZOOKEEPER_DOLPHINSCHEDULER_ROOT = "zookeeper.dolphinscheduler.root";
public static final String SCHEDULER_QUEUE_IMPL = "escheduler.queue.impl";
public static final String SCHEDULER_QUEUE_IMPL = "dolphinscheduler.queue.impl";
/**

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java
Unescape View File

@ -45,7 +45,7 @@ public class TaskQueueFactory {
logger.info("task queue impl use zookeeper ");
return TaskQueueZkImpl.getInstance();
}else{
logger.error("property escheduler.queue.impl can't be blank, system will exit ");
logger.error("property dolphinscheduler.queue.impl can't be blank, system will exit ");
System.exit(-1);
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java
Unescape View File

@ -380,8 +380,8 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
*/
private void init(){
try {
String tasksQueuePath = getTasksPath(Constants.SCHEDULER_TASKS_QUEUE);
String tasksCancelPath = getTasksPath(Constants.SCHEDULER_TASKS_KILL);
String tasksQueuePath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_QUEUE);
String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL);
for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){
if(zkClient.checkExists().forPath(taskQueuePath) == null){
@ -404,8 +404,8 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
@Override
public void delete(){
try {
String tasksQueuePath = getTasksPath(Constants.SCHEDULER_TASKS_QUEUE);
String tasksCancelPath = getTasksPath(Constants.SCHEDULER_TASKS_KILL);
String tasksQueuePath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_QUEUE);
String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL);
for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){
if(zkClient.checkExists().forPath(taskQueuePath) != null){
@ -432,7 +432,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
* @return
*/
public String getTasksPath(String key){
return conf.getString(Constants.ZOOKEEPER_SCHEDULER_ROOT) + Constants.SINGLE_SLASH + key;
return conf.getString(Constants.ZOOKEEPER_DOLPHINSCHEDULER_ROOT) + Constants.SINGLE_SLASH + key;
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
Unescape View File

@ -37,7 +37,7 @@ public class CommonUtils {
* @return get the path of system environment variables
*/
public static String getSystemEnvPath() {
String envPath = PropertyUtils.getString(Constants.ESCHEDULER_ENV_PATH);
String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH);
if (StringUtils.isEmpty(envPath)) {
envPath = System.getProperty("user.home") + File.separator + ".bash_profile";
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
Unescape View File

@ -70,7 +70,7 @@ public class HadoopUtils implements Closeable {
}
/**
* init escheduler root path in hdfs
* init dolphinscheduler root path in hdfs
*/
private void initHdfsPath(){
String hdfsPath = PropertyUtils.getString(Constants.DATA_STORE_2_HDFS_BASEPATH);

14
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java
Unescape View File

@ -414,20 +414,20 @@ public abstract class AbstractZKClient {
* get worker node parent path
* @return
*/
protected String getWorkerZNodeParentPath(){return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_WORKERS);};
protected String getWorkerZNodeParentPath(){return conf.getString(Constants.ZOOKEEPER_DOLPHINSCHEDULER_WORKERS);};
/**
* get master node parent path
* @return
*/
protected String getMasterZNodeParentPath(){return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_MASTERS);}
protected String getMasterZNodeParentPath(){return conf.getString(Constants.ZOOKEEPER_DOLPHINSCHEDULER_MASTERS);}
/**
* get master lock path
* @return
*/
public String getMasterLockPath(){
return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_MASTERS);
return conf.getString(Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_MASTERS);
}
/**
@ -455,7 +455,7 @@ public abstract class AbstractZKClient {
* @return
*/
protected String getDeadZNodeParentPath(){
return conf.getString(ZOOKEEPER_ESCHEDULER_DEAD_SERVERS);
return conf.getString(ZOOKEEPER_DOLPHINSCHEDULER_DEAD_SERVERS);
}
/**
@ -463,7 +463,7 @@ public abstract class AbstractZKClient {
* @return
*/
public String getMasterStartUpLockPath(){
return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS);
return conf.getString(Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS);
}
/**
@ -471,7 +471,7 @@ public abstract class AbstractZKClient {
* @return
*/
public String getMasterFailoverLockPath(){
return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_MASTERS);
return conf.getString(Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS);
}
/**
@ -479,7 +479,7 @@ public abstract class AbstractZKClient {
* @return
*/
public String getWorkerFailoverLockPath(){
return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS);
return conf.getString(Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS);
}
/**

14
dolphinscheduler-common/src/main/resources/common/common.properties
Unescape View File

@ -1,20 +1,20 @@
#task queue implementation, default "zookeeper"
escheduler.queue.impl=zookeeper
dolphinscheduler.queue.impl=zookeeper
# user data directory path, self configuration, please make sure the directory exists and have read write permissions
data.basedir.path=/tmp/escheduler
data.basedir.path=/tmp/dolphinscheduler
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
data.download.basedir.path=/tmp/escheduler/download
data.download.basedir.path=/tmp/dolphinscheduler/download
# process execute directory. self configuration, please make sure the directory exists and have read write permissions
process.exec.basepath=/tmp/escheduler/exec
process.exec.basepath=/tmp/dolphinscheduler/exec
# Users who have permission to create directories under the HDFS root path
hdfs.root.user=hdfs
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/escheduler" is recommended
data.store2hdfs.basepath=/escheduler
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
data.store2hdfs.basepath=/dolphinscheduler
# resource upload startup type : HDFS,S3,NONE
res.upload.startup.type=NONE
@ -32,7 +32,7 @@ login.user.keytab.username=hdfs-mycluster@ESZ.COM
login.user.keytab.path=/opt/hdfs.headless.keytab
# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions
escheduler.env.path=/opt/.dolphinscheduler_env.sh
dolphinscheduler.env.path=/opt/.dolphinscheduler_env.sh
#resource.view.suffixs
resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml

14
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueImplTest.java
Unescape View File

@ -65,12 +65,12 @@ public class TaskQueueImplTest extends StandaloneZKServerForTest {
//add
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_0_1_1_-1");
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"0_1_1_1_-1");
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"0_0_0_1_" + IpUtils.ipToLong(OSUtils.getHost()));
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_2_1_1_" + IpUtils.ipToLong(OSUtils.getHost()) + 10);
tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"1_0_1_1_-1");
tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"0_1_1_1_-1");
tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"0_0_0_1_" + IpUtils.ipToLong(OSUtils.getHost()));
tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"1_2_1_1_" + IpUtils.ipToLong(OSUtils.getHost()) + 10);
List<String> tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1);
List<String> tasks = tasksQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, 1);
if(tasks.size() <= 0){
return;
@ -100,11 +100,11 @@ public class TaskQueueImplTest extends StandaloneZKServerForTest {
//${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId}
//format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId}
String formatTask = String.format("%s_%d_%s_%d", i, i + 1, j, j == 0 ? 0 : j + new Random().nextInt(100));
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE, formatTask);
tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, formatTask);
}
}
String node1 = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1).get(0);
String node1 = tasksQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, 1).get(0);
assertEquals(node1,"0");
}

5
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java
Unescape View File

@ -33,7 +33,6 @@ import com.alibaba.fastjson.JSONObject;
import com.cronutils.model.Cron;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.quartz.CronExpression;
@ -987,7 +986,7 @@ public class ProcessDao extends AbstractBaseDao {
return true;
}
logger.info("task ready to queue: {}" , task);
taskQueue.add(SCHEDULER_TASKS_QUEUE, taskZkInfo(task));
taskQueue.add(DOLPHINSCHEDULER_TASKS_QUEUE, taskZkInfo(task));
logger.info(String.format("master insert into queue success, task : %s", task.getName()) );
return true;
}catch (Exception e){
@ -1106,7 +1105,7 @@ public class ProcessDao extends AbstractBaseDao {
String taskZkInfo = taskZkInfo(task);
return taskQueue.checkTaskExists(SCHEDULER_TASKS_QUEUE, taskZkInfo);
return taskQueue.checkTaskExists(DOLPHINSCHEDULER_TASKS_QUEUE, taskZkInfo);
}
/**

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
Unescape View File

@ -163,7 +163,7 @@ public class MasterServer extends AbstractServer {
public void run() {
logger.info("master server stopped");
if (zkMasterClient.getActiveMasterNum() <= 1) {
for (int i = 0; i < Constants.ESCHEDULER_WARN_TIMES_FAILOVER;i++) {
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) {
zkMasterClient.getAlertDao().sendServerStopedAlert(
1, OSUtils.getHost(), "Master-Server");
}

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java
Unescape View File

@ -31,7 +31,7 @@ import org.slf4j.LoggerFactory;
import java.util.Date;
import static org.apache.dolphinscheduler.common.Constants.SCHEDULER_TASKS_KILL;
import static org.apache.dolphinscheduler.common.Constants.DOLPHINSCHEDULER_TASKS_KILL;
/**
* master task exec thread
@ -134,7 +134,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread {
alreadyKilled = true;
String queueValue = String.format("%s-%d",
taskInstance.getHost(), taskInstance.getId());
taskQueue.sadd(SCHEDULER_TASKS_KILL, queueValue);
taskQueue.sadd(DOLPHINSCHEDULER_TASKS_KILL, queueValue);
logger.info("master add kill task :{} id:{} to kill queue",
taskInstance.getName(), taskInstance.getId() );

8
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
Unescape View File

@ -176,7 +176,7 @@ public class WorkerServer extends AbstractServer {
logger.warn("worker server stopped");
// worker server exit alert
if (zkWorkerClient.getActiveMasterNum() <= 1) {
for (int i = 0; i < Constants.ESCHEDULER_WARN_TIMES_FAILOVER;i++) {
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) {
alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server");
}
}
@ -291,7 +291,7 @@ public class WorkerServer extends AbstractServer {
Runnable killProcessThread = new Runnable() {
@Override
public void run() {
Set<String> taskInfoSet = taskQueue.smembers(Constants.SCHEDULER_TASKS_KILL);
Set<String> taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL);
while (Stopper.isRunning()){
try {
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
@ -316,13 +316,13 @@ public class WorkerServer extends AbstractServer {
}else{
ProcessUtils.kill(taskInstance);
}
taskQueue.srem(Constants.SCHEDULER_TASKS_KILL,taskInfo);
taskQueue.srem(Constants.DOLPHINSCHEDULER_TASKS_KILL,taskInfo);
}
}
}
}
taskInfoSet = taskQueue.smembers(Constants.SCHEDULER_TASKS_KILL);
taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL);
}
}
};

12
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java
Unescape View File

@ -153,17 +153,17 @@ public class FetchTaskThread implements Runnable{
}
//whether have tasks, if no tasks , no need lock //get all tasks
List<String> tasksQueueList = taskQueue.getAllTasks(Constants.SCHEDULER_TASKS_QUEUE);
List<String> tasksQueueList = taskQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_QUEUE);
if (CollectionUtils.isEmpty(tasksQueueList)){
continue;
}
// creating distributed locks, lock path /escheduler/lock/worker
// creating distributed locks, lock path /dolphinscheduler/lock/worker
mutex = zkWorkerClient.acquireZkLock(zkWorkerClient.getZkClient(),
zkWorkerClient.getWorkerLockPath());
// task instance id str
List<String> taskQueueStrArr = taskQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, taskNum);
List<String> taskQueueStrArr = taskQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, taskNum);
for(String taskQueueStr : taskQueueStrArr){
if (StringUtils.isEmpty(taskQueueStr)) {
@ -226,7 +226,7 @@ public class FetchTaskThread implements Runnable{
workerExecService.submit(new TaskScheduleThread(taskInstance, processDao));
// remove node from zk
taskQueue.removeNode(Constants.SCHEDULER_TASKS_QUEUE, taskQueueStr);
taskQueue.removeNode(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, taskQueueStr);
}
}catch (Exception e){
@ -245,7 +245,7 @@ public class FetchTaskThread implements Runnable{
private boolean verifyTaskInstanceIsNull(String taskQueueStr) {
if (taskInstance == null ) {
logger.error("task instance is null. task id : {} ", taskInstId);
taskQueue.removeNode(Constants.SCHEDULER_TASKS_QUEUE, taskQueueStr);
taskQueue.removeNode(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, taskQueueStr);
return true;
}
return false;
@ -263,7 +263,7 @@ public class FetchTaskThread implements Runnable{
taskInstance.getProcessDefine().getId(),
taskInstance.getProcessInstance().getId(),
taskInstance.getId());
taskQueue.removeNode(Constants.SCHEDULER_TASKS_QUEUE, taskQueueStr);
taskQueue.removeNode(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, taskQueueStr);
return true;
}
return false;

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java
Unescape View File

@ -265,7 +265,7 @@ public class ZKMasterClient extends AbstractZKClient {
private void alertServerDown(String serverHost, ZKNodeType zkNodeType) {
String serverType = zkNodeType.toString();
for (int i = 0; i < Constants.ESCHEDULER_WARN_TIMES_FAILOVER; i++) {
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) {
alertDao.sendServerStopedAlert(1, serverHost, serverType);
}
}

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java
Unescape View File

@ -159,7 +159,7 @@ public class ZKWorkerClient extends AbstractZKClient {
* @return
*/
public String getWorkerLockPath(){
return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_WORKERS);
return conf.getString(Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_WORKERS);
}

Write Preview
Loading…
Cancel
Save