Browse Source

Merge pull request #18 from apache/dev

update
pull/2/head
samz406 5 years ago committed by GitHub
parent
commit
257a3ee3e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 51
      .github/workflows/SonarCloud.yml
  2. 3
      README.md
  3. 2
      README_zh_CN.md
  4. 11
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
  5. 53
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
  6. 17
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java
  7. 18
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java
  8. 49
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java
  9. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
  10. 40
      dolphinscheduler-api/src/main/resources/application-combined.properties
  11. 181
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java
  12. 219
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java
  13. 101
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java
  14. 356
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java
  15. 208
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java
  16. 121
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java
  17. 225
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
  18. 434
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
  19. 173
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java
  20. 21
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  21. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
  22. 98
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java
  23. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java
  24. 51
      dolphinscheduler-common/src/main/resources/common.properties
  25. 35
      dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties
  26. 18
      dolphinscheduler-common/src/main/resources/quartz.properties
  27. 43
      dolphinscheduler-common/src/main/resources/zookeeper.properties
  28. 71
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java
  29. 74
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java
  30. 40
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java
  31. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java
  32. 0
      dolphinscheduler-dao/src/main/resources/application.properties
  33. 156
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java
  34. 264
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java
  35. 6
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
  36. 20
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java
  37. 20
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
  38. 6
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java
  39. 18
      dolphinscheduler-server/src/main/resources/application-master.properties
  40. 18
      dolphinscheduler-server/src/main/resources/application-worker.properties
  41. 8
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue
  42. 124
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue
  43. 64
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue
  44. 8
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue
  45. 16
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
  46. 2
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js
  47. 2
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue
  48. 109
      install.sh
  49. 11
      pom.xml
  50. 8
      script/dolphinscheduler-daemon.sh

51
.github/workflows/SonarCloud.yml

@ -0,0 +1,51 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: SonarCloud
on: [push, pull_request]
jobs:
sonarCloudTrigger:
name: SonarCloud Trigger
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-java@v1
with:
java-version: 8
- uses: actions/cache@v1
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube
restore-keys: |
${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube
${{ runner.os }}-maven-
- name: Maven clean
run: mvn clean
- name: Run SonarCloud analyse
run: >
mvn clean --batch-mode
org.jacoco:jacoco-maven-plugin:prepare-agent
verify
org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
-Dmaven.test.skip=true
-Dsonar.host.url=https://sonarcloud.io
-Dsonar.organization=apache
-Dsonar.projectKey=apache-dolphinscheduler
-Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

3
README.md

@ -4,7 +4,8 @@ Dolphin Scheduler Official Website
[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler)
[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev)
![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
> Dolphin Scheduler for Big Data > Dolphin Scheduler for Big Data

2
README_zh_CN.md

@ -4,7 +4,7 @@ Dolphin Scheduler Official Website
[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler)
[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev)
![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
> Dolphin Scheduler for Big Data > Dolphin Scheduler for Big Data

11
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java

@ -39,19 +39,15 @@ public class AlertServer {
private AlertSender alertSender; private AlertSender alertSender;
private static volatile AlertServer instance; private static AlertServer instance;
public AlertServer() { public AlertServer() {
} }
public static AlertServer getInstance(){ public synchronized static AlertServer getInstance(){
if (null == instance) { if (null == instance) {
synchronized (AlertServer.class) { instance = new AlertServer();
if(null == instance) {
instance = new AlertServer();
}
}
} }
return instance; return instance;
} }
@ -63,6 +59,7 @@ public class AlertServer {
Thread.sleep(Constants.ALERT_SCAN_INTERVEL); Thread.sleep(Constants.ALERT_SCAN_INTERVEL);
} catch (InterruptedException e) { } catch (InterruptedException e) {
logger.error(e.getMessage(),e); logger.error(e.getMessage(),e);
Thread.currentThread().interrupt();
} }
List<Alert> alerts = alertDao.listWaitExecutionAlert(); List<Alert> alerts = alertDao.listWaitExecutionAlert();
alertSender = new AlertSender(alerts, alertDao); alertSender = new AlertSender(alerts, alertDao);

53
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java

@ -86,20 +86,24 @@ public class EnterpriseWeChatUtils {
String resp; String resp;
CloseableHttpClient httpClient = HttpClients.createDefault(); CloseableHttpClient httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace);
CloseableHttpResponse response = httpClient.execute(httpGet);
try { try {
HttpEntity entity = response.getEntity(); HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace);
resp = EntityUtils.toString(entity, Constants.UTF_8); CloseableHttpResponse response = httpClient.execute(httpGet);
EntityUtils.consume(entity); try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, Constants.UTF_8);
EntityUtils.consume(entity);
} finally {
response.close();
}
Map<String, Object> map = JSON.parseObject(resp,
new TypeToken<Map<String, Object>>() {
}.getType());
return map.get("access_token").toString();
} finally { } finally {
response.close(); httpClient.close();
} }
Map<String, Object> map = JSON.parseObject(resp,
new TypeToken<Map<String, Object>>() {
}.getType());
return map.get("access_token").toString();
} }
/** /**
@ -167,20 +171,25 @@ public class EnterpriseWeChatUtils {
public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException {
String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token); String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token);
CloseableHttpClient httpclient = HttpClients.createDefault(); CloseableHttpClient httpClient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace);
httpPost.setEntity(new StringEntity(data, charset));
CloseableHttpResponse response = httpclient.execute(httpPost);
String resp;
try { try {
HttpEntity entity = response.getEntity(); HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace);
resp = EntityUtils.toString(entity, charset); httpPost.setEntity(new StringEntity(data, charset));
EntityUtils.consume(entity); CloseableHttpResponse response = httpClient.execute(httpPost);
String resp;
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, charset);
EntityUtils.consume(entity);
} finally {
response.close();
}
logger.info("Enterprise WeChat send [{}], param:{}, resp:{}",
enterpriseWeChatPushUrl, data, resp);
return resp;
} finally { } finally {
response.close(); httpClient.close();
} }
logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", enterpriseWeChatPushUrl, data, resp);
return resp;
} }
/** /**

17
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java

@ -41,7 +41,7 @@ import static org.apache.dolphinscheduler.api.utils.CheckUtils.checkDesc;
/** /**
* project service * project service
*HttpTask./ *HttpTask./
**/ **/
@Service @Service
public class ProjectService extends BaseService{ public class ProjectService extends BaseService{
@ -121,7 +121,7 @@ public class ProjectService extends BaseService{
* @param loginUser login user * @param loginUser login user
* @param project project * @param project project
* @param projectName project name * @param projectName project name
* @return true if the login user havve permission to see the project * @return true if the login user have permission to see the project
*/ */
public Map<String, Object> checkProjectAndAuth(User loginUser, Project project, String projectName) { public Map<String, Object> checkProjectAndAuth(User loginUser, Project project, String projectName) {
@ -143,7 +143,7 @@ public class ProjectService extends BaseService{
public boolean hasProjectAndPerm(User loginUser, Project project, Map<String, Object> result) { public boolean hasProjectAndPerm(User loginUser, Project project, Map<String, Object> result) {
boolean checkResult = false; boolean checkResult = false;
if (project == null) { if (project == null) {
putMsg(result, Status.PROJECT_NOT_FOUNT, project.getName()); putMsg(result, Status.PROJECT_NOT_FOUNT, "");
} else if (!checkReadPermission(loginUser, project)) { } else if (!checkReadPermission(loginUser, project)) {
putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), project.getName()); putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), project.getName());
} else { } else {
@ -199,13 +199,14 @@ public class ProjectService extends BaseService{
if (checkResult != null) { if (checkResult != null) {
return checkResult; return checkResult;
} }
List<ProcessDefinition> processDefinitionList = processDefinitionMapper.queryAllDefinitionList(projectId);
if (!hasPerm(loginUser, project.getUserId())) { if (!hasPerm(loginUser, project.getUserId())) {
putMsg(result, Status.USER_NO_OPERATION_PERM); putMsg(result, Status.USER_NO_OPERATION_PERM);
return result; return result;
} }
List<ProcessDefinition> processDefinitionList = processDefinitionMapper.queryAllDefinitionList(projectId);
if(processDefinitionList.size() > 0){ if(processDefinitionList.size() > 0){
putMsg(result, Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL); putMsg(result, Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL);
return result; return result;
@ -227,7 +228,8 @@ public class ProjectService extends BaseService{
* @return check result * @return check result
*/ */
private Map<String, Object> getCheckResult(User loginUser, Project project) { private Map<String, Object> getCheckResult(User loginUser, Project project) {
Map<String, Object> checkResult = checkProjectAndAuth(loginUser, project, project.getName()); String projectName = project == null ? null:project.getName();
Map<String, Object> checkResult = checkProjectAndAuth(loginUser, project, projectName);
Status status = (Status) checkResult.get(Constants.STATUS); Status status = (Status) checkResult.get(Constants.STATUS);
if (status != Status.SUCCESS) { if (status != Status.SUCCESS) {
return checkResult; return checkResult;
@ -247,6 +249,11 @@ public class ProjectService extends BaseService{
public Map<String, Object> update(User loginUser, Integer projectId, String projectName, String desc) { public Map<String, Object> update(User loginUser, Integer projectId, String projectName, String desc) {
Map<String, Object> result = new HashMap<>(5); Map<String, Object> result = new HashMap<>(5);
Map<String, Object> descCheck = checkDesc(desc);
if (descCheck.get(Constants.STATUS) != Status.SUCCESS) {
return descCheck;
}
Project project = projectMapper.selectById(projectId); Project project = projectMapper.selectById(projectId);
boolean hasProjectAndPerm = hasProjectAndPerm(loginUser, project, result); boolean hasProjectAndPerm = hasProjectAndPerm(loginUser, project, result);
if (!hasProjectAndPerm) { if (!hasProjectAndPerm) {

18
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java

@ -115,12 +115,12 @@ public class QueueService extends BaseService {
} }
if (StringUtils.isEmpty(queue)) { if (StringUtils.isEmpty(queue)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue); putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue");
return result; return result;
} }
if (StringUtils.isEmpty(queueName)) { if (StringUtils.isEmpty(queueName)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName); putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName");
return result; return result;
} }
@ -163,6 +163,16 @@ public class QueueService extends BaseService {
return result; return result;
} }
if (StringUtils.isEmpty(queue)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue");
return result;
}
if (StringUtils.isEmpty(queueName)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName");
return result;
}
Queue queueObj = queueMapper.selectById(id); Queue queueObj = queueMapper.selectById(id);
if (queueObj == null) { if (queueObj == null) {
putMsg(result, Status.QUEUE_NOT_EXIST, id); putMsg(result, Status.QUEUE_NOT_EXIST, id);
@ -222,12 +232,12 @@ public class QueueService extends BaseService {
Result result = new Result(); Result result = new Result();
if (StringUtils.isEmpty(queue)) { if (StringUtils.isEmpty(queue)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue); putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue");
return result; return result;
} }
if (StringUtils.isEmpty(queueName)) { if (StringUtils.isEmpty(queueName)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName); putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName");
return result; return result;
} }

49
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java

@ -221,9 +221,13 @@ public class UsersService extends BaseService {
return result; return result;
} }
Date now = new Date();
if (StringUtils.isNotEmpty(userName)) { if (StringUtils.isNotEmpty(userName)) {
if (!CheckUtils.checkUserName(userName)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userName);
return result;
}
User tempUser = userMapper.queryByUserNameAccurately(userName); User tempUser = userMapper.queryByUserNameAccurately(userName);
if (tempUser != null && tempUser.getId() != userId) { if (tempUser != null && tempUser.getId() != userId) {
putMsg(result, Status.USER_NAME_EXIST); putMsg(result, Status.USER_NAME_EXIST);
@ -233,14 +237,30 @@ public class UsersService extends BaseService {
} }
if (StringUtils.isNotEmpty(userPassword)) { if (StringUtils.isNotEmpty(userPassword)) {
if (!CheckUtils.checkPassword(userPassword)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userPassword);
return result;
}
user.setUserPassword(EncryptionUtils.getMd5(userPassword)); user.setUserPassword(EncryptionUtils.getMd5(userPassword));
} }
if (StringUtils.isNotEmpty(email)) { if (StringUtils.isNotEmpty(email)) {
if (!CheckUtils.checkEmail(email)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,email);
return result;
}
user.setEmail(email); user.setEmail(email);
} }
if (StringUtils.isNotEmpty(phone)) {
if (!CheckUtils.checkPhone(phone)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,phone);
return result;
}
user.setPhone(phone);
}
user.setQueue(queue); user.setQueue(queue);
user.setPhone(phone); Date now = new Date();
user.setUpdateTime(now); user.setUpdateTime(now);
//if user switches the tenant, the user's resources need to be copied to the new tenant //if user switches the tenant, the user's resources need to be copied to the new tenant
@ -318,10 +338,15 @@ public class UsersService extends BaseService {
Map<String, Object> result = new HashMap<>(5); Map<String, Object> result = new HashMap<>(5);
//only admin can operate //only admin can operate
if (!isAdmin(loginUser)) { if (!isAdmin(loginUser)) {
putMsg(result, Status.USER_NO_OPERATION_PERM, id);
return result;
}
//check exist
User tempUser = userMapper.selectById(id);
if (tempUser == null) {
putMsg(result, Status.USER_NOT_EXIST, id); putMsg(result, Status.USER_NOT_EXIST, id);
return result; return result;
} }
// delete user // delete user
User user = userMapper.queryTenantCodeByUserId(id); User user = userMapper.queryTenantCodeByUserId(id);
@ -357,6 +382,12 @@ public class UsersService extends BaseService {
return result; return result;
} }
//check exist
User tempUser = userMapper.selectById(userId);
if (tempUser == null) {
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
//if the selected projectIds are empty, delete all items associated with the user //if the selected projectIds are empty, delete all items associated with the user
projectUserMapper.deleteProjectRelation(0, userId); projectUserMapper.deleteProjectRelation(0, userId);
@ -443,6 +474,11 @@ public class UsersService extends BaseService {
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result; return result;
} }
User user = userMapper.selectById(userId);
if(user == null){
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
udfUserMapper.deleteByUserId(userId); udfUserMapper.deleteByUserId(userId);
@ -485,6 +521,11 @@ public class UsersService extends BaseService {
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result; return result;
} }
User user = userMapper.selectById(userId);
if(user == null){
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
datasourceUserMapper.deleteByUserId(userId); datasourceUserMapper.deleteByUserId(userId);

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java

@ -76,6 +76,11 @@ public class WorkerGroupService extends BaseService {
WorkerGroup workerGroup = null; WorkerGroup workerGroup = null;
if(id != 0){ if(id != 0){
workerGroup = workerGroupMapper.selectById(id); workerGroup = workerGroupMapper.selectById(id);
//check exist
if (workerGroup == null){
workerGroup = new WorkerGroup();
workerGroup.setCreateTime(now);
}
}else{ }else{
workerGroup = new WorkerGroup(); workerGroup = new WorkerGroup();
workerGroup.setCreateTime(now); workerGroup.setCreateTime(now);

40
dolphinscheduler-api/src/main/resources/application-combined.properties

@ -1,40 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
logging.config=classpath:combined_logback.xml
# server port
server.port=12345
# session config
server.servlet.session.timeout=7200
server.servlet.context-path=/dolphinscheduler/
# file size limit for upload
spring.servlet.multipart.max-file-size=1024MB
spring.servlet.multipart.max-request-size=1024MB
#post content
server.jetty.max-http-post-size=5000000
spring.messages.encoding=UTF-8
#i18n classpath folder , file prefix messages, if have many files, use "," seperator
spring.messages.basename=i18n/messages
server.is-combined-server=true

181
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java

@ -0,0 +1,181 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.AccessToken;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class AccessTokenServiceTest {
private static final Logger logger = LoggerFactory.getLogger(AccessTokenServiceTest.class);
@InjectMocks
private AccessTokenService accessTokenService ;
@Mock
private AccessTokenMapper accessTokenMapper;
@Before
public void setUp() {
}
@After
public void after(){
}
@Test
public void testQueryAccessTokenList(){
IPage<AccessToken> tokenPage = new Page<>();
tokenPage.setRecords(getList());
tokenPage.setTotal(1L);
when(accessTokenMapper.selectAccessTokenPage(any(Page.class),eq("zhangsan"),eq(0))).thenReturn(tokenPage);
User user =new User();
Map<String, Object> result = accessTokenService.queryAccessTokenList(user,"zhangsan",1,10);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
PageInfo<AccessToken> pageInfo = (PageInfo<AccessToken>) result.get(Constants.DATA_LIST);
Assert.assertTrue(pageInfo.getTotalCount()>0);
}
@Test
public void testCreateToken(){
when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2);
Map<String, Object> result = accessTokenService.createToken(1,getDate(),"AccessTokenServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testGenerateToken(){
Map<String, Object> result = accessTokenService.generateToken(Integer.MAX_VALUE,getDate());
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
String token = (String) result.get(Constants.DATA_LIST);
Assert.assertNotNull(token);
}
@Test
public void testDelAccessTokenById(){
when(accessTokenMapper.selectById(1)).thenReturn(getEntity());
User userLogin = new User();
// not exist
Map<String, Object> result = accessTokenService.delAccessTokenById(userLogin,0);
logger.info(result.toString());
Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS));
// no operate
result = accessTokenService.delAccessTokenById(userLogin,1);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
//success
userLogin.setId(1);
userLogin.setUserType(UserType.ADMIN_USER);
result = accessTokenService.delAccessTokenById(userLogin,1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testUpdateToken(){
when(accessTokenMapper.selectById(1)).thenReturn(getEntity());
Map<String, Object> result = accessTokenService.updateToken(1,Integer.MAX_VALUE,getDate(),"token");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
// not exist
result = accessTokenService.updateToken(2,Integer.MAX_VALUE,getDate(),"token");
logger.info(result.toString());
Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS));
}
/**
* create entity
* @return
*/
private AccessToken getEntity(){
AccessToken accessToken = new AccessToken();
accessToken.setId(1);
accessToken.setUserId(1);
accessToken.setToken("AccessTokenServiceTest");
Date date = DateUtils.addDays(new Date(),30);
accessToken.setExpireTime(date);
return accessToken;
}
/**
* entity list
* @return
*/
private List<AccessToken> getList(){
List<AccessToken> list = new ArrayList<>();
list.add(getEntity());
return list;
}
/**
* get dateStr
* @return
*/
private String getDate(){
Date date = DateUtils.addDays(new Date(),30);
return org.apache.dolphinscheduler.common.utils.DateUtils.dateToString(date);
}
}

219
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java

@ -0,0 +1,219 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.AlertType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
@RunWith(MockitoJUnitRunner.class)
public class AlertGroupServiceTest {
private static final Logger logger = LoggerFactory.getLogger(AlertGroupServiceTest.class);
@InjectMocks
private AlertGroupService alertGroupService;
@Mock
private AlertGroupMapper alertGroupMapper;
@Mock
private UserAlertGroupMapper userAlertGroupMapper;
private String groupName = "AlertGroupServiceTest";
@Before
public void setUp() {
}
@After
public void after(){
}
@Test
public void testQueryAlertgroup(){
Mockito.when(alertGroupMapper.queryAllGroupList()).thenReturn(getList());
HashMap<String, Object> result= alertGroupService.queryAlertgroup();
logger.info(result.toString());
List<AlertGroup> alertGroups = (List<AlertGroup>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(alertGroups));
}
@Test
public void testListPaging(){
IPage<AlertGroup> page = new Page<>(1,10);
page.setTotal(1L);
page.setRecords(getList());
Mockito.when(alertGroupMapper.queryAlertGroupPage(any(Page.class),eq(groupName))).thenReturn(page);
User user = new User();
// no operate
Map<String, Object> result = alertGroupService.listPaging(user,groupName,1,10);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
//success
user.setUserType(UserType.ADMIN_USER);
result = alertGroupService.listPaging(user,groupName,1,10);
logger.info(result.toString());
PageInfo<AlertGroup> pageInfo = (PageInfo<AlertGroup>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
}
@Test
public void testCreateAlertgroup(){
Mockito.when(alertGroupMapper.insert(any(AlertGroup.class))).thenReturn(2);
User user = new User();
//no operate
Map<String, Object> result = alertGroupService.createAlertgroup(user,groupName, AlertType.EMAIL,groupName);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
user.setUserType(UserType.ADMIN_USER);
//success
result = alertGroupService.createAlertgroup(user,groupName, AlertType.EMAIL,groupName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testUpdateAlertgroup(){
User user = new User();
// no operate
Map<String, Object> result = alertGroupService.updateAlertgroup(user,1,groupName, AlertType.SMS,groupName);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
user.setUserType(UserType.ADMIN_USER);
// not exist
result = alertGroupService.updateAlertgroup(user,1,groupName, AlertType.SMS,groupName);
logger.info(result.toString());
Assert.assertEquals(Status.ALERT_GROUP_NOT_EXIST,result.get(Constants.STATUS));
//success
Mockito.when(alertGroupMapper.selectById(2)).thenReturn(getEntity());
result = alertGroupService.updateAlertgroup(user,2,groupName, AlertType.SMS,groupName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testDelAlertgroupById(){
User user = new User();
// no operate
Map<String, Object> result = alertGroupService.delAlertgroupById(user,1);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
user.setUserType(UserType.ADMIN_USER);
// not exist
result = alertGroupService.delAlertgroupById(user,2);
logger.info(result.toString());
Assert.assertEquals(Status.ALERT_GROUP_NOT_EXIST,result.get(Constants.STATUS));
//success
Mockito.when(alertGroupMapper.selectById(2)).thenReturn(getEntity());
result = alertGroupService.delAlertgroupById(user,2);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testGrantUser(){
Map<String, Object> result = alertGroupService.grantUser(getLoginUser(),1,"123,321");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testVerifyGroupName(){
//group name not exist
Result result = alertGroupService.verifyGroupName(getLoginUser(), groupName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg());
Mockito.when(alertGroupMapper.queryByGroupName(groupName)).thenReturn(getList());
//group name exist
result = alertGroupService.verifyGroupName(getLoginUser(), groupName);
logger.info(result.toString());
Assert.assertEquals(Status.ALERT_GROUP_EXIST.getMsg(),result.getMsg());
}
/**
* create admin user
* @return
*/
private User getLoginUser(){
User loginUser = new User();
loginUser.setUserType(UserType.ADMIN_USER);
loginUser.setId(99999999);
return loginUser;
}
/**
* get list
* @return
*/
private List<AlertGroup> getList(){
List<AlertGroup> alertGroups = new ArrayList<>();
alertGroups.add(getEntity());
return alertGroups;
}
/**
* get entity
* @return
*/
private AlertGroup getEntity(){
AlertGroup alertGroup = new AlertGroup();
alertGroup.setId(1);
alertGroup.setGroupName(groupName);
alertGroup.setGroupType(AlertType.EMAIL);
return alertGroup;
}
}

101
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java

@ -0,0 +1,101 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.MonitorDBDao;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@RunWith(MockitoJUnitRunner.class)
public class MonitorServiceTest {
private static final Logger logger = LoggerFactory.getLogger(MonitorServiceTest.class);
@InjectMocks
private MonitorService monitorService;
@Mock
private MonitorDBDao monitorDBDao;
@Test
public void testQueryDatabaseState(){
Mockito.when(monitorDBDao.queryDatabaseState()).thenReturn(getList());
Map<String,Object> result = monitorService.queryDatabaseState(null);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
List<MonitorRecord> monitorRecordList = (List<MonitorRecord>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(monitorRecordList));
}
@Test
public void testQueryMaster(){
//TODO need zk
// Map<String,Object> result = monitorService.queryMaster(null);
// logger.info(result.toString());
// Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testQueryZookeeperState(){
//TODO need zk
// Map<String,Object> result = monitorService.queryZookeeperState(null);
// logger.info(result.toString());
// Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testGetServerListFromZK(){
//TODO need zk
// List<Server> serverList = monitorService.getServerListFromZK(true);
// logger.info(serverList.toString());
}
private List<MonitorRecord> getList(){
List<MonitorRecord> monitorRecordList = new ArrayList<>();
monitorRecordList.add(getEntity());
return monitorRecordList;
}
private MonitorRecord getEntity(){
MonitorRecord monitorRecord = new MonitorRecord();
monitorRecord.setDbType(DbType.MYSQL);
return monitorRecord;
}
private List<Server> getServerList(){
List<Server> servers = new ArrayList<>();
servers.add(new Server());
return servers;
}
}

356
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java

@ -0,0 +1,356 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@RunWith(MockitoJUnitRunner.class)
public class ProjectServiceTest {
private static final Logger logger = LoggerFactory.getLogger(ProjectServiceTest.class);
@InjectMocks
private ProjectService projectService;
@Mock
private ProjectMapper projectMapper;
@Mock
private UserMapper userMapper;
@Mock
private ProjectUserMapper projectUserMapper;
@Mock
private ProcessDefinitionMapper processDefinitionMapper;
private String projectName = "ProjectServiceTest";
private String userName = "ProjectServiceTest";
@Before
public void setUp() {
}
@After
public void after(){
}
@Test
public void testCreateProject(){
User loginUser = getLoginUser();
loginUser.setId(1);
Map<String, Object> result = projectService.createProject(loginUser, projectName, getDesc());
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS));
//project name exist
Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject());
result = projectService.createProject(loginUser, projectName, projectName);
logger.info(result.toString());
Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS,result.get(Constants.STATUS));
//success
Mockito.when(projectMapper.insert(Mockito.any(Project.class))).thenReturn(1);
result = projectService.createProject(loginUser, "test", "test");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testQueryById(){
//not exist
Map<String, Object> result = projectService.queryById(Integer.MAX_VALUE);
Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS));
logger.info(result.toString());
//success
Mockito.when(projectMapper.selectById(1)).thenReturn(getProject());
result = projectService.queryById(1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testCheckProjectAndAuth(){
Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser());
User loginUser = getLoginUser();
Map<String, Object> result = projectService.checkProjectAndAuth(loginUser,null,projectName);
logger.info(result.toString());
Status status = (Status)result.get(Constants.STATUS);
Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS));
Project project = getProject();
//USER_NO_OPERATION_PROJECT_PERM
project.setUserId(2);
result = projectService.checkProjectAndAuth(loginUser,project,projectName);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM,result.get(Constants.STATUS));
//success
project.setUserId(1);
result = projectService.checkProjectAndAuth(loginUser,project,projectName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testHasProjectAndPerm(){
// Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser());
User loginUser = getLoginUser();
Project project = getProject();
Map<String, Object> result = new HashMap<>();
// not exist user
User tempUser = new User();
tempUser.setId(Integer.MAX_VALUE);
boolean checkResult = projectService.hasProjectAndPerm(tempUser,project,result);
logger.info(result.toString());
Assert.assertFalse(checkResult);
//success
result = new HashMap<>();
project.setUserId(1);
checkResult = projectService.hasProjectAndPerm(loginUser,project,result);
logger.info(result.toString());
Assert.assertTrue(checkResult);
}
@Test
public void testQueryProjectListPaging(){
IPage<Project> page = new Page<>(1,10);
page.setRecords(getList());
page.setTotal(1L);
Mockito.when(projectMapper.queryProjectListPaging(Mockito.any(Page.class), Mockito.eq(1), Mockito.eq(projectName))).thenReturn(page);
User loginUser = getLoginUser();
// project owner
Map<String, Object> result = projectService.queryProjectListPaging(loginUser,10,1,projectName);
logger.info(result.toString());
PageInfo<Project> pageInfo = (PageInfo<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
//admin
Mockito.when(projectMapper.queryProjectListPaging(Mockito.any(Page.class), Mockito.eq(0), Mockito.eq(projectName))).thenReturn(page);
loginUser.setUserType(UserType.ADMIN_USER);
result = projectService.queryProjectListPaging(loginUser,10,1,projectName);
logger.info(result.toString());
pageInfo = (PageInfo<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
}
@Test
public void testDeleteProject(){
Mockito.when(projectMapper.selectById(1)).thenReturn(getProject());
User loginUser = getLoginUser();
//PROJECT_NOT_FOUNT
Map<String, Object> result= projectService.deleteProject(loginUser,12);
logger.info(result.toString());
Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS));
loginUser.setId(2);
//USER_NO_OPERATION_PROJECT_PERM
result= projectService.deleteProject(loginUser,1);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM,result.get(Constants.STATUS));
//DELETE_PROJECT_ERROR_DEFINES_NOT_NULL
Mockito.when(processDefinitionMapper.queryAllDefinitionList(1)).thenReturn(getProcessDefinitions());
loginUser.setUserType(UserType.ADMIN_USER);
result= projectService.deleteProject(loginUser,1);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL,result.get(Constants.STATUS));
//success
Mockito.when(projectMapper.deleteById(1)).thenReturn(1);
Mockito.when(processDefinitionMapper.queryAllDefinitionList(1)).thenReturn(new ArrayList<>());
result= projectService.deleteProject(loginUser,1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testUpdate(){
User loginUser = getLoginUser();
Project project = getProject();
project.setId(2);
Mockito.when(projectMapper.queryByName(projectName)).thenReturn(project);
Mockito.when( projectMapper.selectById(1)).thenReturn(getProject());
// PROJECT_NOT_FOUNT
Map<String, Object> result = projectService.update(loginUser,12,projectName,"desc");
logger.info(result.toString());
Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS));
//PROJECT_ALREADY_EXISTS
result = projectService.update(loginUser,1,projectName,"desc");
logger.info(result.toString());
Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS,result.get(Constants.STATUS));
//success
project.setUserId(1);
Mockito.when(projectMapper.updateById(Mockito.any(Project.class))).thenReturn(1);
result = projectService.update(loginUser,1,"test","desc");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testQueryAuthorizedProject(){
User loginUser = getLoginUser();
Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList());
//USER_NO_OPERATION_PERM
Map<String, Object> result = projectService.queryAuthorizedProject(loginUser,3);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
//success
loginUser.setUserType(UserType.ADMIN_USER);
result = projectService.queryAuthorizedProject(loginUser,1);
logger.info(result.toString());
List<Project> projects = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(projects));
}
@Test
public void testQueryAllProjectList(){
Mockito.when(projectMapper.selectList(null)).thenReturn(getList());
Mockito.when(processDefinitionMapper.selectList(null)).thenReturn(getProcessDefinitions());
Map<String, Object> result = projectService.queryAllProjectList();
logger.info(result.toString());
List<Project> projects = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(projects));
}
@Test
public void testQueryUnauthorizedProject(){
// Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList());
Mockito.when(projectMapper.queryProjectExceptUserId(2)).thenReturn(getList());
User loginUser = new User();
loginUser.setUserType(UserType.ADMIN_USER);
Map<String, Object> result = projectService.queryUnauthorizedProject(loginUser,2);
logger.info(result.toString());
List<Project> projects = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(projects));
}
private Project getProject(){
Project project = new Project();
project.setId(1);
project.setName(projectName);
project.setUserId(1);
return project;
}
private List<Project> getList(){
List<Project> list = new ArrayList<>();
list.add(getProject());
return list;
}
/**
* create admin user
* @return
*/
private User getLoginUser(){
User loginUser = new User();
loginUser.setUserType(UserType.GENERAL_USER);
loginUser.setUserName(userName);
loginUser.setId(1);
return loginUser;
}
/**
* get project user
*/
private ProjectUser getProjectUser(){
ProjectUser projectUser = new ProjectUser();
projectUser.setProjectId(1);
projectUser.setUserId(1);
return projectUser;
}
private List<ProcessDefinition> getProcessDefinitions(){
List<ProcessDefinition> list = new ArrayList<>();
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setProjectId(1);
list.add(processDefinition);
return list;
}
private String getDesc(){
return "projectUserMapper.deleteProjectRelation(projectId,userId)projectUserMappe" +
".deleteProjectRelation(projectId,userId)projectUserMappe" +
"r.deleteProjectRelation(projectId,userId)projectUserMapper" +
".deleteProjectRelation(projectId,userId)projectUserMapper.deleteProjectRelation(projectId,userId)";
}
}

208
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java

@ -0,0 +1,208 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.QueueMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@RunWith(MockitoJUnitRunner.class)
public class QueueServiceTest {
private static final Logger logger = LoggerFactory.getLogger(QueueServiceTest.class);
@InjectMocks
private QueueService queueService;
@Mock
private QueueMapper queueMapper;
@Mock
private UserMapper userMapper;
private String queueName = "QueueServiceTest";
@Before
public void setUp() {
}
@After
public void after(){
}
@Test
public void testQueryList(){
Mockito.when(queueMapper.selectList(null)).thenReturn(getQueueList());
Map<String, Object> result = queueService.queryList(getLoginUser());
logger.info(result.toString());
List<Queue> queueList = (List<Queue>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(queueList));
}
@Test
public void testQueryListPage(){
IPage<Queue> page = new Page<>(1,10);
page.setTotal(1L);
page.setRecords(getQueueList());
Mockito.when(queueMapper.queryQueuePaging(Mockito.any(Page.class), Mockito.eq(queueName))).thenReturn(page);
Map<String, Object> result = queueService.queryList(getLoginUser(),queueName,1,10);
logger.info(result.toString());
PageInfo<Queue> pageInfo = (PageInfo<Queue>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
}
@Test
public void testCreateQueue(){
// queue is null
Map<String, Object> result = queueService.createQueue(getLoginUser(),null,queueName);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS));
// queueName is null
result = queueService.createQueue(getLoginUser(),queueName,null);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS));
// correct
result = queueService.createQueue(getLoginUser(),queueName,queueName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testUpdateQueue(){
Mockito.when(queueMapper.selectById(1)).thenReturn(getQueue());
Mockito.when(queueMapper.queryAllQueueList("test", null)).thenReturn(getQueueList());
Mockito.when(queueMapper.queryAllQueueList(null, "test")).thenReturn(getQueueList());
Mockito.when(userMapper.queryUserListByQueue(queueName)).thenReturn(getUserList());
// not exist
Map<String, Object> result = queueService.updateQueue(getLoginUser(),0,"queue",queueName);
logger.info(result.toString());
Assert.assertEquals(Status.QUEUE_NOT_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode());
//no need update
result = queueService.updateQueue(getLoginUser(),1,queueName,queueName);
logger.info(result.toString());
Assert.assertEquals(Status.NEED_NOT_UPDATE_QUEUE.getCode(),((Status)result.get(Constants.STATUS)).getCode());
//queue exist
result = queueService.updateQueue(getLoginUser(),1,"test",queueName);
logger.info(result.toString());
Assert.assertEquals(Status.QUEUE_VALUE_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode());
// queueName exist
result = queueService.updateQueue(getLoginUser(),1,"test1","test");
logger.info(result.toString());
Assert.assertEquals(Status.QUEUE_NAME_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode());
//success
result = queueService.updateQueue(getLoginUser(),1,"test1","test1");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getCode(),((Status)result.get(Constants.STATUS)).getCode());
}
@Test
public void testVerifyQueue(){
Mockito.when(queueMapper.queryAllQueueList(queueName, null)).thenReturn(getQueueList());
Mockito.when(queueMapper.queryAllQueueList(null, queueName)).thenReturn(getQueueList());
//queue null
Result result = queueService.verifyQueue(null,queueName);
logger.info(result.toString());
Assert.assertEquals(result.getCode().intValue(), Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode());
//queueName null
result = queueService.verifyQueue(queueName,null);
logger.info(result.toString());
Assert.assertEquals(result.getCode().intValue(), Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode());
//exist queueName
result = queueService.verifyQueue(queueName,queueName);
logger.info(result.toString());
Assert.assertEquals(result.getCode().intValue(), Status.QUEUE_NAME_EXIST.getCode());
//exist queue
result = queueService.verifyQueue(queueName,"test");
logger.info(result.toString());
Assert.assertEquals(result.getCode().intValue(), Status.QUEUE_VALUE_EXIST.getCode());
// success
result = queueService.verifyQueue("test","test");
logger.info(result.toString());
Assert.assertEquals(result.getCode().intValue(), Status.SUCCESS.getCode());
}
/**
* create admin user
* @return
*/
private User getLoginUser(){
User loginUser = new User();
loginUser.setUserType(UserType.ADMIN_USER);
loginUser.setId(99999999);
return loginUser;
}
private List<User> getUserList(){
List<User> list = new ArrayList<>();
list.add(getLoginUser());
return list;
}
/**
* get queue
* @return
*/
private Queue getQueue(){
Queue queue = new Queue();
queue.setId(1);
queue.setQueue(queueName);
queue.setQueueName(queueName);
return queue;
}
private List<Queue> getQueueList(){
List<Queue> queueList = new ArrayList<>();
queueList.add(getQueue());
return queueList;
}
}

121
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java

@ -16,37 +16,130 @@
*/ */
package org.apache.dolphinscheduler.api.service; package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang3.StringUtils; import org.apache.dolphinscheduler.dao.mapper.SessionMapper;
import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.mock.web.MockCookie;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.mock.web.MockHttpServletRequest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@RunWith(SpringRunner.class) @RunWith(MockitoJUnitRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
public class SessionServiceTest { public class SessionServiceTest {
private static final Logger logger = LoggerFactory.getLogger(SessionServiceTest.class); private static final Logger logger = LoggerFactory.getLogger(SessionServiceTest.class);
@Autowired @InjectMocks
private SessionService sessionService; private SessionService sessionService;
@Mock
private SessionMapper sessionMapper;
private String sessionId ="aaaaaaaaaaaaaaaaaa";
@Before
public void setUp() {
}
@After
public void after(){
}
/**
* create session
*/
@Test
public void testGetSession(){
Mockito.when(sessionMapper.selectById(sessionId)).thenReturn(getSession());
// get sessionId from header
MockHttpServletRequest mockHttpServletRequest = new MockHttpServletRequest();
mockHttpServletRequest.addHeader(Constants.SESSION_ID,sessionId);
mockHttpServletRequest.addHeader("HTTP_X_FORWARDED_FOR","127.0.0.1");
//query
Session session = sessionService.getSession(mockHttpServletRequest);
Assert.assertNotNull(session);
logger.info("session ip {}",session.getIp());
// get sessionId from cookie
mockHttpServletRequest = new MockHttpServletRequest();
mockHttpServletRequest.addHeader("HTTP_X_FORWARDED_FOR","127.0.0.1");
MockCookie mockCookie = new MockCookie(Constants.SESSION_ID,sessionId);
mockHttpServletRequest.setCookies(mockCookie);
//query
session = sessionService.getSession(mockHttpServletRequest);
Assert.assertNotNull(session);
logger.info("session ip {}",session.getIp());
Assert.assertEquals(session.getIp(),"127.0.0.1");
}
/**
* create session
*/
@Test
public void testCreateSession(){
String ip = "127.0.0.1";
User user = new User();
user.setUserType(UserType.GENERAL_USER);
user.setId(1);
Mockito.when(sessionMapper.queryByUserId(1)).thenReturn(getSessions());
String sessionId = sessionService.createSession(user, ip);
logger.info("createSessionId is "+sessionId);
Assert.assertTrue(StringUtils.isNotEmpty(sessionId));
}
/**
* sign out
* remove ip restrictions
*/
@Test @Test
public void createSession(){ public void testSignOut(){
int userId = 88888888;
String ip = "127.0.0.1";
User user = new User();
user.setId(userId);
sessionService.signOut(ip ,user);
User loginUser = new User(); }
loginUser.setId(1);
loginUser.setUserType(UserType.GENERAL_USER);
String session = sessionService.createSession(loginUser, "127.0.0.1"); private Session getSession(){
Assert.assertTrue(StringUtils.isNotEmpty(session));
Session session = new Session();
session.setId(sessionId);
session.setIp("127.0.0.1");
session.setLastLoginTime(DateUtils.addDays(new Date(),40));
session.setUserId(1);
return session;
} }
private List<Session> getSessions(){
List<Session> sessionList = new ArrayList<>();
sessionList.add(getSession());
return sessionList;
}
} }

225
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java

@ -16,38 +16,241 @@
*/ */
package org.apache.dolphinscheduler.api.service; package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.ApiApplicationServer; import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.List;
import java.util.Map; import java.util.Map;
@RunWith(SpringRunner.class) @RunWith(MockitoJUnitRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
public class TenantServiceTest { public class TenantServiceTest {
private static final Logger logger = LoggerFactory.getLogger(TenantServiceTest.class); private static final Logger logger = LoggerFactory.getLogger(TenantServiceTest.class);
@Autowired @InjectMocks
private TenantService tenantService; private TenantService tenantService;
@Mock
private TenantMapper tenantMapper;
@Mock
private ProcessDefinitionMapper processDefinitionMapper;
@Mock
private ProcessInstanceMapper processInstanceMapper;
@Mock
private UserMapper userMapper;
private String tenantCode ="TenantServiceTest";
private String tenantName ="TenantServiceTest";
@Test
public void testCreateTenant(){
User loginUser = getLoginUser();
Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList());
try {
//check tenantCode
Map<String, Object> result = tenantService.createTenant(getLoginUser(), "%!1111", tenantName, 1, "TenantServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.VERIFY_TENANT_CODE_ERROR,result.get(Constants.STATUS));
//check exist
result = tenantService.createTenant(loginUser, tenantCode, tenantName, 1, "TenantServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS));
// success
result = tenantService.createTenant(loginUser, "test", "test", 1, "TenantServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
} catch (Exception e) {
logger.error("create tenant error",e);
Assert.assertTrue(false);
}
}
@Test
public void testQueryTenantListPage(){
IPage<Tenant> page = new Page<>(1,10);
page.setRecords(getList());
page.setTotal(1L);
Mockito.when(tenantMapper.queryTenantPaging(Mockito.any(Page.class), Mockito.eq("TenantServiceTest"))).thenReturn(page);
Map<String, Object> result = tenantService.queryTenantList(getLoginUser(), "TenantServiceTest", 1, 10);
logger.info(result.toString());
PageInfo<Tenant> pageInfo = (PageInfo<Tenant>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
}
@Test
public void testUpdateTenant(){
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
try {
// id not exist
Map<String, Object> result = tenantService.updateTenant(getLoginUser(), 912222, tenantCode, tenantName, 1, "desc");
logger.info(result.toString());
// success
Assert.assertEquals(Status.TENANT_NOT_EXIST,result.get(Constants.STATUS));
result = tenantService.updateTenant(getLoginUser(), 1, tenantCode, "TenantServiceTest001", 1, "desc");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
} catch (Exception e) {
logger.error("update tenant error",e);
Assert.assertTrue(false);
}
}
@Test @Test
public void queryTenantList(){ public void testDeleteTenantById(){
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, Constants.NOT_TERMINATED_STATES)).thenReturn(getInstanceList());
Mockito.when(processDefinitionMapper.queryDefinitionListByTenant(2)).thenReturn(getDefinitionsList());
Mockito.when( userMapper.queryUserListByTenant(3)).thenReturn(getUserList());
try {
//TENANT_NOT_EXIST
Map<String, Object> result = tenantService.deleteTenantById(getLoginUser(),12);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST,result.get(Constants.STATUS));
//DELETE_TENANT_BY_ID_FAIL
result = tenantService.deleteTenantById(getLoginUser(),1);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL,result.get(Constants.STATUS));
//DELETE_TENANT_BY_ID_FAIL_DEFINES
Mockito.when(tenantMapper.queryById(2)).thenReturn(getTenant(2));
result = tenantService.deleteTenantById(getLoginUser(),2);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_DEFINES,result.get(Constants.STATUS));
//DELETE_TENANT_BY_ID_FAIL_USERS
Mockito.when(tenantMapper.queryById(3)).thenReturn(getTenant(3));
result = tenantService.deleteTenantById(getLoginUser(),3);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_USERS,result.get(Constants.STATUS));
// success
Mockito.when(tenantMapper.queryById(4)).thenReturn(getTenant(4));
result = tenantService.deleteTenantById(getLoginUser(),4);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
} catch (Exception e) {
logger.error("delete tenant error",e);
Assert.assertTrue(false);
}
}
@Test
public void testQueryTenantList(){
Mockito.when( tenantMapper.selectList(null)).thenReturn(getList());
Map<String, Object> result = tenantService.queryTenantList(getLoginUser());
logger.info(result.toString());
List<Tenant> tenantList = (List<Tenant>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList));
}
@Test
public void testVerifyTenantCode(){
Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList());
// tenantCode not exist
Result result = tenantService.verifyTenantCode("s00000000000l887888885554444sfjdskfjslakslkdf");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg());
// tenantCode exist
result = tenantService.verifyTenantCode(getTenant().getTenantCode());
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NAME_EXIST.getMsg(),result.getMsg());
}
/**
* get user
* @return
*/
private User getLoginUser(){
User loginUser = new User(); User loginUser = new User();
loginUser.setUserType(UserType.ADMIN_USER); loginUser.setUserType(UserType.ADMIN_USER);
Map<String, Object> map = tenantService.queryTenantList(loginUser); return loginUser;
Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); }
logger.info(map.toString());
/**
* get list
* @return
*/
private List<Tenant> getList(){
List<Tenant> tenantList = new ArrayList<>();
tenantList.add(getTenant());
return tenantList;
} }
/**
* get tenant
* @return
*/
private Tenant getTenant(){
return getTenant(1);
}
/**
* get tenant
* @return
*/
private Tenant getTenant(int id){
Tenant tenant = new Tenant();
tenant.setId(id);
tenant.setTenantCode(tenantCode);
tenant.setTenantName(tenantName);
return tenant;
}
private List<User> getUserList(){
List<User> userList = new ArrayList<>();
userList.add(getLoginUser());
return userList;
}
private List<ProcessInstance> getInstanceList(){
List<ProcessInstance> processInstances = new ArrayList<>();
ProcessInstance processInstance = new ProcessInstance();
processInstances.add(processInstance);
return processInstances;
}
private List<ProcessDefinition> getDefinitionsList(){
List<ProcessDefinition> processDefinitions = new ArrayList<>();
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinitions.add(processDefinition);
return processDefinitions;
}
} }

434
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java

@ -16,39 +16,449 @@
*/ */
package org.apache.dolphinscheduler.api.service; package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.ApiApplicationServer; import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.List;
import java.util.Map; import java.util.Map;
@RunWith(SpringRunner.class) import static org.mockito.ArgumentMatchers.any;
@SpringBootTest(classes = ApiApplicationServer.class) import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class UsersServiceTest { public class UsersServiceTest {
private static final Logger logger = LoggerFactory.getLogger(UsersServiceTest.class); private static final Logger logger = LoggerFactory.getLogger(UsersServiceTest.class);
@Autowired @InjectMocks
private UsersService usersService; private UsersService usersService;
@Mock
private UserMapper userMapper;
@Mock
private TenantMapper tenantMapper;
@Mock
private ProjectUserMapper projectUserMapper;
@Mock
private ResourceUserMapper resourcesUserMapper;
@Mock
private UDFUserMapper udfUserMapper;
@Mock
private DataSourceUserMapper datasourceUserMapper;
@Mock
private AlertGroupMapper alertGroupMapper;
private String queueName ="UsersServiceTestQueue";
@Before
public void before(){
}
@After
public void after(){
}
@Test
public void testCreateUser(){
User user = new User();
user.setUserType(UserType.ADMIN_USER);
String userName = "userTest0001~";
String userPassword = "userTest";
String email = "123@qq.com";
int tenantId = Integer.MAX_VALUE;
String phone= "13456432345";
try {
//userName error
Map<String, Object> result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
userName = "userTest0001";
userPassword = "userTest000111111111111111";
//password error
result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
userPassword = "userTest0001";
email = "1q.com";
//email error
result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
email = "122222@qq.com";
phone ="2233";
//phone error
result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
phone = "13456432345";
//tenantId not exists
result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST, result.get(Constants.STATUS));
//success
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
result = usersService.createUser(user, userName, userPassword, email, 1, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
} catch (Exception e) {
logger.error(Status.CREATE_USER_ERROR.getMsg(),e);
Assert.assertTrue(false);
}
}
@Test
public void testQueryUser(){
String userName = "userTest0001";
String userPassword = "userTest0001";
when(userMapper.queryUserByNamePassword(userName,EncryptionUtils.getMd5(userPassword))).thenReturn(getGeneralUser());
User queryUser = usersService.queryUser(userName, userPassword);
logger.info(queryUser.toString());
Assert.assertTrue(queryUser!=null);
}
@Test
public void testQueryUserList(){
User user = new User();
//no operate
Map<String, Object> result = usersService.queryUserList(user);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
user.setUserType(UserType.ADMIN_USER);
when(userMapper.selectList(null )).thenReturn(getUserList());
result = usersService.queryUserList(user);
List<User> userList = (List<User>) result.get(Constants.DATA_LIST);
Assert.assertTrue(userList.size()>0);
}
@Test
public void testQueryUserListPage(){
User user = new User();
IPage<User> page = new Page<>(1,10);
page.setRecords(getUserList());
when(userMapper.queryUserPaging(any(Page.class), eq("userTest"))).thenReturn(page);
//no operate
Map<String, Object> result = usersService.queryUserList(user,"userTest",1,10);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
user.setUserType(UserType.ADMIN_USER);
result = usersService.queryUserList(user,"userTest",1,10);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
PageInfo<User> pageInfo = (PageInfo<User>) result.get(Constants.DATA_LIST);
Assert.assertTrue(pageInfo.getLists().size()>0);
}
@Test
public void testUpdateUser(){
String userName = "userTest0001";
String userPassword = "userTest0001";
try {
//user not exist
Map<String, Object> result = usersService.updateUser(0,userName,userPassword,"3443@qq.com",1,"13457864543","queue");
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
logger.info(result.toString());
//success
when(userMapper.selectById(1)).thenReturn(getUser());
result = usersService.updateUser(1,userName,userPassword,"32222s@qq.com",1,"13457864543","queue");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
} catch (Exception e) {
logger.error("update user error",e);
Assert.assertTrue(false);
}
}
@Test
public void testDeleteUserById(){
User loginUser = new User();
try {
when(userMapper.queryTenantCodeByUserId(1)).thenReturn(getUser());
when(userMapper.selectById(1)).thenReturn(getUser());
//no operate
Map<String, Object> result = usersService.deleteUserById(loginUser,3);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
// user not exist
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.deleteUserById(loginUser,3);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
result = usersService.deleteUserById(loginUser,1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
} catch (Exception e) {
logger.error("delete user error",e);
Assert.assertTrue(false);
}
}
@Test
public void testGrantProject(){
when(userMapper.selectById(1)).thenReturn(getUser());
User loginUser = new User();
String projectIds= "100000,120000";
Map<String, Object> result = usersService.grantProject(loginUser, 1, projectIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//user not exist
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.grantProject(loginUser, 2, projectIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
result = usersService.grantProject(loginUser, 1, projectIds);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testGrantResources(){
String resourceIds = "100000,120000";
when(userMapper.selectById(1)).thenReturn(getUser());
User loginUser = new User();
Map<String, Object> result = usersService.grantResources(loginUser, 1, resourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//user not exist
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.grantResources(loginUser, 2, resourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
result = usersService.grantResources(loginUser, 1, resourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testGrantUDFFunction(){
String udfIds = "100000,120000";
when(userMapper.selectById(1)).thenReturn(getUser());
User loginUser = new User();
Map<String, Object> result = usersService.grantUDFFunction(loginUser, 1, udfIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//user not exist
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.grantUDFFunction(loginUser, 2, udfIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
result = usersService.grantUDFFunction(loginUser, 1, udfIds);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testGrantDataSource(){
String datasourceIds = "100000,120000";
when(userMapper.selectById(1)).thenReturn(getUser());
User loginUser = new User();
Map<String, Object> result = usersService.grantDataSource(loginUser, 1, datasourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//user not exist
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.grantDataSource(loginUser, 2, datasourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
result = usersService.grantDataSource(loginUser, 1, datasourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test @Test
public void getUserInfo(){ public void getUserInfo(){
User loginUser = new User(); User loginUser = new User();
loginUser.setId(19); loginUser.setUserName("admin");
loginUser.setUserType(UserType.GENERAL_USER); loginUser.setUserType(UserType.ADMIN_USER);
Map<String, Object> map = usersService.getUserInfo(loginUser); // get admin user
Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); Map<String, Object> result = usersService.getUserInfo(loginUser);
logger.info(map.toString()); logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
User tempUser = (User) result.get(Constants.DATA_LIST);
//check userName
Assert.assertEquals("admin",tempUser.getUserName());
//get general user
loginUser.setUserType(null);
loginUser.setId(1);
when(userMapper.queryDetailsById(1)).thenReturn(getGeneralUser());
result = usersService.getUserInfo(loginUser);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
tempUser = (User) result.get(Constants.DATA_LIST);
//check userName
Assert.assertEquals("userTest0001",tempUser.getUserName());
} }
@Test
public void testQueryAllGeneralUsers(){
User loginUser = new User();
//no operate
Map<String, Object> result = usersService.queryAllGeneralUsers(loginUser);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
loginUser.setUserType(UserType.ADMIN_USER);
when(userMapper.queryAllGeneralUser()).thenReturn(getUserList());
result = usersService.queryAllGeneralUsers(loginUser);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<User> userList = (List<User>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(userList));
}
@Test
public void testVerifyUserName(){
//not exist user
Result result = usersService.verifyUserName("admin89899");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
//exist user
when(userMapper.queryByUserNameAccurately("userTest0001")).thenReturn(getUser());
result = usersService.verifyUserName("userTest0001");
logger.info(result.toString());
Assert.assertEquals(Status.USER_NAME_EXIST.getMsg(), result.getMsg());
}
@Test
public void testUnauthorizedUser(){
User loginUser = new User();
when(userMapper.selectList(null )).thenReturn(getUserList());
when( userMapper.queryUserListByAlertGroupId(2)).thenReturn(getUserList());
//no operate
Map<String, Object> result = usersService.unauthorizedUser(loginUser, 2);
logger.info(result.toString());
loginUser.setUserType(UserType.ADMIN_USER);
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
result = usersService.unauthorizedUser(loginUser, 2);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testAuthorizedUser(){
User loginUser = new User();
when(userMapper.queryUserListByAlertGroupId(2)).thenReturn(getUserList());
//no operate
Map<String, Object> result = usersService.authorizedUser(loginUser, 2);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.authorizedUser(loginUser, 2);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<User> userList = (List<User>) result.get(Constants.DATA_LIST);
logger.info(result.toString());
Assert.assertTrue(CollectionUtils.isNotEmpty(userList));
}
/**
* get user
* @return
*/
private User getGeneralUser(){
User user = new User();
user.setUserType(UserType.GENERAL_USER);
user.setUserName("userTest0001");
user.setUserPassword("userTest0001");
return user;
}
private List<User> getUserList(){
List<User> userList = new ArrayList<>();
userList.add(getGeneralUser());
return userList;
}
/**
* get user
*/
private User getUser(){
User user = new User();
user.setUserType(UserType.ADMIN_USER);
user.setUserName("userTest0001");
user.setUserPassword("userTest0001");
return user;
}
private Tenant getTenant(){
Tenant tenant = new Tenant();
tenant.setId(1);
return tenant;
}
} }

173
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java

@ -0,0 +1,173 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@RunWith(MockitoJUnitRunner.class)
public class WorkerGroupServiceTest {
private static final Logger logger = LoggerFactory.getLogger(WorkerGroupServiceTest.class);
@InjectMocks
private WorkerGroupService workerGroupService;
@Mock
private WorkerGroupMapper workerGroupMapper;
@Mock
private ProcessInstanceMapper processInstanceMapper;
private String groupName="groupName000001";
/**
* create or update a worker group
*/
@Test
public void testSaveWorkerGroup(){
User user = new User();
// general user add
user.setUserType(UserType.GENERAL_USER);
Map<String, Object> result = workerGroupService.saveWorkerGroup(user, 0, groupName, "127.0.0.1");
logger.info(result.toString());
Assert.assertEquals( Status.USER_NO_OPERATION_PERM.getMsg(),(String) result.get(Constants.MSG));
//success
user.setUserType(UserType.ADMIN_USER);
result = workerGroupService.saveWorkerGroup(user, 0, groupName, "127.0.0.1");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG));
// group name exist
Mockito.when(workerGroupMapper.selectById(2)).thenReturn(getWorkerGroup(2));
Mockito.when(workerGroupMapper.queryWorkerGroupByName(groupName)).thenReturn(getList());
result = workerGroupService.saveWorkerGroup(user, 2, groupName, "127.0.0.1");
logger.info(result.toString());
Assert.assertEquals(Status.NAME_EXIST,result.get(Constants.STATUS));
}
/**
* query worker group paging
*/
@Test
public void testQueryAllGroupPaging(){
User user = new User();
// general user add
user.setUserType(UserType.GENERAL_USER);
Map<String, Object> result = workerGroupService.queryAllGroupPaging(user, 1, 10, groupName);
logger.info(result.toString());
Assert.assertEquals((String) result.get(Constants.MSG), Status.USER_NO_OPERATION_PERM.getMsg());
//success
user.setUserType(UserType.ADMIN_USER);
Page<WorkerGroup> page = new Page<>(1,10);
page.setRecords(getList());
page.setSize(1L);
Mockito.when(workerGroupMapper.queryListPaging(Mockito.any(Page.class), Mockito.eq(groupName))).thenReturn(page);
result = workerGroupService.queryAllGroupPaging(user, 1, 10, groupName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG));
PageInfo<WorkerGroup> pageInfo = (PageInfo<WorkerGroup>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
}
/**
* delete group by id
*/
@Test
public void testDeleteWorkerGroupById(){
//DELETE_WORKER_GROUP_BY_ID_FAIL
Mockito.when(processInstanceMapper.queryByWorkerGroupIdAndStatus(1, Constants.NOT_TERMINATED_STATES)).thenReturn(getProcessInstanceList());
Map<String, Object> result = workerGroupService.deleteWorkerGroupById(1);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_WORKER_GROUP_BY_ID_FAIL.getCode(),((Status) result.get(Constants.STATUS)).getCode());
//correct
result = workerGroupService.deleteWorkerGroupById(2);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG));
}
@Test
public void testQueryAllGroup(){
Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(getList());
Map<String, Object> result = workerGroupService.queryAllGroup();
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG));
List<WorkerGroup> workerGroupList = (List<WorkerGroup>) result.get(Constants.DATA_LIST);
Assert.assertTrue(workerGroupList.size()>0);
}
/**
* get processInstances
* @return
*/
private List<ProcessInstance> getProcessInstanceList(){
List<ProcessInstance> processInstances = new ArrayList<>();
processInstances.add(new ProcessInstance());
return processInstances;
}
/**
* get Group
* @return
*/
private WorkerGroup getWorkerGroup(int id){
WorkerGroup workerGroup = new WorkerGroup();
workerGroup.setName(groupName);
workerGroup.setId(id);
return workerGroup;
}
private WorkerGroup getWorkerGroup(){
return getWorkerGroup(1);
}
private List<WorkerGroup> getList(){
List<WorkerGroup> list = new ArrayList<>();
list.add(getWorkerGroup());
return list;
}
}

21
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -26,25 +26,10 @@ import java.util.regex.Pattern;
*/ */
public final class Constants { public final class Constants {
/**
* zookeeper properties path
*/
public static final String ZOOKEEPER_PROPERTIES_PATH = "zookeeper.properties";
/**
* hadoop properties path
*/
public static final String HADOOP_PROPERTIES_PATH = "/common/hadoop/hadoop.properties";
/** /**
* common properties path * common properties path
*/ */
public static final String COMMON_PROPERTIES_PATH = "/common/common.properties"; public static final String COMMON_PROPERTIES_PATH = "/common.properties";
/**
* dao properties path
*/
public static final String DAO_PROPERTIES_PATH = "application.properties";
/** /**
* fs.defaultFS * fs.defaultFS
@ -227,6 +212,8 @@ public final class Constants {
public static final String ZOOKEEPER_CONNECTION_TIMEOUT = "zookeeper.connection.timeout"; public static final String ZOOKEEPER_CONNECTION_TIMEOUT = "zookeeper.connection.timeout";
public static final String ZOOKEEPER_RETRY_SLEEP = "zookeeper.retry.sleep"; public static final String ZOOKEEPER_RETRY_SLEEP = "zookeeper.retry.sleep";
public static final String ZOOKEEPER_RETRY_BASE_SLEEP = "zookeeper.retry.base.sleep";
public static final String ZOOKEEPER_RETRY_MAX_SLEEP = "zookeeper.retry.max.sleep";
public static final String ZOOKEEPER_RETRY_MAXTIME = "zookeeper.retry.maxtime"; public static final String ZOOKEEPER_RETRY_MAXTIME = "zookeeper.retry.maxtime";
@ -469,7 +456,7 @@ public final class Constants {
/** /**
* task record configuration path * task record configuration path
*/ */
public static final String APPLICATION_PROPERTIES = "application-dao.properties"; public static final String APPLICATION_PROPERTIES = "application.properties";
public static final String TASK_RECORD_URL = "task.record.datasource.url"; public static final String TASK_RECORD_URL = "task.record.datasource.url";

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@ -29,7 +29,6 @@ import java.util.Map;
import java.util.Properties; import java.util.Properties;
import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
import static org.apache.dolphinscheduler.common.Constants.HADOOP_PROPERTIES_PATH;
/** /**
* property utils * property utils
@ -51,7 +50,7 @@ public class PropertyUtils {
} }
private void init(){ private void init(){
String[] propertyFiles = new String[]{HADOOP_PROPERTIES_PATH,COMMON_PROPERTIES_PATH}; String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH};
for (String fileName : propertyFiles) { for (String fileName : propertyFiles) {
InputStream fis = null; InputStream fis = null;
try { try {

98
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java

@ -16,6 +16,22 @@
*/ */
package org.apache.dolphinscheduler.common.zk; package org.apache.dolphinscheduler.common.zk;
import static org.apache.dolphinscheduler.common.Constants.ADD_ZK_OP;
import static org.apache.dolphinscheduler.common.Constants.DELETE_ZK_OP;
import static org.apache.dolphinscheduler.common.Constants.MASTER_PREFIX;
import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH;
import static org.apache.dolphinscheduler.common.Constants.UNDERLINE;
import static org.apache.dolphinscheduler.common.Constants.WORKER_PREFIX;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.IStoppable;
import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.enums.ZKNodeType;
@ -23,26 +39,9 @@ import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.ResInfo; import org.apache.dolphinscheduler.common.utils.ResInfo;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.framework.state.ConnectionStateListener;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.zookeeper.CreateMode;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.*;
import static org.apache.dolphinscheduler.common.Constants.*;
/** /**
* abstract zookeeper client * abstract zookeeper client
@ -70,8 +69,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
return; return;
} }
byte[] bytes = zkClient.getData().forPath(znode); String resInfoStr = super.get(znode);
String resInfoStr = new String(bytes);
String[] splits = resInfoStr.split(Constants.COMMA); String[] splits = resInfoStr.split(Constants.COMMA);
if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){
return; return;
@ -107,8 +105,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
String type = serverType.equals(MASTER_PREFIX) ? MASTER_PREFIX : WORKER_PREFIX; String type = serverType.equals(MASTER_PREFIX) ? MASTER_PREFIX : WORKER_PREFIX;
String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo;
if(zkClient.checkExists().forPath(zNode) == null || if(!isExisted(zNode) || isExisted(deadServerPath)){
zkClient.checkExists().forPath(deadServerPath) != null ){
return true; return true;
} }
@ -118,14 +115,12 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
public void removeDeadServerByHost(String host, String serverType) throws Exception { public void removeDeadServerByHost(String host, String serverType) throws Exception {
List<String> deadServers = zkClient.getChildren().forPath(getDeadZNodeParentPath()); List<String> deadServers = super.getChildrenKeys(getDeadZNodeParentPath());
for(String serverPath : deadServers){ for(String serverPath : deadServers){
if(serverPath.startsWith(serverType+UNDERLINE+host)){ if(serverPath.startsWith(serverType+UNDERLINE+host)){
String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath; String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath;
if(zkClient.checkExists().forPath(server) != null){ super.remove(server);
zkClient.delete().forPath(server); logger.info("{} server {} deleted from zk dead server path success" , serverType , host);
logger.info("{} server {} deleted from zk dead server path success" , serverType , host);
}
} }
} }
} }
@ -143,8 +138,8 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
// create temporary sequence nodes for master znode // create temporary sequence nodes for master znode
String parentPath = getZNodeParentPath(zkNodeType); String parentPath = getZNodeParentPath(zkNodeType);
String serverPathPrefix = parentPath + "/" + OSUtils.getHost(); String serverPathPrefix = parentPath + "/" + OSUtils.getHost();
String registerPath = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath( String registerPath = serverPathPrefix + UNDERLINE;
serverPathPrefix + "_", heartbeatZKInfo.getBytes()); super.persistEphemeral(registerPath, heartbeatZKInfo);
logger.info("register {} node {} success" , zkNodeType.toString(), registerPath); logger.info("register {} node {} success" , zkNodeType.toString(), registerPath);
return registerPath; return registerPath;
} }
@ -165,7 +160,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
} }
registerPath = createZNodePath(zkNodeType); registerPath = createZNodePath(zkNodeType);
// handle dead server // handle dead server
handleDeadServer(registerPath, zkNodeType, Constants.DELETE_ZK_OP); handleDeadServer(registerPath, zkNodeType, Constants.DELETE_ZK_OP);
return registerPath; return registerPath;
@ -196,10 +191,10 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
}else if(opType.equals(ADD_ZK_OP)){ }else if(opType.equals(ADD_ZK_OP)){
String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo;
if(zkClient.checkExists().forPath(deadServerPath) == null){ if(!super.isExisted(deadServerPath)){
//add dead server info to zk dead server path : /dead-servers/ //add dead server info to zk dead server path : /dead-servers/
zkClient.create().forPath(deadServerPath,(type + UNDERLINE + ipSeqNo).getBytes()); super.persist(deadServerPath,(type + UNDERLINE + ipSeqNo));
logger.info("{} server dead , and {} added to zk dead server path success" , logger.info("{} server dead , and {} added to zk dead server path success" ,
zkNodeType.toString(), zNode); zkNodeType.toString(), zNode);
@ -226,19 +221,13 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
List<String> childrenList = new ArrayList<>(); List<String> childrenList = new ArrayList<>();
try { try {
// read master node parent path from conf // read master node parent path from conf
if(zkClient.checkExists().forPath(getZNodeParentPath(ZKNodeType.MASTER)) != null){ if(super.isExisted(getZNodeParentPath(ZKNodeType.MASTER))){
childrenList = zkClient.getChildren().forPath(getZNodeParentPath(ZKNodeType.MASTER)); childrenList = super.getChildrenKeys(getZNodeParentPath(ZKNodeType.MASTER));
} }
} catch (Exception e) { } catch (Exception e) {
if(e.getMessage().contains("java.lang.IllegalStateException: instance must be started")){ logger.error("getActiveMasterNum error",e);
logger.error("zookeeper service not started",e);
}else{
logger.error(e.getMessage(),e);
}
}finally {
return childrenList.size();
} }
return childrenList.size();
} }
/** /**
@ -280,10 +269,9 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
Map<String, String> masterMap = new HashMap<>(); Map<String, String> masterMap = new HashMap<>();
try { try {
String path = getZNodeParentPath(zkNodeType); String path = getZNodeParentPath(zkNodeType);
List<String> serverList = getZkClient().getChildren().forPath(path); List<String> serverList = super.getChildrenKeys(path);
for(String server : serverList){ for(String server : serverList){
byte[] bytes = getZkClient().getData().forPath(path + "/" + server); masterMap.putIfAbsent(server, super.get(path + "/" + server));
masterMap.putIfAbsent(server, new String(bytes));
} }
} catch (Exception e) { } catch (Exception e) {
logger.error("get server list failed : " + e.getMessage(), e); logger.error("get server list failed : " + e.getMessage(), e);
@ -307,7 +295,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
} }
Map<String, String> serverMaps = getServerMaps(zkNodeType); Map<String, String> serverMaps = getServerMaps(zkNodeType);
for(String hostKey : serverMaps.keySet()){ for(String hostKey : serverMaps.keySet()){
if(hostKey.startsWith(host)){ if(hostKey.startsWith(host + UNDERLINE)){
return true; return true;
} }
} }
@ -430,27 +418,15 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
*/ */
protected void initSystemZNode(){ protected void initSystemZNode(){
try { try {
createNodePath(getMasterZNodeParentPath()); persist(getMasterZNodeParentPath(), "");
createNodePath(getWorkerZNodeParentPath()); persist(getWorkerZNodeParentPath(), "");
createNodePath(getDeadZNodeParentPath()); persist(getDeadZNodeParentPath(), "");
} catch (Exception e) { } catch (Exception e) {
logger.error("init system znode failed : " + e.getMessage(),e); logger.error("init system znode failed : " + e.getMessage(),e);
} }
} }
/**
* create zookeeper node path if not exists
* @param zNodeParentPath zookeeper parent path
* @throws Exception errors
*/
private void createNodePath(String zNodeParentPath) throws Exception {
if(null == zkClient.checkExists().forPath(zNodeParentPath)){
zkClient.create().creatingParentContainersIfNeeded()
.withMode(CreateMode.PERSISTENT).forPath(zNodeParentPath);
}
}
/** /**
* server self dead, stop all threads * server self dead, stop all threads
* @param serverHost server host * @param serverHost server host

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java

@ -24,7 +24,7 @@ import org.springframework.stereotype.Component;
* zookeeper conf * zookeeper conf
*/ */
@Component @Component
@PropertySource("classpath:zookeeper.properties") @PropertySource("classpath:common.properties")
public class ZookeeperConfig { public class ZookeeperConfig {
//zk connect config //zk connect config

51
dolphinscheduler-common/src/main/resources/common/common.properties → dolphinscheduler-common/src/main/resources/common.properties

@ -18,14 +18,21 @@
#task queue implementation, default "zookeeper" #task queue implementation, default "zookeeper"
dolphinscheduler.queue.impl=zookeeper dolphinscheduler.queue.impl=zookeeper
# user data directory path, self configuration, please make sure the directory exists and have read write permissions #zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
data.basedir.path=/tmp/dolphinscheduler zookeeper.quorum=localhost:2181
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions #dolphinscheduler root directory
data.download.basedir.path=/tmp/dolphinscheduler/download zookeeper.dolphinscheduler.root=/dolphinscheduler
# process execute directory. self configuration, please make sure the directory exists and have read write permissions #dolphinscheduler failover directory
process.exec.basepath=/tmp/dolphinscheduler/exec zookeeper.session.timeout=300
zookeeper.connection.timeout=300
zookeeper.retry.base.sleep=100
zookeeper.retry.max.sleep=30000
zookeeper.retry.maxtime=5
# resource upload startup type : HDFS,S3,NONE
res.upload.startup.type=NONE
# Users who have permission to create directories under the HDFS root path # Users who have permission to create directories under the HDFS root path
hdfs.root.user=hdfs hdfs.root.user=hdfs
@ -33,8 +40,15 @@ hdfs.root.user=hdfs
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
data.store2hdfs.basepath=/dolphinscheduler data.store2hdfs.basepath=/dolphinscheduler
# resource upload startup type : HDFS,S3,NONE # user data directory path, self configuration, please make sure the directory exists and have read write permissions
res.upload.startup.type=NONE data.basedir.path=/tmp/dolphinscheduler
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
data.download.basedir.path=/tmp/dolphinscheduler/download
# process execute directory. self configuration, please make sure the directory exists and have read write permissions
process.exec.basepath=/tmp/dolphinscheduler/exec
# whether kerberos starts # whether kerberos starts
hadoop.security.authentication.startup.state=false hadoop.security.authentication.startup.state=false
@ -57,3 +71,24 @@ resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties
# is development state? default "false" # is development state? default "false"
development.state=true development.state=true
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
# to the conf directory,support s3,for example : s3a://dolphinscheduler
fs.defaultFS=hdfs://mycluster:8020
# s3 need,s3 endpoint
fs.s3a.endpoint=http://192.168.199.91:9010
# s3 need,s3 access key
fs.s3a.access.key=A3DXS30FO22544RE
# s3 need,s3 secret key
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
#resourcemanager ha note this need ips , this empty if single
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s

35
dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties

@ -1,35 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
# to the conf directory,support s3,for example : s3a://dolphinscheduler
fs.defaultFS=hdfs://mycluster:8020
# s3 need,s3 endpoint
fs.s3a.endpoint=http://192.168.199.91:9010
# s3 need,s3 access key
fs.s3a.access.key=A3DXS30FO22544RE
# s3 need,s3 secret key
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
#resourcemanager ha note this need ips , this empty if single
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s

18
dolphinscheduler-common/src/main/resources/quartz.properties

@ -18,6 +18,15 @@
#============================================================================ #============================================================================
# Configure Main Scheduler Properties # Configure Main Scheduler Properties
#============================================================================ #============================================================================
#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate
org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
org.quartz.dataSource.myDs.driver = org.postgresql.Driver
#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8
org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinscheduler?characterEncoding=utf8
org.quartz.dataSource.myDs.user = test
org.quartz.dataSource.myDs.password = test
org.quartz.scheduler.instanceName = DolphinScheduler org.quartz.scheduler.instanceName = DolphinScheduler
org.quartz.scheduler.instanceId = AUTO org.quartz.scheduler.instanceId = AUTO
org.quartz.scheduler.makeSchedulerThreadDaemon = true org.quartz.scheduler.makeSchedulerThreadDaemon = true
@ -37,8 +46,7 @@ org.quartz.threadPool.threadPriority = 5
#============================================================================ #============================================================================
org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX
#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate
org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
org.quartz.jobStore.tablePrefix = QRTZ_ org.quartz.jobStore.tablePrefix = QRTZ_
org.quartz.jobStore.isClustered = true org.quartz.jobStore.isClustered = true
org.quartz.jobStore.misfireThreshold = 60000 org.quartz.jobStore.misfireThreshold = 60000
@ -49,11 +57,5 @@ org.quartz.jobStore.dataSource = myDs
# Configure Datasources # Configure Datasources
#============================================================================ #============================================================================
org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider
#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
org.quartz.dataSource.myDs.driver = org.postgresql.Driver
#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8
org.quartz.dataSource.myDs.URL = jdbc:postgresql://192.168.xx.xx:5432/dolphinscheduler?characterEncoding=utf8
org.quartz.dataSource.myDs.user = xx
org.quartz.dataSource.myDs.password = xx
org.quartz.dataSource.myDs.maxConnections = 10 org.quartz.dataSource.myDs.maxConnections = 10
org.quartz.dataSource.myDs.validationQuery = select 1 org.quartz.dataSource.myDs.validationQuery = select 1

43
dolphinscheduler-common/src/main/resources/zookeeper.properties

@ -1,43 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
zookeeper.quorum=localhost:2181
#dolphinscheduler root directory
zookeeper.dolphinscheduler.root=/dolphinscheduler
#zookeeper server dirctory
#zookeeper.dolphinscheduler.dead.servers=/dolphinscheduler/dead-servers
#zookeeper.dolphinscheduler.masters=/dolphinscheduler/masters
#zookeeper.dolphinscheduler.workers=/dolphinscheduler/workers
#zookeeper lock dirctory
#zookeeper.dolphinscheduler.lock.masters=/dolphinscheduler/lock/masters
#zookeeper.dolphinscheduler.lock.workers=/dolphinscheduler/lock/workers
#dolphinscheduler failover directory
#zookeeper.dolphinscheduler.lock.failover.masters=/dolphinscheduler/lock/failover/masters
#zookeeper.dolphinscheduler.lock.failover.workers=/dolphinscheduler/lock/failover/workers
#zookeeper.dolphinscheduler.lock.failover.startup.masters=/dolphinscheduler/lock/failover/startup-masters
#dolphinscheduler failover directory
zookeeper.session.timeout=300
zookeeper.connection.timeout=300
zookeeper.retry.base.sleep=100
zookeeper.retry.max.sleep=30000
zookeeper.retry.maxtime=5

71
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java

@ -32,7 +32,6 @@ import java.util.Map;
/** /**
* alert * alert
*/ */
@Data
@TableName("t_ds_alert") @TableName("t_ds_alert")
public class Alert { public class Alert {
@ -217,6 +216,72 @@ public class Alert {
this.updateTime = updateTime; this.updateTime = updateTime;
} }
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Alert alert = (Alert) o;
if (id != alert.id) {
return false;
}
if (alertGroupId != alert.alertGroupId) {
return false;
}
if (!title.equals(alert.title)) {
return false;
}
if (showType != alert.showType) {
return false;
}
if (!content.equals(alert.content)) {
return false;
}
if (alertType != alert.alertType) {
return false;
}
if (alertStatus != alert.alertStatus) {
return false;
}
if (!log.equals(alert.log)) {
return false;
}
if (!receivers.equals(alert.receivers)) {
return false;
}
if (!receiversCc.equals(alert.receiversCc)) {
return false;
}
if (!createTime.equals(alert.createTime)) {
return false;
}
return updateTime.equals(alert.updateTime) && info.equals(alert.info);
}
@Override
public int hashCode() {
int result = id;
result = 31 * result + title.hashCode();
result = 31 * result + showType.hashCode();
result = 31 * result + content.hashCode();
result = 31 * result + alertType.hashCode();
result = 31 * result + alertStatus.hashCode();
result = 31 * result + log.hashCode();
result = 31 * result + alertGroupId;
result = 31 * result + receivers.hashCode();
result = 31 * result + receiversCc.hashCode();
result = 31 * result + createTime.hashCode();
result = 31 * result + updateTime.hashCode();
result = 31 * result + info.hashCode();
return result;
}
@Override @Override
public String toString() { public String toString() {
return "Alert{" + return "Alert{" +
@ -228,10 +293,10 @@ public class Alert {
", alertStatus=" + alertStatus + ", alertStatus=" + alertStatus +
", log='" + log + '\'' + ", log='" + log + '\'' +
", alertGroupId=" + alertGroupId + ", alertGroupId=" + alertGroupId +
", createTime=" + createTime +
", updateTime=" + updateTime +
", receivers='" + receivers + '\'' + ", receivers='" + receivers + '\'' +
", receiversCc='" + receiversCc + '\'' + ", receiversCc='" + receiversCc + '\'' +
", createTime=" + createTime +
", updateTime=" + updateTime +
", info=" + info + ", info=" + info +
'}'; '}';
} }

74
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java

@ -28,7 +28,6 @@ import java.util.Date;
/** /**
* command * command
*/ */
@Data
@TableName("t_ds_command") @TableName("t_ds_command")
public class Command { public class Command {
@ -265,6 +264,79 @@ public class Command {
this.workerGroupId = workerGroupId; this.workerGroupId = workerGroupId;
} }
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Command command = (Command) o;
if (id != command.id) {
return false;
}
if (processDefinitionId != command.processDefinitionId) {
return false;
}
if (executorId != command.executorId) {
return false;
}
if (workerGroupId != command.workerGroupId) {
return false;
}
if (commandType != command.commandType) {
return false;
}
if (commandParam != null ? !commandParam.equals(command.commandParam) : command.commandParam != null) {
return false;
}
if (taskDependType != command.taskDependType) {
return false;
}
if (failureStrategy != command.failureStrategy) {
return false;
}
if (warningType != command.warningType) {
return false;
}
if (warningGroupId != null ? !warningGroupId.equals(command.warningGroupId) : command.warningGroupId != null) {
return false;
}
if (scheduleTime != null ? !scheduleTime.equals(command.scheduleTime) : command.scheduleTime != null) {
return false;
}
if (startTime != null ? !startTime.equals(command.startTime) : command.startTime != null) {
return false;
}
if (processInstancePriority != command.processInstancePriority) {
return false;
}
return !(updateTime != null ? !updateTime.equals(command.updateTime) : command.updateTime != null);
}
@Override
public int hashCode() {
int result = id;
result = 31 * result + (commandType != null ? commandType.hashCode() : 0);
result = 31 * result + processDefinitionId;
result = 31 * result + executorId;
result = 31 * result + (commandParam != null ? commandParam.hashCode() : 0);
result = 31 * result + (taskDependType != null ? taskDependType.hashCode() : 0);
result = 31 * result + (failureStrategy != null ? failureStrategy.hashCode() : 0);
result = 31 * result + (warningType != null ? warningType.hashCode() : 0);
result = 31 * result + (warningGroupId != null ? warningGroupId.hashCode() : 0);
result = 31 * result + (scheduleTime != null ? scheduleTime.hashCode() : 0);
result = 31 * result + (startTime != null ? startTime.hashCode() : 0);
result = 31 * result + (processInstancePriority != null ? processInstancePriority.hashCode() : 0);
result = 31 * result + (updateTime != null ? updateTime.hashCode() : 0);
result = 31 * result + workerGroupId;
return result;
}
@Override @Override
public String toString() { public String toString() {
return "Command{" + return "Command{" +

40
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java

@ -33,13 +33,6 @@ public class CommandCount {
private int count; private int count;
@Override
public String toString(){
return "command count:" +
" commandType: "+ commandType.toString() +
" count: "+ count;
}
public CommandType getCommandType() { public CommandType getCommandType() {
return commandType; return commandType;
} }
@ -55,4 +48,37 @@ public class CommandCount {
public void setCount(int count) { public void setCount(int count) {
this.count = count; this.count = count;
} }
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
CommandCount that = (CommandCount) o;
if (count != that.count) {
return false;
}
return commandType == that.commandType;
}
@Override
public int hashCode() {
int result = commandType != null ? commandType.hashCode() : 0;
result = 31 * result + count;
return result;
}
@Override
public String toString() {
return "CommandCount{" +
"commandType=" + commandType +
", count=" + count +
'}';
}
} }

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java

@ -49,7 +49,7 @@ public class PropertyUtils {
* init * init
*/ */
private void init(){ private void init(){
String[] propertyFiles = new String[]{Constants.DAO_PROPERTIES_PATH}; String[] propertyFiles = new String[]{Constants.APPLICATION_PROPERTIES};
for (String fileName : propertyFiles) { for (String fileName : propertyFiles) {
InputStream fis = null; InputStream fis = null;
try { try {

0
dolphinscheduler-dao/src/main/resources/application-dao.properties → dolphinscheduler-dao/src/main/resources/application.properties

156
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java

@ -19,44 +19,55 @@ package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.AlertType;
import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.Alert;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
import java.util.Date; import java.util.*;
import java.util.List;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
/**
* alert mapper test
*/
@RunWith(SpringRunner.class) @RunWith(SpringRunner.class)
@SpringBootTest @SpringBootTest
@Transactional
@Rollback(true)
public class AlertMapperTest { public class AlertMapperTest {
@Autowired @Autowired
AlertMapper alertMapper; private AlertMapper alertMapper;
/** /**
* insert * test insert
* @return Alert * @return
*/ */
private Alert insertOne(){ @Test
//insertOne public void testInsert(){
Alert alert = new Alert(); Alert expectedAlert = createAlert();
alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]"); assertNotNull(expectedAlert.getId());
alert.setLog("success"); assertThat(expectedAlert.getId(), greaterThan(0));
alert.setReceivers("xx@aa.com"); }
alert.setAlertType(AlertType.EMAIL);
alert.setShowType(ShowType.TABLE);
alert.setAlertGroupId(1); /**
alert.setAlertStatus(AlertStatus.EXECUTION_SUCCESS); * test select by id
alert.setCreateTime(new Date()); * @return
alert.setUpdateTime(new Date()); */
alertMapper.insert(alert); @Test
return alert; public void testSelectById(){
Alert expectedAlert = createAlert();
Alert actualAlert = alertMapper.selectById(expectedAlert.getId());
assertEquals(expectedAlert, actualAlert);
} }
/** /**
@ -64,13 +75,18 @@ public class AlertMapperTest {
*/ */
@Test @Test
public void testUpdate(){ public void testUpdate(){
//insertOne
Alert alert = insertOne(); Alert expectedAlert = createAlert();
//update
alert.setTitle("hello"); expectedAlert.setAlertStatus(AlertStatus.EXECUTION_FAILURE);
int update = alertMapper.updateById(alert); expectedAlert.setLog("error");
Assert.assertEquals(update, 1); expectedAlert.setUpdateTime(DateUtils.getCurrentDate());
alertMapper.deleteById(alert.getId());
alertMapper.updateById(expectedAlert);
Alert actualAlert = alertMapper.selectById(expectedAlert.getId());
assertEquals(expectedAlert, actualAlert);
} }
/** /**
@ -78,33 +94,83 @@ public class AlertMapperTest {
*/ */
@Test @Test
public void testDelete(){ public void testDelete(){
Alert expectedAlert = createAlert();
alertMapper.deleteById(expectedAlert.getId());
Alert actualAlert = alertMapper.selectById(expectedAlert.getId());
Alert alert = insertOne(); assertNull(actualAlert);
int delete = alertMapper.deleteById(alert.getId());
Assert.assertEquals(delete, 1);
} }
/** /**
* test query * test list alert by status
*/ */
@Test @Test
public void testQuery() { public void testListAlertByStatus() {
Alert alert = insertOne(); Integer count = 10;
//query AlertStatus waitExecution = AlertStatus.WAIT_EXECUTION;
List<Alert> alerts = alertMapper.selectList(null);
Assert.assertNotEquals(alerts.size(), 0); Map<Integer,Alert> expectedAlertMap = createAlertMap(count, waitExecution);
alertMapper.deleteById(alert.getId());
List<Alert> actualAlerts = alertMapper.listAlertByStatus(waitExecution);
for (Alert actualAlert : actualAlerts){
Alert expectedAlert = expectedAlertMap.get(actualAlert.getId());
if (expectedAlert != null){
assertEquals(expectedAlert,actualAlert);
}
}
} }
/** /**
* test list alert by status * create alert map
* @param count alert count
* @param alertStatus alert status
* @return alert map
*/ */
@Test private Map<Integer,Alert> createAlertMap(Integer count,AlertStatus alertStatus){
public void testListAlertByStatus() { Map<Integer,Alert> alertMap = new HashMap<>();
Alert alert = insertOne();
//query for (int i = 0 ; i < count ;i++){
List<Alert> alerts = alertMapper.listAlertByStatus(AlertStatus.EXECUTION_SUCCESS); Alert alert = createAlert(alertStatus);
Assert.assertNotEquals(alerts.size(), 0); alertMap.put(alert.getId(),alert);
alertMapper.deleteById(alert.getId()); }
return alertMap;
}
/**
* create alert
* @return alert
* @throws Exception
*/
private Alert createAlert(){
return createAlert(AlertStatus.WAIT_EXECUTION);
}
/**
* create alert
* @param alertStatus alert status
* @return alert
*/
private Alert createAlert(AlertStatus alertStatus){
Alert alert = new Alert();
alert.setShowType(ShowType.TABLE);
alert.setTitle("test alert");
alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]");
alert.setAlertType(AlertType.EMAIL);
alert.setAlertStatus(alertStatus);
alert.setLog("success");
alert.setReceivers("aa@aa.com");
alert.setReceiversCc("bb@aa.com");
alert.setCreateTime(DateUtils.getCurrentDate());
alert.setUpdateTime(DateUtils.getCurrentDate());
alertMapper.insert(alert);
return alert;
} }
} }

264
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.dao.mapper; package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.CommandCount; import org.apache.dolphinscheduler.dao.entity.CommandCount;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
@ -25,13 +26,25 @@ import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
import java.util.Date; import java.util.Date;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
/**
* command mapper test
*/
@RunWith(SpringRunner.class) @RunWith(SpringRunner.class)
@SpringBootTest @SpringBootTest
@Transactional
@Rollback(true)
public class CommandMapperTest { public class CommandMapperTest {
@ -41,71 +54,85 @@ public class CommandMapperTest {
@Autowired @Autowired
ProcessDefinitionMapper processDefinitionMapper; ProcessDefinitionMapper processDefinitionMapper;
/** /**
* insert * test insert
* @return Command
*/ */
private Command insertOne(){ @Test
//insertOne public void testInsert(){
Command command = new Command(); Command command = createCommand();
command.setCommandType(CommandType.START_PROCESS); assertNotNull(command.getId());
command.setProcessDefinitionId(1); assertThat(command.getId(),greaterThan(0));
command.setExecutorId(4);
command.setProcessInstancePriority(Priority.MEDIUM);
command.setFailureStrategy(FailureStrategy.CONTINUE);
command.setWorkerGroupId(-1);
command.setWarningGroupId(1);
command.setUpdateTime(new Date());
commandMapper.insert(command);
return command;
} }
/** /**
* test update * test select by id
*/ */
@Test @Test
public void testUpdate(){ public void testSelectById() {
//insertOne Command expectedCommand = createCommand();
Command command = insertOne(); //query
//update Command actualCommand = commandMapper.selectById(expectedCommand.getId());
command.setStartTime(new Date());
int update = commandMapper.updateById(command); assertEquals(expectedCommand, actualCommand);
Assert.assertEquals(update, 1);
commandMapper.deleteById(command.getId());
} }
/** /**
* test delete * test update
*/ */
@Test @Test
public void testDelete(){ public void testUpdate(){
Command expectedCommand = createCommand();
// update the command time if current command if recover from waiting
expectedCommand.setUpdateTime(DateUtils.getCurrentDate());
commandMapper.updateById(expectedCommand);
Command actualCommand = commandMapper.selectById(expectedCommand.getId());
assertEquals(expectedCommand,actualCommand);
Command Command = insertOne();
int delete = commandMapper.deleteById(Command.getId());
Assert.assertEquals(delete, 1);
} }
/** /**
* test query * test delete
*/ */
@Test @Test
public void testQuery() { public void testDelete(){
Command command = insertOne(); Command expectedCommand = createCommand();
//query
List<Command> commands = commandMapper.selectList(null); commandMapper.deleteById(expectedCommand.getId());
Assert.assertNotEquals(commands.size(), 0);
commandMapper.deleteById(command.getId()); Command actualCommand = commandMapper.selectById(expectedCommand.getId());
assertNull(actualCommand);
} }
/** /**
* test query all * test query all
*/ */
@Test @Test
public void testGetAll() { public void testGetAll() {
Command command = insertOne(); Integer count = 10;
List<Command> commands = commandMapper.selectList(null);
Assert.assertNotEquals(commands.size(), 0); Map<Integer, Command> commandMap = createCommandMap(count);
commandMapper.deleteById(command.getId());
List<Command> actualCommands = commandMapper.selectList(null);
assertThat(actualCommands.size(), greaterThanOrEqualTo(count));
for (Command actualCommand : actualCommands){
Command expectedCommand = commandMap.get(actualCommand.getId());
if (expectedCommand != null){
assertEquals(expectedCommand,actualCommand);
}
}
} }
/** /**
@ -113,28 +140,14 @@ public class CommandMapperTest {
*/ */
@Test @Test
public void testGetOneToRun() { public void testGetOneToRun() {
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setReleaseState(ReleaseState.ONLINE);
processDefinition.setName("ut test");
processDefinition.setProjectId(1);
processDefinition.setFlag(Flag.YES);
processDefinitionMapper.insert(processDefinition);
Command command = new Command(); ProcessDefinition processDefinition = createProcessDefinition();
command.setCommandType(CommandType.START_PROCESS);
command.setProcessDefinitionId(processDefinition.getId()); Command expectedCommand = createCommand(CommandType.START_PROCESS,processDefinition.getId());
command.setExecutorId(4);
command.setProcessInstancePriority(Priority.MEDIUM);
command.setFailureStrategy(FailureStrategy.CONTINUE);
command.setWorkerGroupId(-1);
command.setWarningGroupId(1);
command.setUpdateTime(new Date());
commandMapper.insert(command);
Command command2 = commandMapper.getOneToRun(); Command actualCommand = commandMapper.getOneToRun();
Assert.assertNotEquals(command2, null);
commandMapper.deleteById(command.getId()); assertEquals(expectedCommand, actualCommand);
processDefinitionMapper.deleteById(processDefinition.getId());
} }
/** /**
@ -142,35 +155,122 @@ public class CommandMapperTest {
*/ */
@Test @Test
public void testCountCommandState() { public void testCountCommandState() {
Command command = insertOne(); Integer count = 10;
ProcessDefinition processDefinition = createProcessDefinition();
CommandCount expectedCommandCount = createCommandMap(count, CommandType.START_PROCESS, processDefinition.getId());
Integer[] projectIdArray = {processDefinition.getProjectId()};
Date startTime = DateUtils.stringToDate("2019-12-29 00:10:00");
Date endTime = DateUtils.stringToDate("2019-12-29 23:59:59");
List<CommandCount> actualCommandCounts = commandMapper.countCommandState(0, startTime, endTime, projectIdArray);
assertThat(actualCommandCounts.size(),greaterThanOrEqualTo(1));
Boolean flag = false;
for (CommandCount actualCommandCount : actualCommandCounts){
if (actualCommandCount.getCommandType().equals(expectedCommandCount.getCommandType())){
assertEquals(expectedCommandCount,actualCommandCount);
flag = true;
}
}
assertTrue(flag);
}
//insertOne
/**
* create command map
* @param count map count
* @param commandType comman type
* @param processDefinitionId process definition id
* @return command map
*/
private CommandCount createCommandMap(
Integer count,
CommandType commandType,
Integer processDefinitionId){
CommandCount commandCount = new CommandCount();
for (int i = 0 ;i < count ;i++){
createCommand(commandType,processDefinitionId);
}
commandCount.setCommandType(commandType);
commandCount.setCount(count);
return commandCount;
}
/**
* create process definition
* @return process definition
*/
private ProcessDefinition createProcessDefinition(){
ProcessDefinition processDefinition = new ProcessDefinition(); ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setName("def 1"); processDefinition.setReleaseState(ReleaseState.ONLINE);
processDefinition.setProjectId(1010); processDefinition.setName("ut test");
processDefinition.setUserId(101); processDefinition.setProjectId(1);
processDefinition.setUpdateTime(new Date()); processDefinition.setFlag(Flag.YES);
processDefinition.setCreateTime(new Date());
processDefinitionMapper.insert(processDefinition); processDefinitionMapper.insert(processDefinition);
command.setProcessDefinitionId(processDefinition.getId()); return processDefinition;
commandMapper.updateById(command); }
/**
* create command map
* @param count map count
* @return command map
*/
private Map<Integer,Command> createCommandMap(Integer count){
Map<Integer,Command> commandMap = new HashMap<>();
for (int i = 0; i < count ;i++){
Command command = createCommand();
commandMap.put(command.getId(),command);
}
return commandMap;
}
List<CommandCount> commandCounts = commandMapper.countCommandState(
4, null, null, new Integer[0]
);
Integer[] projectIdArray = new Integer[2]; /**
projectIdArray[0] = processDefinition.getProjectId(); * create command
projectIdArray[1] = 200; * @return
List<CommandCount> commandCounts2 = commandMapper.countCommandState( */
4, null, null, projectIdArray private Command createCommand(){
); return createCommand(CommandType.START_PROCESS,1);
}
/**
* create command
* @return Command
*/
private Command createCommand(CommandType commandType,Integer processDefinitionId){
Command command = new Command();
command.setCommandType(commandType);
command.setProcessDefinitionId(processDefinitionId);
command.setExecutorId(4);
command.setCommandParam("test command param");
command.setTaskDependType(TaskDependType.TASK_ONLY);
command.setFailureStrategy(FailureStrategy.CONTINUE);
command.setWarningType(WarningType.ALL);
command.setWarningGroupId(1);
command.setScheduleTime(DateUtils.stringToDate("2019-12-29 12:10:00"));
command.setProcessInstancePriority(Priority.MEDIUM);
command.setStartTime(DateUtils.stringToDate("2019-12-29 10:10:00"));
command.setUpdateTime(DateUtils.stringToDate("2019-12-29 10:10:00"));
command.setWorkerGroupId(-1);
commandMapper.insert(command);
commandMapper.deleteById(command.getId()); return command;
processDefinitionMapper.deleteById(processDefinition.getId());
Assert.assertNotEquals(commandCounts.size(), 0);
Assert.assertNotEquals(commandCounts2.size(), 0);
} }
} }

6
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java

@ -152,10 +152,8 @@ public class MasterServer implements IStoppable {
@Override @Override
public void run() { public void run() {
if (zkMasterClient.getActiveMasterNum() <= 1) { if (zkMasterClient.getActiveMasterNum() <= 1) {
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { zkMasterClient.getAlertDao().sendServerStopedAlert(
zkMasterClient.getAlertDao().sendServerStopedAlert( 1, OSUtils.getHost(), "Master-Server");
1, OSUtils.getHost(), "Master-Server");
}
} }
stop("shutdownhook"); stop("shutdownhook");
} }

20
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java

@ -90,15 +90,15 @@ public class AlertManager {
* process instance format * process instance format
*/ */
private static final String PROCESS_INSTANCE_FORMAT = private static final String PROCESS_INSTANCE_FORMAT =
"\"Id:%d\"," + "\"id:%d\"," +
"\"Name:%s\"," + "\"name:%s\"," +
"\"Job type: %s\"," + "\"job type: %s\"," +
"\"State: %s\"," + "\"state: %s\"," +
"\"Recovery:%s\"," + "\"recovery:%s\"," +
"\"Run time: %d\"," + "\"run time: %d\"," +
"\"Start time: %s\"," + "\"start time: %s\"," +
"\"End time: %s\"," + "\"end time: %s\"," +
"\"Host: %s\"" ; "\"host: %s\"" ;
/** /**
* get process instance content * get process instance content
@ -234,7 +234,7 @@ public class AlertManager {
String cmdName = getCommandCnName(processInstance.getCommandType()); String cmdName = getCommandCnName(processInstance.getCommandType());
String success = processInstance.getState().typeIsSuccess() ? "success" :"failed"; String success = processInstance.getState().typeIsSuccess() ? "success" :"failed";
alert.setTitle(cmdName + success); alert.setTitle(cmdName + " " + success);
ShowType showType = processInstance.getState().typeIsSuccess() ? ShowType.TEXT : ShowType.TABLE; ShowType showType = processInstance.getState().typeIsSuccess() ? ShowType.TEXT : ShowType.TABLE;
alert.setShowType(showType); alert.setShowType(showType);
String content = getContentProcessInstance(processInstance, taskInstances); String content = getContentProcessInstance(processInstance, taskInstances);

20
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java

@ -177,9 +177,7 @@ public class WorkerServer implements IStoppable {
public void run() { public void run() {
// worker server exit alert // worker server exit alert
if (zkWorkerClient.getActiveMasterNum() <= 1) { if (zkWorkerClient.getActiveMasterNum() <= 1) {
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server");
alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server");
}
} }
stop("shutdownhook"); stop("shutdownhook");
} }
@ -290,22 +288,20 @@ public class WorkerServer implements IStoppable {
Runnable killProcessThread = new Runnable() { Runnable killProcessThread = new Runnable() {
@Override @Override
public void run() { public void run() {
Set<String> taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL);
while (Stopper.isRunning()){ while (Stopper.isRunning()){
try { Set<String> taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL);
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
} catch (InterruptedException e) {
logger.error("interrupted exception",e);
}
// if set is null , return
if (CollectionUtils.isNotEmpty(taskInfoSet)){ if (CollectionUtils.isNotEmpty(taskInfoSet)){
for (String taskInfo : taskInfoSet){ for (String taskInfo : taskInfoSet){
killTask(taskInfo, processDao); killTask(taskInfo, processDao);
removeKillInfoFromQueue(taskInfo); removeKillInfoFromQueue(taskInfo);
} }
} }
try {
taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL); Thread.sleep(Constants.SLEEP_TIME_MILLIS);
} catch (InterruptedException e) {
logger.error("interrupted exception",e);
Thread.currentThread().interrupt();
}
} }
} }
}; };

6
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java

@ -247,10 +247,8 @@ public class ZKMasterClient extends AbstractZKClient {
*/ */
private void alertServerDown(String serverHost, ZKNodeType zkNodeType) { private void alertServerDown(String serverHost, ZKNodeType zkNodeType) {
String serverType = zkNodeType.toString(); String serverType = zkNodeType.toString();
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { alertDao.sendServerStopedAlert(1, serverHost, serverType);
alertDao.sendServerStopedAlert(1, serverHost, serverType);
}
} }
/** /**

18
dolphinscheduler-server/src/main/resources/application-master.properties

@ -1,18 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
logging.config=classpath:master_logback.xml

18
dolphinscheduler-server/src/main/resources/application-worker.properties

@ -1,18 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
logging.config=classpath:worker_logback.xml

8
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue

@ -257,8 +257,8 @@
_rtParam () { _rtParam () {
return { return {
taskInstId: this.stateId || this.logId, taskInstId: this.stateId || this.logId,
skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '0000' : 0}`), skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '000' : 0}`),
limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}0000`) limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}000`)
} }
} }
}, },
@ -316,7 +316,7 @@
} }
.refresh-log { .refresh-log {
>i { >i {
font-size: 24px; font-size: 20px;
vertical-align: middle; vertical-align: middle;
transform: scale(1); transform: scale(1);
} }
@ -353,7 +353,7 @@
font-weight: bold; font-weight: bold;
resize:none; resize:none;
line-height: 1.6; line-height: 1.6;
padding: 6px; padding: 0px;
} }
} }
} }

124
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue

@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<template>
<div class="script-model">
<m-list-box>
<div slot="content">
<div class="from-mirror1">
<textarea
id="code-shell-mirror1"
name="code-shell-mirror1"
style="opacity: 0">
</textarea>
</div>
</div>
</m-list-box>
<a class="ans-modal-box-close">
<i class="ans-icon-min" @click="closeModal"></i>
</a>
</div>
</template>
<script>
import _ from 'lodash'
import i18n from '@/module/i18n'
import mListBox from './listBox'
import disabledState from '@/module/mixin/disabledState'
import codemirror from '@/conf/home/pages/resource/pages/file/pages/_source/codemirror'
let editor
export default {
name: 'shell',
data () {
return {
// script
rawScript: '',
}
},
mixins: [disabledState],
props: {
item: String
},
methods: {
/**
* Processing code highlighting
*/
_handlerEditor () {
// editor
let self =this
editor = codemirror('code-shell-mirror1', {
mode: 'shell',
readOnly: this.isDetails
})
editor.on("change",function(){
self.$emit('getSriptBoxValue',editor.getValue())
})
this.keypress = () => {
if (!editor.getOption('readOnly')) {
editor.showHint({
completeSingle: false
})
}
}
// Monitor keyboard
editor.on('keypress', this.keypress)
editor.setValue(this.rawScript)
return editor
},
closeModal() {
let self = this
self.$emit('closeAble')
}
},
watch: {},
created () {
let o = this.item
// Non-null objects represent backfill
if (!_.isEmpty(o)) {
this.rawScript = o
}
},
mounted () {
setTimeout(() => {
this._handlerEditor()
}, 200)
},
destroyed () {
if (editor) {
editor.toTextArea() // Uninstall
editor.off($('.code-shell-mirror1'), 'keypress', this.keypress)
}
},
components: { mListBox }
}
</script>
<style lang="scss" rel="stylesheet/scss" scope>
.script-model {
width:100%;
}
.from-mirror1 {
.CodeMirror {
min-height: 600px;
max-height: 700px;
}
}
</style>

64
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue

@ -21,10 +21,13 @@
<div slot="content"> <div slot="content">
<div class="from-mirror"> <div class="from-mirror">
<textarea <textarea
id="code-shell-mirror" id="code-shell-mirror"
name="code-shell-mirror" name="code-shell-mirror"
style="opacity: 0"> style="opacity: 0">
</textarea> </textarea>
<a class="ans-modal-box-max">
<i class="ans-icon-max" @click="setEditorVal"></i>
</a>
</div> </div>
</div> </div>
</m-list-box> </m-list-box>
@ -55,6 +58,7 @@
import _ from 'lodash' import _ from 'lodash'
import i18n from '@/module/i18n' import i18n from '@/module/i18n'
import mListBox from './_source/listBox' import mListBox from './_source/listBox'
import mScriptBox from './_source/scriptBox'
import mResources from './_source/resources' import mResources from './_source/resources'
import mLocalParams from './_source/localParams' import mLocalParams from './_source/localParams'
import disabledState from '@/module/mixin/disabledState' import disabledState from '@/module/mixin/disabledState'
@ -85,8 +89,37 @@
_onLocalParams (a) { _onLocalParams (a) {
this.localParams = a this.localParams = a
}, },
setEditorVal() {
let self = this
let modal = self.$modal.dialog({
className: 'scriptModal',
closable: false,
showMask: true,
maskClosable: true,
onClose: function() {
},
render (h) {
return h(mScriptBox, {
on: {
getSriptBoxValue (val) {
editor.setValue(val)
},
closeAble () {
// this.$modal.destroy()
modal.remove()
}
},
props: {
item: editor.getValue()
}
})
}
})
},
/** /**
* return resourceList * return resourceList
*
*/ */
_onResourcesData (a) { _onResourcesData (a) {
this.resourceList = a this.resourceList = a
@ -109,7 +142,6 @@
if (!this.$refs.refLocalParams._verifProp()) { if (!this.$refs.refLocalParams._verifProp()) {
return false return false
} }
// storage // storage
this.$emit('on-params', { this.$emit('on-params', {
resourceList: this.resourceList, resourceList: this.resourceList,
@ -138,7 +170,6 @@
// Monitor keyboard // Monitor keyboard
editor.on('keypress', this.keypress) editor.on('keypress', this.keypress)
editor.setValue(this.rawScript) editor.setValue(this.rawScript)
return editor return editor
@ -176,6 +207,27 @@
editor.off($('.code-shell-mirror'), 'keypress', this.keypress) editor.off($('.code-shell-mirror'), 'keypress', this.keypress)
} }
}, },
components: { mLocalParams, mListBox, mResources } components: { mLocalParams, mListBox, mResources, mScriptBox }
} }
</script> </script>
<style lang="scss" rel="stylesheet/scss" scope>
.scriptModal {
.ans-modal-box-content-wrapper {
width: 90%;
.ans-modal-box-close {
right: -12px;
top: -16px;
color: #fff;
}
}
}
.ans-modal-box-close {
z-index: 100;
}
.ans-modal-box-max {
position: absolute;
right: -12px;
top: -16px;
}
</style>

8
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue

@ -19,6 +19,12 @@
<div class="title-box"> <div class="title-box">
<span>{{$t('Please set the parameters before starting')}}</span> <span>{{$t('Please set the parameters before starting')}}</span>
</div> </div>
<div class="clearfix list">
<div class="text">
{{$t('Process Name')}}
</div>
<div style="line-height: 32px;">{{workflowName}}</div>
</div>
<div class="clearfix list"> <div class="clearfix list">
<div class="text"> <div class="text">
{{$t('Failure Strategy')}} {{$t('Failure Strategy')}}
@ -174,6 +180,7 @@
processDefinitionId: 0, processDefinitionId: 0,
failureStrategy: 'CONTINUE', failureStrategy: 'CONTINUE',
warningTypeList: warningTypeList, warningTypeList: warningTypeList,
workflowName: '',
warningType: '', warningType: '',
notifyGroupList: [], notifyGroupList: [],
warningGroupId: '', warningGroupId: '',
@ -276,6 +283,7 @@
this.warningGroupId = '' this.warningGroupId = ''
}) })
}) })
this.workflowName = this.item.name
}, },
computed: {}, computed: {},
components: { mEmail, mPriority, mWorkerGroups } components: { mEmail, mPriority, mWorkerGroups }

16
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue

@ -313,7 +313,23 @@
this.crontab = this.item.crontab this.crontab = this.item.crontab
} }
if(this.type == 'timing') { if(this.type == 'timing') {
let date = new Date()
let year = date.getFullYear()
let month = date.getMonth() + 1
let day = date.getDate()
if (month < 10) {
month = "0" + month;
}
if (day < 10) {
day = "0" + day;
}
let startDate = year + "-" + month + "-" + day + ' ' + '00:00:00'
let endDate = (year+100) + "-" + month + "-" + day + ' ' + '00:00:00'
let times = []
times[0] = startDate
times[1] = endDate
this.crontab = '0 0 * * * ? *' this.crontab = '0 0 * * * ? *'
this.scheduleTime = times
} }
this.receivers = _.cloneDeep(this.receiversD) this.receivers = _.cloneDeep(this.receiversD)
this.receiversCc = _.cloneDeep(this.receiversCcD) this.receiversCc = _.cloneDeep(this.receiversCcD)

2
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js

@ -27,7 +27,7 @@ let pie = {
avoidLabelOverlap: true, // Whether to prevent the label overlap policy avoidLabelOverlap: true, // Whether to prevent the label overlap policy
hoverAnimation: true, // Whether to enable hover to enlarge the animation on the sector. hoverAnimation: true, // Whether to enable hover to enlarge the animation on the sector.
radius: ['30%', '60%'], radius: ['30%', '60%'],
center: ['50%', '50%'], center: ['53%', '60%'],
label: { label: {
align: 'left', align: 'left',
normal: { normal: {

2
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue

@ -29,7 +29,7 @@
<th>{{$t('Number')}}</th> <th>{{$t('Number')}}</th>
<th>{{$t('State')}}</th> <th>{{$t('State')}}</th>
</tr> </tr>
<tr v-for="(item,$index) in processStateList"> <tr v-for="(item,$index) in processStateList" :key="$index">
<td><span>{{$index+1}}</span></td> <td><span>{{$index+1}}</span></td>
<td><span><a href="javascript:" @click="searchParams.projectId && _goProcess(item.key)" :class="searchParams.projectId ?'links':''">{{item.value}}</a></span></td> <td><span><a href="javascript:" @click="searchParams.projectId && _goProcess(item.key)" :class="searchParams.projectId ?'links':''">{{item.value}}</a></span></td>
<td><span class="ellipsis" style="width: 98%;" :title="item.key">{{item.key}}</span></td> <td><span class="ellipsis" style="width: 98%;" :title="item.key">{{item.key}}</span></td>

109
install.sh

@ -218,30 +218,6 @@ keytabPath="$installPath/conf/hdfs.headless.keytab"
# zk root directory # zk root directory
zkRoot="/dolphinscheduler" zkRoot="/dolphinscheduler"
# used to record the zk directory of the hanging machine
zkDeadServers="$zkRoot/dead-servers"
# masters directory
zkMasters="$zkRoot/masters"
# workers directory
zkWorkers="$zkRoot/workers"
# zk master distributed lock
mastersLock="$zkRoot/lock/masters"
# zk worker distributed lock
workersLock="$zkRoot/lock/workers"
# zk master fault-tolerant distributed lock
mastersFailover="$zkRoot/lock/failover/masters"
# zk worker fault-tolerant distributed lock
workersFailover="$zkRoot/lock/failover/workers"
# zk master start fault tolerant distributed lock
mastersStartupFailover="$zkRoot/lock/failover/startup-masters"
# zk session timeout # zk session timeout
zkSessionTimeout="300" zkSessionTimeout="300"
@ -321,11 +297,10 @@ apiMaxHttpPostSize="5000000"
# 1,replace file # 1,replace file
echo "1,replace file" echo "1,replace file"
if [ $dbtype == "mysql" ];then if [ $dbtype == "mysql" ];then
sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application-dao.properties sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application.properties
sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties
sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties
@ -335,10 +310,10 @@ if [ $dbtype == "mysql" ];then
fi fi
if [ $dbtype == "postgresql" ];then if [ $dbtype == "postgresql" ];then
sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application-dao.properties sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application.properties
sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties
sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties
@ -349,46 +324,34 @@ fi
sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.properties
sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties
sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties
sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties
sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties
sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties
sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common/common.properties sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common.properties
sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common/common.properties sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common.properties
sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common.properties
sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties
sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common.properties
sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common/common.properties sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common.properties
sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common/common.properties sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common.properties
sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common.properties
sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common.properties
sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common/common.properties sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common.properties
sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common/common.properties sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties
sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common/common.properties sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties
sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common/common.properties sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.dead.servers.*#zookeeper.dolphinscheduler.dead.servers=${zkDeadServers}#g" conf/zookeeper.properties sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.masters.*#zookeeper.dolphinscheduler.masters=${zkMasters}#g" conf/zookeeper.properties sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.workers.*#zookeeper.dolphinscheduler.workers=${zkWorkers}#g" conf/zookeeper.properties sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.masters.*#zookeeper.dolphinscheduler.lock.masters=${mastersLock}#g" conf/zookeeper.properties sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.workers.*#zookeeper.dolphinscheduler.lock.workers=${workersLock}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.masters.*#zookeeper.dolphinscheduler.lock.failover.masters=${mastersFailover}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.workers.*#zookeeper.dolphinscheduler.lock.failover.workers=${workersFailover}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.startup.masters.*#zookeeper.dolphinscheduler.lock.failover.startup.masters=${mastersStartupFailover}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/zookeeper.properties
sed -i ${txt} "s#server.port.*#server.port=${masterPort}#g" conf/application-master.properties
sed -i ${txt} "s#server.port.*#server.port=${workerPort}#g" conf/application-worker.properties
sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties
sed -i ${txt} "s#server.servlet.session.timeout.*#server.servlet.session.timeout=${apiServerSessionTimeout}#g" conf/application-api.properties sed -i ${txt} "s#server.servlet.session.timeout.*#server.servlet.session.timeout=${apiServerSessionTimeout}#g" conf/application-api.properties

11
pom.xml

@ -666,6 +666,15 @@
<include>**/api/utils/CheckUtilsTest.java</include> <include>**/api/utils/CheckUtilsTest.java</include>
<include>**/api/utils/FileUtilsTest.java</include> <include>**/api/utils/FileUtilsTest.java</include>
<include>**/api/enums/*.java</include> <include>**/api/enums/*.java</include>
<include>**/api/service/AccessTokenServiceTest.java</include>
<include>**/api/service/QueueServiceTest.java</include>
<include>**/api/service/MonitorServiceTest.java</include>
<include>**/api/service/SessionServiceTest.java</include>
<include>**/api/service/UsersServiceTest.java</include>
<include>**/api/service/TenantServiceTest.java</include>
<include>**/api/service/WorkerGroupServiceTest.java</include>
<include>**/api/service/AlertGroupServiceTest.java</include>
<include>**/api/service/ProjectServiceTest.java</include>
<include>**/alert/utils/ExcelUtilsTest.java</include> <include>**/alert/utils/ExcelUtilsTest.java</include>
<include>**/alert/utils/FuncUtilsTest.java</include> <include>**/alert/utils/FuncUtilsTest.java</include>
<include>**/alert/utils/JSONUtilsTest.java</include> <include>**/alert/utils/JSONUtilsTest.java</include>
@ -674,6 +683,8 @@
<include>**/server/utils/FlinkArgsUtilsTest.java</include> <include>**/server/utils/FlinkArgsUtilsTest.java</include>
<include>**/dao/mapper/AccessTokenMapperTest.java</include> <include>**/dao/mapper/AccessTokenMapperTest.java</include>
<include>**/dao/mapper/AlertGroupMapperTest.java</include> <include>**/dao/mapper/AlertGroupMapperTest.java</include>
<include>**/dao/mapper/AlertMapperTest.java</include>
<include>**/dao/mapper/CommandMapperTest.java</include>
</includes> </includes>
<!-- <skip>true</skip> --> <!-- <skip>true</skip> -->
</configuration> </configuration>

8
script/dolphinscheduler-daemon.sh

@ -57,13 +57,13 @@ pid=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command.pid
cd $DOLPHINSCHEDULER_HOME cd $DOLPHINSCHEDULER_HOME
if [ "$command" = "api-server" ]; then if [ "$command" = "api-server" ]; then
LOG_FILE="-Dspring.profiles.active=api" LOG_FILE="-Dlogging.config=classpath:apiserver_logback.xml -Dspring.profiles.active=api"
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
elif [ "$command" = "master-server" ]; then elif [ "$command" = "master-server" ]; then
LOG_FILE="-Dspring.profiles.active=master -Ddruid.mysql.usePingMethod=false" LOG_FILE="-Dlogging.config=classpath:master_logback.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.master.MasterServer CLASS=org.apache.dolphinscheduler.server.master.MasterServer
elif [ "$command" = "worker-server" ]; then elif [ "$command" = "worker-server" ]; then
LOG_FILE="-Dspring.profiles.active=worker -Ddruid.mysql.usePingMethod=false" LOG_FILE="-Dlogging.config=classpath:worker_logback.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
elif [ "$command" = "alert-server" ]; then elif [ "$command" = "alert-server" ]; then
LOG_FILE="-Dlogback.configurationFile=conf/alert_logback.xml" LOG_FILE="-Dlogback.configurationFile=conf/alert_logback.xml"
@ -71,7 +71,7 @@ elif [ "$command" = "alert-server" ]; then
elif [ "$command" = "logger-server" ]; then elif [ "$command" = "logger-server" ]; then
CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer
elif [ "$command" = "combined-server" ]; then elif [ "$command" = "combined-server" ]; then
LOG_FILE="-Dspring.profiles.active=combined" LOG_FILE="-Dlogging.config=classpath:combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true"
CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer
else else
echo "Error: No command named \`$command' was found." echo "Error: No command named \`$command' was found."

Loading…
Cancel
Save