dailidong 5 years ago
parent
commit
04f8fe898c
  1. 51
      .github/workflows/SonarCloud.yml
  2. 3
      README.md
  3. 2
      README_zh_CN.md
  4. 11
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
  5. 53
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
  6. 18
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java
  7. 49
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java
  8. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
  9. 71
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java
  10. 74
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java
  11. 40
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java
  12. 156
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java
  13. 264
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java
  14. 6
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
  15. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
  16. 6
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java
  17. 8
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue
  18. 124
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue
  19. 64
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue
  20. 16
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
  21. 2
      pom.xml

51
.github/workflows/SonarCloud.yml

@ -0,0 +1,51 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: SonarCloud
on: [push, pull_request]
jobs:
sonarCloudTrigger:
name: SonarCloud Trigger
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-java@v1
with:
java-version: 8
- uses: actions/cache@v1
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube
restore-keys: |
${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube
${{ runner.os }}-maven-
- name: Maven clean
run: mvn clean
- name: Run SonarCloud analyse
run: >
mvn clean --batch-mode
org.jacoco:jacoco-maven-plugin:prepare-agent
verify
org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
-Dmaven.test.skip=true
-Dsonar.host.url=https://sonarcloud.io
-Dsonar.organization=apache
-Dsonar.projectKey=apache-dolphinscheduler
-Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

3
README.md

@ -4,7 +4,8 @@ Dolphin Scheduler Official Website
[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler)
[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev)
![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
> Dolphin Scheduler for Big Data

2
README_zh_CN.md

@ -4,7 +4,7 @@ Dolphin Scheduler Official Website
[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler)
[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev)
![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
> Dolphin Scheduler for Big Data

11
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java

@ -39,19 +39,15 @@ public class AlertServer {
private AlertSender alertSender;
private static volatile AlertServer instance;
private static AlertServer instance;
public AlertServer() {
}
public static AlertServer getInstance(){
public synchronized static AlertServer getInstance(){
if (null == instance) {
synchronized (AlertServer.class) {
if(null == instance) {
instance = new AlertServer();
}
}
instance = new AlertServer();
}
return instance;
}
@ -63,6 +59,7 @@ public class AlertServer {
Thread.sleep(Constants.ALERT_SCAN_INTERVEL);
} catch (InterruptedException e) {
logger.error(e.getMessage(),e);
Thread.currentThread().interrupt();
}
List<Alert> alerts = alertDao.listWaitExecutionAlert();
alertSender = new AlertSender(alerts, alertDao);

53
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java

@ -86,20 +86,24 @@ public class EnterpriseWeChatUtils {
String resp;
CloseableHttpClient httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace);
CloseableHttpResponse response = httpClient.execute(httpGet);
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, Constants.UTF_8);
EntityUtils.consume(entity);
HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace);
CloseableHttpResponse response = httpClient.execute(httpGet);
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, Constants.UTF_8);
EntityUtils.consume(entity);
} finally {
response.close();
}
Map<String, Object> map = JSON.parseObject(resp,
new TypeToken<Map<String, Object>>() {
}.getType());
return map.get("access_token").toString();
} finally {
response.close();
httpClient.close();
}
Map<String, Object> map = JSON.parseObject(resp,
new TypeToken<Map<String, Object>>() {
}.getType());
return map.get("access_token").toString();
}
/**
@ -167,20 +171,25 @@ public class EnterpriseWeChatUtils {
public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException {
String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token);
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace);
httpPost.setEntity(new StringEntity(data, charset));
CloseableHttpResponse response = httpclient.execute(httpPost);
String resp;
CloseableHttpClient httpClient = HttpClients.createDefault();
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, charset);
EntityUtils.consume(entity);
HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace);
httpPost.setEntity(new StringEntity(data, charset));
CloseableHttpResponse response = httpClient.execute(httpPost);
String resp;
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, charset);
EntityUtils.consume(entity);
} finally {
response.close();
}
logger.info("Enterprise WeChat send [{}], param:{}, resp:{}",
enterpriseWeChatPushUrl, data, resp);
return resp;
} finally {
response.close();
httpClient.close();
}
logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", enterpriseWeChatPushUrl, data, resp);
return resp;
}
/**

18
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java

@ -115,12 +115,12 @@ public class QueueService extends BaseService {
}
if (StringUtils.isEmpty(queue)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue);
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue");
return result;
}
if (StringUtils.isEmpty(queueName)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName);
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName");
return result;
}
@ -163,6 +163,16 @@ public class QueueService extends BaseService {
return result;
}
if (StringUtils.isEmpty(queue)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue");
return result;
}
if (StringUtils.isEmpty(queueName)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName");
return result;
}
Queue queueObj = queueMapper.selectById(id);
if (queueObj == null) {
putMsg(result, Status.QUEUE_NOT_EXIST, id);
@ -222,12 +232,12 @@ public class QueueService extends BaseService {
Result result = new Result();
if (StringUtils.isEmpty(queue)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue);
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue");
return result;
}
if (StringUtils.isEmpty(queueName)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName);
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName");
return result;
}

49
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java

@ -221,9 +221,13 @@ public class UsersService extends BaseService {
return result;
}
Date now = new Date();
if (StringUtils.isNotEmpty(userName)) {
if (!CheckUtils.checkUserName(userName)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userName);
return result;
}
User tempUser = userMapper.queryByUserNameAccurately(userName);
if (tempUser != null && tempUser.getId() != userId) {
putMsg(result, Status.USER_NAME_EXIST);
@ -233,14 +237,30 @@ public class UsersService extends BaseService {
}
if (StringUtils.isNotEmpty(userPassword)) {
if (!CheckUtils.checkPassword(userPassword)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userPassword);
return result;
}
user.setUserPassword(EncryptionUtils.getMd5(userPassword));
}
if (StringUtils.isNotEmpty(email)) {
if (!CheckUtils.checkEmail(email)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,email);
return result;
}
user.setEmail(email);
}
if (StringUtils.isNotEmpty(phone)) {
if (!CheckUtils.checkPhone(phone)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,phone);
return result;
}
user.setPhone(phone);
}
user.setQueue(queue);
user.setPhone(phone);
Date now = new Date();
user.setUpdateTime(now);
//if user switches the tenant, the user's resources need to be copied to the new tenant
@ -318,10 +338,15 @@ public class UsersService extends BaseService {
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
if (!isAdmin(loginUser)) {
putMsg(result, Status.USER_NO_OPERATION_PERM, id);
return result;
}
//check exist
User tempUser = userMapper.selectById(id);
if (tempUser == null) {
putMsg(result, Status.USER_NOT_EXIST, id);
return result;
}
// delete user
User user = userMapper.queryTenantCodeByUserId(id);
@ -357,6 +382,12 @@ public class UsersService extends BaseService {
return result;
}
//check exist
User tempUser = userMapper.selectById(userId);
if (tempUser == null) {
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
//if the selected projectIds are empty, delete all items associated with the user
projectUserMapper.deleteProjectRelation(0, userId);
@ -443,6 +474,11 @@ public class UsersService extends BaseService {
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result;
}
User user = userMapper.selectById(userId);
if(user == null){
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
udfUserMapper.deleteByUserId(userId);
@ -485,6 +521,11 @@ public class UsersService extends BaseService {
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result;
}
User user = userMapper.selectById(userId);
if(user == null){
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
datasourceUserMapper.deleteByUserId(userId);

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java

@ -76,6 +76,11 @@ public class WorkerGroupService extends BaseService {
WorkerGroup workerGroup = null;
if(id != 0){
workerGroup = workerGroupMapper.selectById(id);
//check exist
if (workerGroup == null){
workerGroup = new WorkerGroup();
workerGroup.setCreateTime(now);
}
}else{
workerGroup = new WorkerGroup();
workerGroup.setCreateTime(now);

71
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java

@ -32,7 +32,6 @@ import java.util.Map;
/**
* alert
*/
@Data
@TableName("t_ds_alert")
public class Alert {
@ -217,6 +216,72 @@ public class Alert {
this.updateTime = updateTime;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Alert alert = (Alert) o;
if (id != alert.id) {
return false;
}
if (alertGroupId != alert.alertGroupId) {
return false;
}
if (!title.equals(alert.title)) {
return false;
}
if (showType != alert.showType) {
return false;
}
if (!content.equals(alert.content)) {
return false;
}
if (alertType != alert.alertType) {
return false;
}
if (alertStatus != alert.alertStatus) {
return false;
}
if (!log.equals(alert.log)) {
return false;
}
if (!receivers.equals(alert.receivers)) {
return false;
}
if (!receiversCc.equals(alert.receiversCc)) {
return false;
}
if (!createTime.equals(alert.createTime)) {
return false;
}
return updateTime.equals(alert.updateTime) && info.equals(alert.info);
}
@Override
public int hashCode() {
int result = id;
result = 31 * result + title.hashCode();
result = 31 * result + showType.hashCode();
result = 31 * result + content.hashCode();
result = 31 * result + alertType.hashCode();
result = 31 * result + alertStatus.hashCode();
result = 31 * result + log.hashCode();
result = 31 * result + alertGroupId;
result = 31 * result + receivers.hashCode();
result = 31 * result + receiversCc.hashCode();
result = 31 * result + createTime.hashCode();
result = 31 * result + updateTime.hashCode();
result = 31 * result + info.hashCode();
return result;
}
@Override
public String toString() {
return "Alert{" +
@ -228,10 +293,10 @@ public class Alert {
", alertStatus=" + alertStatus +
", log='" + log + '\'' +
", alertGroupId=" + alertGroupId +
", createTime=" + createTime +
", updateTime=" + updateTime +
", receivers='" + receivers + '\'' +
", receiversCc='" + receiversCc + '\'' +
", createTime=" + createTime +
", updateTime=" + updateTime +
", info=" + info +
'}';
}

74
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java

@ -28,7 +28,6 @@ import java.util.Date;
/**
* command
*/
@Data
@TableName("t_ds_command")
public class Command {
@ -265,6 +264,79 @@ public class Command {
this.workerGroupId = workerGroupId;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Command command = (Command) o;
if (id != command.id) {
return false;
}
if (processDefinitionId != command.processDefinitionId) {
return false;
}
if (executorId != command.executorId) {
return false;
}
if (workerGroupId != command.workerGroupId) {
return false;
}
if (commandType != command.commandType) {
return false;
}
if (commandParam != null ? !commandParam.equals(command.commandParam) : command.commandParam != null) {
return false;
}
if (taskDependType != command.taskDependType) {
return false;
}
if (failureStrategy != command.failureStrategy) {
return false;
}
if (warningType != command.warningType) {
return false;
}
if (warningGroupId != null ? !warningGroupId.equals(command.warningGroupId) : command.warningGroupId != null) {
return false;
}
if (scheduleTime != null ? !scheduleTime.equals(command.scheduleTime) : command.scheduleTime != null) {
return false;
}
if (startTime != null ? !startTime.equals(command.startTime) : command.startTime != null) {
return false;
}
if (processInstancePriority != command.processInstancePriority) {
return false;
}
return !(updateTime != null ? !updateTime.equals(command.updateTime) : command.updateTime != null);
}
@Override
public int hashCode() {
int result = id;
result = 31 * result + (commandType != null ? commandType.hashCode() : 0);
result = 31 * result + processDefinitionId;
result = 31 * result + executorId;
result = 31 * result + (commandParam != null ? commandParam.hashCode() : 0);
result = 31 * result + (taskDependType != null ? taskDependType.hashCode() : 0);
result = 31 * result + (failureStrategy != null ? failureStrategy.hashCode() : 0);
result = 31 * result + (warningType != null ? warningType.hashCode() : 0);
result = 31 * result + (warningGroupId != null ? warningGroupId.hashCode() : 0);
result = 31 * result + (scheduleTime != null ? scheduleTime.hashCode() : 0);
result = 31 * result + (startTime != null ? startTime.hashCode() : 0);
result = 31 * result + (processInstancePriority != null ? processInstancePriority.hashCode() : 0);
result = 31 * result + (updateTime != null ? updateTime.hashCode() : 0);
result = 31 * result + workerGroupId;
return result;
}
@Override
public String toString() {
return "Command{" +

40
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java

@ -33,13 +33,6 @@ public class CommandCount {
private int count;
@Override
public String toString(){
return "command count:" +
" commandType: "+ commandType.toString() +
" count: "+ count;
}
public CommandType getCommandType() {
return commandType;
}
@ -55,4 +48,37 @@ public class CommandCount {
public void setCount(int count) {
this.count = count;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
CommandCount that = (CommandCount) o;
if (count != that.count) {
return false;
}
return commandType == that.commandType;
}
@Override
public int hashCode() {
int result = commandType != null ? commandType.hashCode() : 0;
result = 31 * result + count;
return result;
}
@Override
public String toString() {
return "CommandCount{" +
"commandType=" + commandType +
", count=" + count +
'}';
}
}

156
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java

@ -19,44 +19,55 @@ package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.enums.AlertType;
import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
import java.util.Date;
import java.util.List;
import java.util.*;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
/**
* alert mapper test
*/
@RunWith(SpringRunner.class)
@SpringBootTest
@Transactional
@Rollback(true)
public class AlertMapperTest {
@Autowired
AlertMapper alertMapper;
private AlertMapper alertMapper;
/**
* insert
* @return Alert
* test insert
* @return
*/
private Alert insertOne(){
//insertOne
Alert alert = new Alert();
alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]");
alert.setLog("success");
alert.setReceivers("xx@aa.com");
alert.setAlertType(AlertType.EMAIL);
alert.setShowType(ShowType.TABLE);
alert.setAlertGroupId(1);
alert.setAlertStatus(AlertStatus.EXECUTION_SUCCESS);
alert.setCreateTime(new Date());
alert.setUpdateTime(new Date());
alertMapper.insert(alert);
return alert;
@Test
public void testInsert(){
Alert expectedAlert = createAlert();
assertNotNull(expectedAlert.getId());
assertThat(expectedAlert.getId(), greaterThan(0));
}
/**
* test select by id
* @return
*/
@Test
public void testSelectById(){
Alert expectedAlert = createAlert();
Alert actualAlert = alertMapper.selectById(expectedAlert.getId());
assertEquals(expectedAlert, actualAlert);
}
/**
@ -64,13 +75,18 @@ public class AlertMapperTest {
*/
@Test
public void testUpdate(){
//insertOne
Alert alert = insertOne();
//update
alert.setTitle("hello");
int update = alertMapper.updateById(alert);
Assert.assertEquals(update, 1);
alertMapper.deleteById(alert.getId());
Alert expectedAlert = createAlert();
expectedAlert.setAlertStatus(AlertStatus.EXECUTION_FAILURE);
expectedAlert.setLog("error");
expectedAlert.setUpdateTime(DateUtils.getCurrentDate());
alertMapper.updateById(expectedAlert);
Alert actualAlert = alertMapper.selectById(expectedAlert.getId());
assertEquals(expectedAlert, actualAlert);
}
/**
@ -78,33 +94,83 @@ public class AlertMapperTest {
*/
@Test
public void testDelete(){
Alert expectedAlert = createAlert();
alertMapper.deleteById(expectedAlert.getId());
Alert actualAlert = alertMapper.selectById(expectedAlert.getId());
Alert alert = insertOne();
int delete = alertMapper.deleteById(alert.getId());
Assert.assertEquals(delete, 1);
assertNull(actualAlert);
}
/**
* test query
* test list alert by status
*/
@Test
public void testQuery() {
Alert alert = insertOne();
//query
List<Alert> alerts = alertMapper.selectList(null);
Assert.assertNotEquals(alerts.size(), 0);
alertMapper.deleteById(alert.getId());
public void testListAlertByStatus() {
Integer count = 10;
AlertStatus waitExecution = AlertStatus.WAIT_EXECUTION;
Map<Integer,Alert> expectedAlertMap = createAlertMap(count, waitExecution);
List<Alert> actualAlerts = alertMapper.listAlertByStatus(waitExecution);
for (Alert actualAlert : actualAlerts){
Alert expectedAlert = expectedAlertMap.get(actualAlert.getId());
if (expectedAlert != null){
assertEquals(expectedAlert,actualAlert);
}
}
}
/**
* test list alert by status
* create alert map
* @param count alert count
* @param alertStatus alert status
* @return alert map
*/
@Test
public void testListAlertByStatus() {
Alert alert = insertOne();
//query
List<Alert> alerts = alertMapper.listAlertByStatus(AlertStatus.EXECUTION_SUCCESS);
Assert.assertNotEquals(alerts.size(), 0);
alertMapper.deleteById(alert.getId());
private Map<Integer,Alert> createAlertMap(Integer count,AlertStatus alertStatus){
Map<Integer,Alert> alertMap = new HashMap<>();
for (int i = 0 ; i < count ;i++){
Alert alert = createAlert(alertStatus);
alertMap.put(alert.getId(),alert);
}
return alertMap;
}
/**
* create alert
* @return alert
* @throws Exception
*/
private Alert createAlert(){
return createAlert(AlertStatus.WAIT_EXECUTION);
}
/**
* create alert
* @param alertStatus alert status
* @return alert
*/
private Alert createAlert(AlertStatus alertStatus){
Alert alert = new Alert();
alert.setShowType(ShowType.TABLE);
alert.setTitle("test alert");
alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]");
alert.setAlertType(AlertType.EMAIL);
alert.setAlertStatus(alertStatus);
alert.setLog("success");
alert.setReceivers("aa@aa.com");
alert.setReceiversCc("bb@aa.com");
alert.setCreateTime(DateUtils.getCurrentDate());
alert.setUpdateTime(DateUtils.getCurrentDate());
alertMapper.insert(alert);
return alert;
}
}

264
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.CommandCount;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
@ -25,13 +26,25 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
/**
* command mapper test
*/
@RunWith(SpringRunner.class)
@SpringBootTest
@Transactional
@Rollback(true)
public class CommandMapperTest {
@ -41,71 +54,85 @@ public class CommandMapperTest {
@Autowired
ProcessDefinitionMapper processDefinitionMapper;
/**
* insert
* @return Command
* test insert
*/
private Command insertOne(){
//insertOne
Command command = new Command();
command.setCommandType(CommandType.START_PROCESS);
command.setProcessDefinitionId(1);
command.setExecutorId(4);
command.setProcessInstancePriority(Priority.MEDIUM);
command.setFailureStrategy(FailureStrategy.CONTINUE);
command.setWorkerGroupId(-1);
command.setWarningGroupId(1);
command.setUpdateTime(new Date());
commandMapper.insert(command);
return command;
@Test
public void testInsert(){
Command command = createCommand();
assertNotNull(command.getId());
assertThat(command.getId(),greaterThan(0));
}
/**
* test update
* test select by id
*/
@Test
public void testUpdate(){
//insertOne
Command command = insertOne();
//update
command.setStartTime(new Date());
int update = commandMapper.updateById(command);
Assert.assertEquals(update, 1);
commandMapper.deleteById(command.getId());
public void testSelectById() {
Command expectedCommand = createCommand();
//query
Command actualCommand = commandMapper.selectById(expectedCommand.getId());
assertEquals(expectedCommand, actualCommand);
}
/**
* test delete
* test update
*/
@Test
public void testDelete(){
public void testUpdate(){
Command expectedCommand = createCommand();
// update the command time if current command if recover from waiting
expectedCommand.setUpdateTime(DateUtils.getCurrentDate());
commandMapper.updateById(expectedCommand);
Command actualCommand = commandMapper.selectById(expectedCommand.getId());
assertEquals(expectedCommand,actualCommand);
Command Command = insertOne();
int delete = commandMapper.deleteById(Command.getId());
Assert.assertEquals(delete, 1);
}
/**
* test query
* test delete
*/
@Test
public void testQuery() {
Command command = insertOne();
//query
List<Command> commands = commandMapper.selectList(null);
Assert.assertNotEquals(commands.size(), 0);
commandMapper.deleteById(command.getId());
public void testDelete(){
Command expectedCommand = createCommand();
commandMapper.deleteById(expectedCommand.getId());
Command actualCommand = commandMapper.selectById(expectedCommand.getId());
assertNull(actualCommand);
}
/**
* test query all
*/
@Test
public void testGetAll() {
Command command = insertOne();
List<Command> commands = commandMapper.selectList(null);
Assert.assertNotEquals(commands.size(), 0);
commandMapper.deleteById(command.getId());
Integer count = 10;
Map<Integer, Command> commandMap = createCommandMap(count);
List<Command> actualCommands = commandMapper.selectList(null);
assertThat(actualCommands.size(), greaterThanOrEqualTo(count));
for (Command actualCommand : actualCommands){
Command expectedCommand = commandMap.get(actualCommand.getId());
if (expectedCommand != null){
assertEquals(expectedCommand,actualCommand);
}
}
}
/**
@ -113,28 +140,14 @@ public class CommandMapperTest {
*/
@Test
public void testGetOneToRun() {
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setReleaseState(ReleaseState.ONLINE);
processDefinition.setName("ut test");
processDefinition.setProjectId(1);
processDefinition.setFlag(Flag.YES);
processDefinitionMapper.insert(processDefinition);
Command command = new Command();
command.setCommandType(CommandType.START_PROCESS);
command.setProcessDefinitionId(processDefinition.getId());
command.setExecutorId(4);
command.setProcessInstancePriority(Priority.MEDIUM);
command.setFailureStrategy(FailureStrategy.CONTINUE);
command.setWorkerGroupId(-1);
command.setWarningGroupId(1);
command.setUpdateTime(new Date());
commandMapper.insert(command);
ProcessDefinition processDefinition = createProcessDefinition();
Command expectedCommand = createCommand(CommandType.START_PROCESS,processDefinition.getId());
Command command2 = commandMapper.getOneToRun();
Assert.assertNotEquals(command2, null);
commandMapper.deleteById(command.getId());
processDefinitionMapper.deleteById(processDefinition.getId());
Command actualCommand = commandMapper.getOneToRun();
assertEquals(expectedCommand, actualCommand);
}
/**
@ -142,35 +155,122 @@ public class CommandMapperTest {
*/
@Test
public void testCountCommandState() {
Command command = insertOne();
Integer count = 10;
ProcessDefinition processDefinition = createProcessDefinition();
CommandCount expectedCommandCount = createCommandMap(count, CommandType.START_PROCESS, processDefinition.getId());
Integer[] projectIdArray = {processDefinition.getProjectId()};
Date startTime = DateUtils.stringToDate("2019-12-29 00:10:00");
Date endTime = DateUtils.stringToDate("2019-12-29 23:59:59");
List<CommandCount> actualCommandCounts = commandMapper.countCommandState(0, startTime, endTime, projectIdArray);
assertThat(actualCommandCounts.size(),greaterThanOrEqualTo(1));
Boolean flag = false;
for (CommandCount actualCommandCount : actualCommandCounts){
if (actualCommandCount.getCommandType().equals(expectedCommandCount.getCommandType())){
assertEquals(expectedCommandCount,actualCommandCount);
flag = true;
}
}
assertTrue(flag);
}
//insertOne
/**
* create command map
* @param count map count
* @param commandType comman type
* @param processDefinitionId process definition id
* @return command map
*/
private CommandCount createCommandMap(
Integer count,
CommandType commandType,
Integer processDefinitionId){
CommandCount commandCount = new CommandCount();
for (int i = 0 ;i < count ;i++){
createCommand(commandType,processDefinitionId);
}
commandCount.setCommandType(commandType);
commandCount.setCount(count);
return commandCount;
}
/**
* create process definition
* @return process definition
*/
private ProcessDefinition createProcessDefinition(){
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setName("def 1");
processDefinition.setProjectId(1010);
processDefinition.setUserId(101);
processDefinition.setUpdateTime(new Date());
processDefinition.setCreateTime(new Date());
processDefinition.setReleaseState(ReleaseState.ONLINE);
processDefinition.setName("ut test");
processDefinition.setProjectId(1);
processDefinition.setFlag(Flag.YES);
processDefinitionMapper.insert(processDefinition);
command.setProcessDefinitionId(processDefinition.getId());
commandMapper.updateById(command);
return processDefinition;
}
/**
* create command map
* @param count map count
* @return command map
*/
private Map<Integer,Command> createCommandMap(Integer count){
Map<Integer,Command> commandMap = new HashMap<>();
for (int i = 0; i < count ;i++){
Command command = createCommand();
commandMap.put(command.getId(),command);
}
return commandMap;
}
List<CommandCount> commandCounts = commandMapper.countCommandState(
4, null, null, new Integer[0]
);
Integer[] projectIdArray = new Integer[2];
projectIdArray[0] = processDefinition.getProjectId();
projectIdArray[1] = 200;
List<CommandCount> commandCounts2 = commandMapper.countCommandState(
4, null, null, projectIdArray
);
/**
* create command
* @return
*/
private Command createCommand(){
return createCommand(CommandType.START_PROCESS,1);
}
/**
* create command
* @return Command
*/
private Command createCommand(CommandType commandType,Integer processDefinitionId){
Command command = new Command();
command.setCommandType(commandType);
command.setProcessDefinitionId(processDefinitionId);
command.setExecutorId(4);
command.setCommandParam("test command param");
command.setTaskDependType(TaskDependType.TASK_ONLY);
command.setFailureStrategy(FailureStrategy.CONTINUE);
command.setWarningType(WarningType.ALL);
command.setWarningGroupId(1);
command.setScheduleTime(DateUtils.stringToDate("2019-12-29 12:10:00"));
command.setProcessInstancePriority(Priority.MEDIUM);
command.setStartTime(DateUtils.stringToDate("2019-12-29 10:10:00"));
command.setUpdateTime(DateUtils.stringToDate("2019-12-29 10:10:00"));
command.setWorkerGroupId(-1);
commandMapper.insert(command);
commandMapper.deleteById(command.getId());
processDefinitionMapper.deleteById(processDefinition.getId());
Assert.assertNotEquals(commandCounts.size(), 0);
Assert.assertNotEquals(commandCounts2.size(), 0);
return command;
}
}

6
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java

@ -152,10 +152,8 @@ public class MasterServer implements IStoppable {
@Override
public void run() {
if (zkMasterClient.getActiveMasterNum() <= 1) {
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) {
zkMasterClient.getAlertDao().sendServerStopedAlert(
1, OSUtils.getHost(), "Master-Server");
}
zkMasterClient.getAlertDao().sendServerStopedAlert(
1, OSUtils.getHost(), "Master-Server");
}
stop("shutdownhook");
}

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java

@ -177,9 +177,7 @@ public class WorkerServer implements IStoppable {
public void run() {
// worker server exit alert
if (zkWorkerClient.getActiveMasterNum() <= 1) {
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) {
alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server");
}
alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server");
}
stop("shutdownhook");
}

6
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java

@ -247,10 +247,8 @@ public class ZKMasterClient extends AbstractZKClient {
*/
private void alertServerDown(String serverHost, ZKNodeType zkNodeType) {
String serverType = zkNodeType.toString();
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) {
alertDao.sendServerStopedAlert(1, serverHost, serverType);
}
String serverType = zkNodeType.toString();
alertDao.sendServerStopedAlert(1, serverHost, serverType);
}
/**

8
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue

@ -257,8 +257,8 @@
_rtParam () {
return {
taskInstId: this.stateId || this.logId,
skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '0000' : 0}`),
limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}0000`)
skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '000' : 0}`),
limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}000`)
}
}
},
@ -316,7 +316,7 @@
}
.refresh-log {
>i {
font-size: 24px;
font-size: 20px;
vertical-align: middle;
transform: scale(1);
}
@ -353,7 +353,7 @@
font-weight: bold;
resize:none;
line-height: 1.6;
padding: 6px;
padding: 0px;
}
}
}

124
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue

@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<template>
<div class="script-model">
<m-list-box>
<div slot="content">
<div class="from-mirror1">
<textarea
id="code-shell-mirror1"
name="code-shell-mirror1"
style="opacity: 0">
</textarea>
</div>
</div>
</m-list-box>
<a class="ans-modal-box-close">
<i class="ans-icon-min" @click="closeModal"></i>
</a>
</div>
</template>
<script>
import _ from 'lodash'
import i18n from '@/module/i18n'
import mListBox from './listBox'
import disabledState from '@/module/mixin/disabledState'
import codemirror from '@/conf/home/pages/resource/pages/file/pages/_source/codemirror'
let editor
export default {
name: 'shell',
data () {
return {
// script
rawScript: '',
}
},
mixins: [disabledState],
props: {
item: String
},
methods: {
/**
* Processing code highlighting
*/
_handlerEditor () {
// editor
let self =this
editor = codemirror('code-shell-mirror1', {
mode: 'shell',
readOnly: this.isDetails
})
editor.on("change",function(){
self.$emit('getSriptBoxValue',editor.getValue())
})
this.keypress = () => {
if (!editor.getOption('readOnly')) {
editor.showHint({
completeSingle: false
})
}
}
// Monitor keyboard
editor.on('keypress', this.keypress)
editor.setValue(this.rawScript)
return editor
},
closeModal() {
let self = this
self.$emit('closeAble')
}
},
watch: {},
created () {
let o = this.item
// Non-null objects represent backfill
if (!_.isEmpty(o)) {
this.rawScript = o
}
},
mounted () {
setTimeout(() => {
this._handlerEditor()
}, 200)
},
destroyed () {
if (editor) {
editor.toTextArea() // Uninstall
editor.off($('.code-shell-mirror1'), 'keypress', this.keypress)
}
},
components: { mListBox }
}
</script>
<style lang="scss" rel="stylesheet/scss" scope>
.script-model {
width:100%;
}
.from-mirror1 {
.CodeMirror {
min-height: 600px;
max-height: 700px;
}
}
</style>

64
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue

@ -21,10 +21,13 @@
<div slot="content">
<div class="from-mirror">
<textarea
id="code-shell-mirror"
name="code-shell-mirror"
style="opacity: 0">
id="code-shell-mirror"
name="code-shell-mirror"
style="opacity: 0">
</textarea>
<a class="ans-modal-box-max">
<i class="ans-icon-max" @click="setEditorVal"></i>
</a>
</div>
</div>
</m-list-box>
@ -55,6 +58,7 @@
import _ from 'lodash'
import i18n from '@/module/i18n'
import mListBox from './_source/listBox'
import mScriptBox from './_source/scriptBox'
import mResources from './_source/resources'
import mLocalParams from './_source/localParams'
import disabledState from '@/module/mixin/disabledState'
@ -85,8 +89,37 @@
_onLocalParams (a) {
this.localParams = a
},
setEditorVal() {
let self = this
let modal = self.$modal.dialog({
className: 'scriptModal',
closable: false,
showMask: true,
maskClosable: true,
onClose: function() {
},
render (h) {
return h(mScriptBox, {
on: {
getSriptBoxValue (val) {
editor.setValue(val)
},
closeAble () {
// this.$modal.destroy()
modal.remove()
}
},
props: {
item: editor.getValue()
}
})
}
})
},
/**
* return resourceList
*
*/
_onResourcesData (a) {
this.resourceList = a
@ -109,7 +142,6 @@
if (!this.$refs.refLocalParams._verifProp()) {
return false
}
// storage
this.$emit('on-params', {
resourceList: this.resourceList,
@ -138,7 +170,6 @@
// Monitor keyboard
editor.on('keypress', this.keypress)
editor.setValue(this.rawScript)
return editor
@ -176,6 +207,27 @@
editor.off($('.code-shell-mirror'), 'keypress', this.keypress)
}
},
components: { mLocalParams, mListBox, mResources }
components: { mLocalParams, mListBox, mResources, mScriptBox }
}
</script>
<style lang="scss" rel="stylesheet/scss" scope>
.scriptModal {
.ans-modal-box-content-wrapper {
width: 90%;
.ans-modal-box-close {
right: -12px;
top: -16px;
color: #fff;
}
}
}
.ans-modal-box-close {
z-index: 100;
}
.ans-modal-box-max {
position: absolute;
right: -12px;
top: -16px;
}
</style>

16
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue

@ -313,7 +313,23 @@
this.crontab = this.item.crontab
}
if(this.type == 'timing') {
let date = new Date()
let year = date.getFullYear()
let month = date.getMonth() + 1
let day = date.getDate()
if (month < 10) {
month = "0" + month;
}
if (day < 10) {
day = "0" + day;
}
let startDate = year + "-" + month + "-" + day + ' ' + '00:00:00'
let endDate = (year+100) + "-" + month + "-" + day + ' ' + '00:00:00'
let times = []
times[0] = startDate
times[1] = endDate
this.crontab = '0 0 * * * ? *'
this.scheduleTime = times
}
this.receivers = _.cloneDeep(this.receiversD)
this.receiversCc = _.cloneDeep(this.receiversCcD)

2
pom.xml

@ -674,6 +674,8 @@
<include>**/server/utils/FlinkArgsUtilsTest.java</include>
<include>**/dao/mapper/AccessTokenMapperTest.java</include>
<include>**/dao/mapper/AlertGroupMapperTest.java</include>
<include>**/dao/mapper/AlertMapperTest.java</include>
<include>**/dao/mapper/CommandMapperTest.java</include>
</includes>
<!-- <skip>true</skip> -->
</configuration>

Loading…
Cancel
Save