diff --git a/.github/workflows/SonarCloud.yml b/.github/workflows/SonarCloud.yml new file mode 100644 index 0000000000..fd5b0585db --- /dev/null +++ b/.github/workflows/SonarCloud.yml @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: SonarCloud +on: [push, pull_request] +jobs: + sonarCloudTrigger: + name: SonarCloud Trigger + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - uses: actions/cache@v1 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube + restore-keys: | + ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube + ${{ runner.os }}-maven- + - name: Maven clean + run: mvn clean + - name: Run SonarCloud analyse + run: > + mvn clean --batch-mode + org.jacoco:jacoco-maven-plugin:prepare-agent + verify + org.sonarsource.scanner.maven:sonar-maven-plugin:sonar + -Dmaven.test.skip=true + -Dsonar.host.url=https://sonarcloud.io + -Dsonar.organization=apache + -Dsonar.projectKey=apache-dolphinscheduler + -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/README.md b/README.md index aee0e3b2e3..bc7dba684f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ Dolphin Scheduler Official Website [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) -![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) + > Dolphin Scheduler for Big Data diff --git a/README_zh_CN.md b/README_zh_CN.md index f8855aca6c..8141763c33 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -4,7 +4,7 @@ Dolphin Scheduler Official Website [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) -![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) > Dolphin Scheduler for Big Data diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index ee38f3d3c9..a41498e258 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -39,19 +39,15 @@ public class AlertServer { private AlertSender alertSender; - private static volatile AlertServer instance; + private static AlertServer instance; public AlertServer() { } - public static AlertServer getInstance(){ + public synchronized static AlertServer getInstance(){ if (null == instance) { - synchronized (AlertServer.class) { - if(null == instance) { - instance = new AlertServer(); - } - } + instance = new AlertServer(); } return instance; } @@ -63,6 +59,7 @@ public class AlertServer { Thread.sleep(Constants.ALERT_SCAN_INTERVEL); } catch (InterruptedException e) { logger.error(e.getMessage(),e); + Thread.currentThread().interrupt(); } List alerts = alertDao.listWaitExecutionAlert(); alertSender = new AlertSender(alerts, alertDao); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index a11d37f3d3..15e7e18855 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -86,20 +86,24 @@ public class EnterpriseWeChatUtils { String resp; CloseableHttpClient httpClient = HttpClients.createDefault(); - HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace); - CloseableHttpResponse response = httpClient.execute(httpGet); try { - HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, Constants.UTF_8); - EntityUtils.consume(entity); + HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace); + CloseableHttpResponse response = httpClient.execute(httpGet); + try { + HttpEntity entity = response.getEntity(); + resp = EntityUtils.toString(entity, Constants.UTF_8); + EntityUtils.consume(entity); + } finally { + response.close(); + } + + Map map = JSON.parseObject(resp, + new TypeToken>() { + }.getType()); + return map.get("access_token").toString(); } finally { - response.close(); + httpClient.close(); } - - Map map = JSON.parseObject(resp, - new TypeToken>() { - }.getType()); - return map.get("access_token").toString(); } /** @@ -167,20 +171,25 @@ public class EnterpriseWeChatUtils { public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token); - CloseableHttpClient httpclient = HttpClients.createDefault(); - HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace); - httpPost.setEntity(new StringEntity(data, charset)); - CloseableHttpResponse response = httpclient.execute(httpPost); - String resp; + CloseableHttpClient httpClient = HttpClients.createDefault(); try { - HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, charset); - EntityUtils.consume(entity); + HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace); + httpPost.setEntity(new StringEntity(data, charset)); + CloseableHttpResponse response = httpClient.execute(httpPost); + String resp; + try { + HttpEntity entity = response.getEntity(); + resp = EntityUtils.toString(entity, charset); + EntityUtils.consume(entity); + } finally { + response.close(); + } + logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", + enterpriseWeChatPushUrl, data, resp); + return resp; } finally { - response.close(); + httpClient.close(); } - logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", enterpriseWeChatPushUrl, data, resp); - return resp; } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java index f860517d0f..862c895c92 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java @@ -115,12 +115,12 @@ public class QueueService extends BaseService { } if (StringUtils.isEmpty(queue)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue"); return result; } if (StringUtils.isEmpty(queueName)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName"); return result; } @@ -163,6 +163,16 @@ public class QueueService extends BaseService { return result; } + if (StringUtils.isEmpty(queue)) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue"); + return result; + } + + if (StringUtils.isEmpty(queueName)) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName"); + return result; + } + Queue queueObj = queueMapper.selectById(id); if (queueObj == null) { putMsg(result, Status.QUEUE_NOT_EXIST, id); @@ -222,12 +232,12 @@ public class QueueService extends BaseService { Result result = new Result(); if (StringUtils.isEmpty(queue)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue"); return result; } if (StringUtils.isEmpty(queueName)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName"); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index 92319e71a1..a4664b6c3e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -221,9 +221,13 @@ public class UsersService extends BaseService { return result; } - Date now = new Date(); - if (StringUtils.isNotEmpty(userName)) { + + if (!CheckUtils.checkUserName(userName)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userName); + return result; + } + User tempUser = userMapper.queryByUserNameAccurately(userName); if (tempUser != null && tempUser.getId() != userId) { putMsg(result, Status.USER_NAME_EXIST); @@ -233,14 +237,30 @@ public class UsersService extends BaseService { } if (StringUtils.isNotEmpty(userPassword)) { + if (!CheckUtils.checkPassword(userPassword)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userPassword); + return result; + } user.setUserPassword(EncryptionUtils.getMd5(userPassword)); } if (StringUtils.isNotEmpty(email)) { + if (!CheckUtils.checkEmail(email)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,email); + return result; + } user.setEmail(email); } + + if (StringUtils.isNotEmpty(phone)) { + if (!CheckUtils.checkPhone(phone)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,phone); + return result; + } + user.setPhone(phone); + } user.setQueue(queue); - user.setPhone(phone); + Date now = new Date(); user.setUpdateTime(now); //if user switches the tenant, the user's resources need to be copied to the new tenant @@ -318,10 +338,15 @@ public class UsersService extends BaseService { Map result = new HashMap<>(5); //only admin can operate if (!isAdmin(loginUser)) { + putMsg(result, Status.USER_NO_OPERATION_PERM, id); + return result; + } + //check exist + User tempUser = userMapper.selectById(id); + if (tempUser == null) { putMsg(result, Status.USER_NOT_EXIST, id); return result; } - // delete user User user = userMapper.queryTenantCodeByUserId(id); @@ -357,6 +382,12 @@ public class UsersService extends BaseService { return result; } + //check exist + User tempUser = userMapper.selectById(userId); + if (tempUser == null) { + putMsg(result, Status.USER_NOT_EXIST, userId); + return result; + } //if the selected projectIds are empty, delete all items associated with the user projectUserMapper.deleteProjectRelation(0, userId); @@ -443,6 +474,11 @@ public class UsersService extends BaseService { if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; } + User user = userMapper.selectById(userId); + if(user == null){ + putMsg(result, Status.USER_NOT_EXIST, userId); + return result; + } udfUserMapper.deleteByUserId(userId); @@ -485,6 +521,11 @@ public class UsersService extends BaseService { if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; } + User user = userMapper.selectById(userId); + if(user == null){ + putMsg(result, Status.USER_NOT_EXIST, userId); + return result; + } datasourceUserMapper.deleteByUserId(userId); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java index 925d01bd09..2d0743a5ab 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java @@ -76,6 +76,11 @@ public class WorkerGroupService extends BaseService { WorkerGroup workerGroup = null; if(id != 0){ workerGroup = workerGroupMapper.selectById(id); + //check exist + if (workerGroup == null){ + workerGroup = new WorkerGroup(); + workerGroup.setCreateTime(now); + } }else{ workerGroup = new WorkerGroup(); workerGroup.setCreateTime(now); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java index d989f82fa2..dac96e8695 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java @@ -32,7 +32,6 @@ import java.util.Map; /** * alert */ -@Data @TableName("t_ds_alert") public class Alert { @@ -217,6 +216,72 @@ public class Alert { this.updateTime = updateTime; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Alert alert = (Alert) o; + + if (id != alert.id) { + return false; + } + if (alertGroupId != alert.alertGroupId) { + return false; + } + if (!title.equals(alert.title)) { + return false; + } + if (showType != alert.showType) { + return false; + } + if (!content.equals(alert.content)) { + return false; + } + if (alertType != alert.alertType) { + return false; + } + if (alertStatus != alert.alertStatus) { + return false; + } + if (!log.equals(alert.log)) { + return false; + } + if (!receivers.equals(alert.receivers)) { + return false; + } + if (!receiversCc.equals(alert.receiversCc)) { + return false; + } + if (!createTime.equals(alert.createTime)) { + return false; + } + return updateTime.equals(alert.updateTime) && info.equals(alert.info); + + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + title.hashCode(); + result = 31 * result + showType.hashCode(); + result = 31 * result + content.hashCode(); + result = 31 * result + alertType.hashCode(); + result = 31 * result + alertStatus.hashCode(); + result = 31 * result + log.hashCode(); + result = 31 * result + alertGroupId; + result = 31 * result + receivers.hashCode(); + result = 31 * result + receiversCc.hashCode(); + result = 31 * result + createTime.hashCode(); + result = 31 * result + updateTime.hashCode(); + result = 31 * result + info.hashCode(); + return result; + } + @Override public String toString() { return "Alert{" + @@ -228,10 +293,10 @@ public class Alert { ", alertStatus=" + alertStatus + ", log='" + log + '\'' + ", alertGroupId=" + alertGroupId + - ", createTime=" + createTime + - ", updateTime=" + updateTime + ", receivers='" + receivers + '\'' + ", receiversCc='" + receiversCc + '\'' + + ", createTime=" + createTime + + ", updateTime=" + updateTime + ", info=" + info + '}'; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java index a180a433bc..2d47b7874a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java @@ -28,7 +28,6 @@ import java.util.Date; /** * command */ -@Data @TableName("t_ds_command") public class Command { @@ -265,6 +264,79 @@ public class Command { this.workerGroupId = workerGroupId; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Command command = (Command) o; + + if (id != command.id) { + return false; + } + if (processDefinitionId != command.processDefinitionId) { + return false; + } + if (executorId != command.executorId) { + return false; + } + if (workerGroupId != command.workerGroupId) { + return false; + } + if (commandType != command.commandType) { + return false; + } + if (commandParam != null ? !commandParam.equals(command.commandParam) : command.commandParam != null) { + return false; + } + if (taskDependType != command.taskDependType) { + return false; + } + if (failureStrategy != command.failureStrategy) { + return false; + } + if (warningType != command.warningType) { + return false; + } + if (warningGroupId != null ? !warningGroupId.equals(command.warningGroupId) : command.warningGroupId != null) { + return false; + } + if (scheduleTime != null ? !scheduleTime.equals(command.scheduleTime) : command.scheduleTime != null) { + return false; + } + if (startTime != null ? !startTime.equals(command.startTime) : command.startTime != null) { + return false; + } + if (processInstancePriority != command.processInstancePriority) { + return false; + } + return !(updateTime != null ? !updateTime.equals(command.updateTime) : command.updateTime != null); + + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + (commandType != null ? commandType.hashCode() : 0); + result = 31 * result + processDefinitionId; + result = 31 * result + executorId; + result = 31 * result + (commandParam != null ? commandParam.hashCode() : 0); + result = 31 * result + (taskDependType != null ? taskDependType.hashCode() : 0); + result = 31 * result + (failureStrategy != null ? failureStrategy.hashCode() : 0); + result = 31 * result + (warningType != null ? warningType.hashCode() : 0); + result = 31 * result + (warningGroupId != null ? warningGroupId.hashCode() : 0); + result = 31 * result + (scheduleTime != null ? scheduleTime.hashCode() : 0); + result = 31 * result + (startTime != null ? startTime.hashCode() : 0); + result = 31 * result + (processInstancePriority != null ? processInstancePriority.hashCode() : 0); + result = 31 * result + (updateTime != null ? updateTime.hashCode() : 0); + result = 31 * result + workerGroupId; + return result; + } + @Override public String toString() { return "Command{" + diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java index 158169b727..df99b54dbe 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java @@ -33,13 +33,6 @@ public class CommandCount { private int count; - @Override - public String toString(){ - return "command count:" + - " commandType: "+ commandType.toString() + - " count: "+ count; - } - public CommandType getCommandType() { return commandType; } @@ -55,4 +48,37 @@ public class CommandCount { public void setCount(int count) { this.count = count; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + CommandCount that = (CommandCount) o; + + if (count != that.count) { + return false; + } + return commandType == that.commandType; + + } + + @Override + public int hashCode() { + int result = commandType != null ? commandType.hashCode() : 0; + result = 31 * result + count; + return result; + } + + @Override + public String toString() { + return "CommandCount{" + + "commandType=" + commandType + + ", count=" + count + + '}'; + } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java index 958cd39c18..4e298e3aa3 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java @@ -19,44 +19,55 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.Alert; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.List; - +import java.util.*; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; +/** + * alert mapper test + */ @RunWith(SpringRunner.class) @SpringBootTest +@Transactional +@Rollback(true) public class AlertMapperTest { @Autowired - AlertMapper alertMapper; + private AlertMapper alertMapper; /** - * insert - * @return Alert + * test insert + * @return */ - private Alert insertOne(){ - //insertOne - Alert alert = new Alert(); - alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]"); - alert.setLog("success"); - alert.setReceivers("xx@aa.com"); - alert.setAlertType(AlertType.EMAIL); - alert.setShowType(ShowType.TABLE); - alert.setAlertGroupId(1); - alert.setAlertStatus(AlertStatus.EXECUTION_SUCCESS); - alert.setCreateTime(new Date()); - alert.setUpdateTime(new Date()); - alertMapper.insert(alert); - return alert; + @Test + public void testInsert(){ + Alert expectedAlert = createAlert(); + assertNotNull(expectedAlert.getId()); + assertThat(expectedAlert.getId(), greaterThan(0)); + } + + + /** + * test select by id + * @return + */ + @Test + public void testSelectById(){ + Alert expectedAlert = createAlert(); + Alert actualAlert = alertMapper.selectById(expectedAlert.getId()); + assertEquals(expectedAlert, actualAlert); } /** @@ -64,13 +75,18 @@ public class AlertMapperTest { */ @Test public void testUpdate(){ - //insertOne - Alert alert = insertOne(); - //update - alert.setTitle("hello"); - int update = alertMapper.updateById(alert); - Assert.assertEquals(update, 1); - alertMapper.deleteById(alert.getId()); + + Alert expectedAlert = createAlert(); + + expectedAlert.setAlertStatus(AlertStatus.EXECUTION_FAILURE); + expectedAlert.setLog("error"); + expectedAlert.setUpdateTime(DateUtils.getCurrentDate()); + + alertMapper.updateById(expectedAlert); + + Alert actualAlert = alertMapper.selectById(expectedAlert.getId()); + + assertEquals(expectedAlert, actualAlert); } /** @@ -78,33 +94,83 @@ public class AlertMapperTest { */ @Test public void testDelete(){ + Alert expectedAlert = createAlert(); + + alertMapper.deleteById(expectedAlert.getId()); + + Alert actualAlert = alertMapper.selectById(expectedAlert.getId()); - Alert alert = insertOne(); - int delete = alertMapper.deleteById(alert.getId()); - Assert.assertEquals(delete, 1); + assertNull(actualAlert); } + /** - * test query + * test list alert by status */ @Test - public void testQuery() { - Alert alert = insertOne(); - //query - List alerts = alertMapper.selectList(null); - Assert.assertNotEquals(alerts.size(), 0); - alertMapper.deleteById(alert.getId()); + public void testListAlertByStatus() { + Integer count = 10; + AlertStatus waitExecution = AlertStatus.WAIT_EXECUTION; + + Map expectedAlertMap = createAlertMap(count, waitExecution); + + List actualAlerts = alertMapper.listAlertByStatus(waitExecution); + + for (Alert actualAlert : actualAlerts){ + Alert expectedAlert = expectedAlertMap.get(actualAlert.getId()); + if (expectedAlert != null){ + assertEquals(expectedAlert,actualAlert); + } + } } /** - * test list alert by status + * create alert map + * @param count alert count + * @param alertStatus alert status + * @return alert map */ - @Test - public void testListAlertByStatus() { - Alert alert = insertOne(); - //query - List alerts = alertMapper.listAlertByStatus(AlertStatus.EXECUTION_SUCCESS); - Assert.assertNotEquals(alerts.size(), 0); - alertMapper.deleteById(alert.getId()); + private Map createAlertMap(Integer count,AlertStatus alertStatus){ + Map alertMap = new HashMap<>(); + + for (int i = 0 ; i < count ;i++){ + Alert alert = createAlert(alertStatus); + alertMap.put(alert.getId(),alert); + } + + return alertMap; + + } + + + /** + * create alert + * @return alert + * @throws Exception + */ + private Alert createAlert(){ + return createAlert(AlertStatus.WAIT_EXECUTION); + } + + /** + * create alert + * @param alertStatus alert status + * @return alert + */ + private Alert createAlert(AlertStatus alertStatus){ + Alert alert = new Alert(); + alert.setShowType(ShowType.TABLE); + alert.setTitle("test alert"); + alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]"); + alert.setAlertType(AlertType.EMAIL); + alert.setAlertStatus(alertStatus); + alert.setLog("success"); + alert.setReceivers("aa@aa.com"); + alert.setReceiversCc("bb@aa.com"); + alert.setCreateTime(DateUtils.getCurrentDate()); + alert.setUpdateTime(DateUtils.getCurrentDate()); + + alertMapper.insert(alert); + return alert; } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 2af44576b5..c35ce7e8ce 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.CommandCount; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; @@ -25,13 +26,25 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; + +/** + * command mapper test + */ @RunWith(SpringRunner.class) @SpringBootTest +@Transactional +@Rollback(true) public class CommandMapperTest { @@ -41,71 +54,85 @@ public class CommandMapperTest { @Autowired ProcessDefinitionMapper processDefinitionMapper; + /** - * insert - * @return Command + * test insert */ - private Command insertOne(){ - //insertOne - Command command = new Command(); - command.setCommandType(CommandType.START_PROCESS); - command.setProcessDefinitionId(1); - command.setExecutorId(4); - command.setProcessInstancePriority(Priority.MEDIUM); - command.setFailureStrategy(FailureStrategy.CONTINUE); - command.setWorkerGroupId(-1); - command.setWarningGroupId(1); - command.setUpdateTime(new Date()); - commandMapper.insert(command); - return command; + @Test + public void testInsert(){ + Command command = createCommand(); + assertNotNull(command.getId()); + assertThat(command.getId(),greaterThan(0)); } + /** - * test update + * test select by id */ @Test - public void testUpdate(){ - //insertOne - Command command = insertOne(); - //update - command.setStartTime(new Date()); - int update = commandMapper.updateById(command); - Assert.assertEquals(update, 1); - commandMapper.deleteById(command.getId()); + public void testSelectById() { + Command expectedCommand = createCommand(); + //query + Command actualCommand = commandMapper.selectById(expectedCommand.getId()); + + assertEquals(expectedCommand, actualCommand); } /** - * test delete + * test update */ @Test - public void testDelete(){ + public void testUpdate(){ + + Command expectedCommand = createCommand(); + + // update the command time if current command if recover from waiting + expectedCommand.setUpdateTime(DateUtils.getCurrentDate()); + + commandMapper.updateById(expectedCommand); + + Command actualCommand = commandMapper.selectById(expectedCommand.getId()); + + assertEquals(expectedCommand,actualCommand); - Command Command = insertOne(); - int delete = commandMapper.deleteById(Command.getId()); - Assert.assertEquals(delete, 1); } /** - * test query + * test delete */ @Test - public void testQuery() { - Command command = insertOne(); - //query - List commands = commandMapper.selectList(null); - Assert.assertNotEquals(commands.size(), 0); - commandMapper.deleteById(command.getId()); + public void testDelete(){ + Command expectedCommand = createCommand(); + + commandMapper.deleteById(expectedCommand.getId()); + + Command actualCommand = commandMapper.selectById(expectedCommand.getId()); + + assertNull(actualCommand); } + + /** * test query all */ @Test public void testGetAll() { - Command command = insertOne(); - List commands = commandMapper.selectList(null); - Assert.assertNotEquals(commands.size(), 0); - commandMapper.deleteById(command.getId()); + Integer count = 10; + + Map commandMap = createCommandMap(count); + + + List actualCommands = commandMapper.selectList(null); + + assertThat(actualCommands.size(), greaterThanOrEqualTo(count)); + + for (Command actualCommand : actualCommands){ + Command expectedCommand = commandMap.get(actualCommand.getId()); + if (expectedCommand != null){ + assertEquals(expectedCommand,actualCommand); + } + } } /** @@ -113,28 +140,14 @@ public class CommandMapperTest { */ @Test public void testGetOneToRun() { - ProcessDefinition processDefinition = new ProcessDefinition(); - processDefinition.setReleaseState(ReleaseState.ONLINE); - processDefinition.setName("ut test"); - processDefinition.setProjectId(1); - processDefinition.setFlag(Flag.YES); - processDefinitionMapper.insert(processDefinition); - Command command = new Command(); - command.setCommandType(CommandType.START_PROCESS); - command.setProcessDefinitionId(processDefinition.getId()); - command.setExecutorId(4); - command.setProcessInstancePriority(Priority.MEDIUM); - command.setFailureStrategy(FailureStrategy.CONTINUE); - command.setWorkerGroupId(-1); - command.setWarningGroupId(1); - command.setUpdateTime(new Date()); - commandMapper.insert(command); + ProcessDefinition processDefinition = createProcessDefinition(); + + Command expectedCommand = createCommand(CommandType.START_PROCESS,processDefinition.getId()); - Command command2 = commandMapper.getOneToRun(); - Assert.assertNotEquals(command2, null); - commandMapper.deleteById(command.getId()); - processDefinitionMapper.deleteById(processDefinition.getId()); + Command actualCommand = commandMapper.getOneToRun(); + + assertEquals(expectedCommand, actualCommand); } /** @@ -142,35 +155,122 @@ public class CommandMapperTest { */ @Test public void testCountCommandState() { - Command command = insertOne(); + Integer count = 10; + + ProcessDefinition processDefinition = createProcessDefinition(); + + CommandCount expectedCommandCount = createCommandMap(count, CommandType.START_PROCESS, processDefinition.getId()); + + Integer[] projectIdArray = {processDefinition.getProjectId()}; + + Date startTime = DateUtils.stringToDate("2019-12-29 00:10:00"); + + Date endTime = DateUtils.stringToDate("2019-12-29 23:59:59"); + + List actualCommandCounts = commandMapper.countCommandState(0, startTime, endTime, projectIdArray); + + assertThat(actualCommandCounts.size(),greaterThanOrEqualTo(1)); + + Boolean flag = false; + for (CommandCount actualCommandCount : actualCommandCounts){ + if (actualCommandCount.getCommandType().equals(expectedCommandCount.getCommandType())){ + assertEquals(expectedCommandCount,actualCommandCount); + flag = true; + } + } + + assertTrue(flag); + } - //insertOne + + /** + * create command map + * @param count map count + * @param commandType comman type + * @param processDefinitionId process definition id + * @return command map + */ + private CommandCount createCommandMap( + Integer count, + CommandType commandType, + Integer processDefinitionId){ + + CommandCount commandCount = new CommandCount(); + + for (int i = 0 ;i < count ;i++){ + createCommand(commandType,processDefinitionId); + } + commandCount.setCommandType(commandType); + commandCount.setCount(count); + + return commandCount; + } + + /** + * create process definition + * @return process definition + */ + private ProcessDefinition createProcessDefinition(){ ProcessDefinition processDefinition = new ProcessDefinition(); - processDefinition.setName("def 1"); - processDefinition.setProjectId(1010); - processDefinition.setUserId(101); - processDefinition.setUpdateTime(new Date()); - processDefinition.setCreateTime(new Date()); + processDefinition.setReleaseState(ReleaseState.ONLINE); + processDefinition.setName("ut test"); + processDefinition.setProjectId(1); + processDefinition.setFlag(Flag.YES); + processDefinitionMapper.insert(processDefinition); - command.setProcessDefinitionId(processDefinition.getId()); - commandMapper.updateById(command); + return processDefinition; + } + /** + * create command map + * @param count map count + * @return command map + */ + private Map createCommandMap(Integer count){ + Map commandMap = new HashMap<>(); + + for (int i = 0; i < count ;i++){ + Command command = createCommand(); + commandMap.put(command.getId(),command); + } + return commandMap; + } - List commandCounts = commandMapper.countCommandState( - 4, null, null, new Integer[0] - ); - Integer[] projectIdArray = new Integer[2]; - projectIdArray[0] = processDefinition.getProjectId(); - projectIdArray[1] = 200; - List commandCounts2 = commandMapper.countCommandState( - 4, null, null, projectIdArray - ); + /** + * create command + * @return + */ + private Command createCommand(){ + return createCommand(CommandType.START_PROCESS,1); + } + + /** + * create command + * @return Command + */ + private Command createCommand(CommandType commandType,Integer processDefinitionId){ + + Command command = new Command(); + command.setCommandType(commandType); + command.setProcessDefinitionId(processDefinitionId); + command.setExecutorId(4); + command.setCommandParam("test command param"); + command.setTaskDependType(TaskDependType.TASK_ONLY); + command.setFailureStrategy(FailureStrategy.CONTINUE); + command.setWarningType(WarningType.ALL); + command.setWarningGroupId(1); + command.setScheduleTime(DateUtils.stringToDate("2019-12-29 12:10:00")); + command.setProcessInstancePriority(Priority.MEDIUM); + command.setStartTime(DateUtils.stringToDate("2019-12-29 10:10:00")); + command.setUpdateTime(DateUtils.stringToDate("2019-12-29 10:10:00")); + command.setWorkerGroupId(-1); + commandMapper.insert(command); - commandMapper.deleteById(command.getId()); - processDefinitionMapper.deleteById(processDefinition.getId()); - Assert.assertNotEquals(commandCounts.size(), 0); - Assert.assertNotEquals(commandCounts2.size(), 0); + return command; } + + + } \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index e8c8b6779e..1f5b16a25a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -152,10 +152,8 @@ public class MasterServer implements IStoppable { @Override public void run() { if (zkMasterClient.getActiveMasterNum() <= 1) { - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - zkMasterClient.getAlertDao().sendServerStopedAlert( - 1, OSUtils.getHost(), "Master-Server"); - } + zkMasterClient.getAlertDao().sendServerStopedAlert( + 1, OSUtils.getHost(), "Master-Server"); } stop("shutdownhook"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index fe9c4991ac..ea8330e18a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -177,9 +177,7 @@ public class WorkerServer implements IStoppable { public void run() { // worker server exit alert if (zkWorkerClient.getActiveMasterNum() <= 1) { - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); - } + alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); } stop("shutdownhook"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index a26a217665..1c7b737f5b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -247,10 +247,8 @@ public class ZKMasterClient extends AbstractZKClient { */ private void alertServerDown(String serverHost, ZKNodeType zkNodeType) { - String serverType = zkNodeType.toString(); - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - alertDao.sendServerStopedAlert(1, serverHost, serverType); - } + String serverType = zkNodeType.toString(); + alertDao.sendServerStopedAlert(1, serverHost, serverType); } /** diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue index 4c1acff5e6..7ec9cdfa7a 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue @@ -257,8 +257,8 @@ _rtParam () { return { taskInstId: this.stateId || this.logId, - skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '0000' : 0}`), - limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}0000`) + skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '000' : 0}`), + limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}000`) } } }, @@ -316,7 +316,7 @@ } .refresh-log { >i { - font-size: 24px; + font-size: 20px; vertical-align: middle; transform: scale(1); } @@ -353,7 +353,7 @@ font-weight: bold; resize:none; line-height: 1.6; - padding: 6px; + padding: 0px; } } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue new file mode 100644 index 0000000000..c16d18cc38 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 230b43d446..8b45e7c3d1 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -21,10 +21,13 @@
+ + +
@@ -55,6 +58,7 @@ import _ from 'lodash' import i18n from '@/module/i18n' import mListBox from './_source/listBox' + import mScriptBox from './_source/scriptBox' import mResources from './_source/resources' import mLocalParams from './_source/localParams' import disabledState from '@/module/mixin/disabledState' @@ -85,8 +89,37 @@ _onLocalParams (a) { this.localParams = a }, + setEditorVal() { + let self = this + let modal = self.$modal.dialog({ + className: 'scriptModal', + closable: false, + showMask: true, + maskClosable: true, + onClose: function() { + + }, + render (h) { + return h(mScriptBox, { + on: { + getSriptBoxValue (val) { + editor.setValue(val) + }, + closeAble () { + // this.$modal.destroy() + modal.remove() + } + }, + props: { + item: editor.getValue() + } + }) + } + }) + }, /** * return resourceList + * */ _onResourcesData (a) { this.resourceList = a @@ -109,7 +142,6 @@ if (!this.$refs.refLocalParams._verifProp()) { return false } - // storage this.$emit('on-params', { resourceList: this.resourceList, @@ -138,7 +170,6 @@ // Monitor keyboard editor.on('keypress', this.keypress) - editor.setValue(this.rawScript) return editor @@ -176,6 +207,27 @@ editor.off($('.code-shell-mirror'), 'keypress', this.keypress) } }, - components: { mLocalParams, mListBox, mResources } + components: { mLocalParams, mListBox, mResources, mScriptBox } } + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index 6ed78980f1..a15999edbd 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -313,7 +313,23 @@ this.crontab = this.item.crontab } if(this.type == 'timing') { + let date = new Date() + let year = date.getFullYear() + let month = date.getMonth() + 1 + let day = date.getDate() + if (month < 10) { + month = "0" + month; + } + if (day < 10) { + day = "0" + day; + } + let startDate = year + "-" + month + "-" + day + ' ' + '00:00:00' + let endDate = (year+100) + "-" + month + "-" + day + ' ' + '00:00:00' + let times = [] + times[0] = startDate + times[1] = endDate this.crontab = '0 0 * * * ? *' + this.scheduleTime = times } this.receivers = _.cloneDeep(this.receiversD) this.receiversCc = _.cloneDeep(this.receiversCcD) diff --git a/pom.xml b/pom.xml index 69f1ff0665..7936bf7dc8 100644 --- a/pom.xml +++ b/pom.xml @@ -674,6 +674,8 @@ **/server/utils/FlinkArgsUtilsTest.java **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java + **/dao/mapper/AlertMapperTest.java + **/dao/mapper/CommandMapperTest.java