From d91e0155c749ad01dc7ffdb8aa2743f7ba1b0b5c Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 17:35:15 +0800 Subject: [PATCH 1/7] 1when queue or queName is null,return info param is null 2 update queue may NPE --- .../api/service/QueueService.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java index f860517d0f..862c895c92 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java @@ -115,12 +115,12 @@ public class QueueService extends BaseService { } if (StringUtils.isEmpty(queue)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue"); return result; } if (StringUtils.isEmpty(queueName)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName"); return result; } @@ -163,6 +163,16 @@ public class QueueService extends BaseService { return result; } + if (StringUtils.isEmpty(queue)) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue"); + return result; + } + + if (StringUtils.isEmpty(queueName)) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName"); + return result; + } + Queue queueObj = queueMapper.selectById(id); if (queueObj == null) { putMsg(result, Status.QUEUE_NOT_EXIST, id); @@ -222,12 +232,12 @@ public class QueueService extends BaseService { Result result = new Result(); if (StringUtils.isEmpty(queue)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue"); return result; } if (StringUtils.isEmpty(queueName)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName"); return result; } From e9fcf6128e051a642808c7815d1ebc30aa32aad6 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Sun, 29 Dec 2019 09:59:49 +0800 Subject: [PATCH 2/7] AlertMappert UT modify #1465 (#1616) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove LogViewServiceGrpc.java file and pom modify * remove kazoo * remove kazoo * remove kazoo * remove common monitor package * add license * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * install.sh remove python kazoo * add system param whether repeat running * remove kazoo modify * BusinessTimeUtils remove whther repeat running inner param * add AccessTokenMapperTest UT * CI UT yml modify，start postgresql and zookeeper by default * add AlertGroupMapperTest UT in github action * Conflicts reslove * AlertMappert UT modify * AlertMappert UT modify * AlertMappert UT modify --- .../dolphinscheduler/dao/entity/Alert.java | 71 +++++++- .../dao/mapper/AlertMapperTest.java | 153 ++++++++++++------ pom.xml | 1 + 3 files changed, 177 insertions(+), 48 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java index d989f82fa2..dac96e8695 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java @@ -32,7 +32,6 @@ import java.util.Map; /** * alert */ -@Data @TableName("t_ds_alert") public class Alert { @@ -217,6 +216,72 @@ public class Alert { this.updateTime = updateTime; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Alert alert = (Alert) o; + + if (id != alert.id) { + return false; + } + if (alertGroupId != alert.alertGroupId) { + return false; + } + if (!title.equals(alert.title)) { + return false; + } + if (showType != alert.showType) { + return false; + } + if (!content.equals(alert.content)) { + return false; + } + if (alertType != alert.alertType) { + return false; + } + if (alertStatus != alert.alertStatus) { + return false; + } + if (!log.equals(alert.log)) { + return false; + } + if (!receivers.equals(alert.receivers)) { + return false; + } + if (!receiversCc.equals(alert.receiversCc)) { + return false; + } + if (!createTime.equals(alert.createTime)) { + return false; + } + return updateTime.equals(alert.updateTime) && info.equals(alert.info); + + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + title.hashCode(); + result = 31 * result + showType.hashCode(); + result = 31 * result + content.hashCode(); + result = 31 * result + alertType.hashCode(); + result = 31 * result + alertStatus.hashCode(); + result = 31 * result + log.hashCode(); + result = 31 * result + alertGroupId; + result = 31 * result + receivers.hashCode(); + result = 31 * result + receiversCc.hashCode(); + result = 31 * result + createTime.hashCode(); + result = 31 * result + updateTime.hashCode(); + result = 31 * result + info.hashCode(); + return result; + } + @Override public String toString() { return "Alert{" + @@ -228,10 +293,10 @@ public class Alert { ", alertStatus=" + alertStatus + ", log='" + log + '\'' + ", alertGroupId=" + alertGroupId + - ", createTime=" + createTime + - ", updateTime=" + updateTime + ", receivers='" + receivers + '\'' + ", receiversCc='" + receiversCc + '\'' + + ", createTime=" + createTime + + ", updateTime=" + updateTime + ", info=" + info + '}'; } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java index 958cd39c18..5f10a7586d 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java @@ -19,44 +19,52 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.Alert; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.List; - +import java.util.*; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; @RunWith(SpringRunner.class) @SpringBootTest +@Transactional +@Rollback(true) public class AlertMapperTest { @Autowired - AlertMapper alertMapper; + private AlertMapper alertMapper; /** - * insert - * @return Alert + * test insert + * @return */ - private Alert insertOne(){ - //insertOne - Alert alert = new Alert(); - alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]"); - alert.setLog("success"); - alert.setReceivers("xx@aa.com"); - alert.setAlertType(AlertType.EMAIL); - alert.setShowType(ShowType.TABLE); - alert.setAlertGroupId(1); - alert.setAlertStatus(AlertStatus.EXECUTION_SUCCESS); - alert.setCreateTime(new Date()); - alert.setUpdateTime(new Date()); - alertMapper.insert(alert); - return alert; + @Test + public void testInsert(){ + Alert expectedAlert = createAlert(); + assertNotNull(expectedAlert.getId()); + assertThat(expectedAlert.getId(), greaterThan(0)); + } + + + /** + * test select by id + * @return + */ + @Test + public void testSelectById(){ + Alert expectedAlert = createAlert(); + Alert actualAlert = alertMapper.selectById(expectedAlert.getId()); + assertEquals(expectedAlert, actualAlert); } /** @@ -64,13 +72,18 @@ public class AlertMapperTest { */ @Test public void testUpdate(){ - //insertOne - Alert alert = insertOne(); - //update - alert.setTitle("hello"); - int update = alertMapper.updateById(alert); - Assert.assertEquals(update, 1); - alertMapper.deleteById(alert.getId()); + + Alert expectedAlert = createAlert(); + + expectedAlert.setAlertStatus(AlertStatus.EXECUTION_FAILURE); + expectedAlert.setLog("error"); + expectedAlert.setUpdateTime(DateUtils.getCurrentDate()); + + alertMapper.updateById(expectedAlert); + + Alert actualAlert = alertMapper.selectById(expectedAlert.getId()); + + assertEquals(expectedAlert, actualAlert); } /** @@ -78,33 +91,83 @@ public class AlertMapperTest { */ @Test public void testDelete(){ + Alert expectedAlert = createAlert(); - Alert alert = insertOne(); - int delete = alertMapper.deleteById(alert.getId()); - Assert.assertEquals(delete, 1); + alertMapper.deleteById(expectedAlert.getId()); + + Alert actualAlert = alertMapper.selectById(expectedAlert.getId()); + + assertNull(actualAlert); } + /** - * test query + * test list alert by status */ @Test - public void testQuery() { - Alert alert = insertOne(); - //query - List alerts = alertMapper.selectList(null); - Assert.assertNotEquals(alerts.size(), 0); - alertMapper.deleteById(alert.getId()); + public void testListAlertByStatus() { + Integer count = 10; + AlertStatus waitExecution = AlertStatus.WAIT_EXECUTION; + + Map expectedAlertMap = createAlertMap(count, waitExecution); + + List actualAlerts = alertMapper.listAlertByStatus(waitExecution); + + for (Alert actualAlert : actualAlerts){ + Alert expectedAlert = expectedAlertMap.get(actualAlert.getId()); + if (expectedAlert != null){ + assertEquals(expectedAlert,actualAlert); + } + } } /** - * test list alert by status + * create alert map + * @param count alert count + * @param alertStatus alert status + * @return alert map */ - @Test - public void testListAlertByStatus() { - Alert alert = insertOne(); - //query - List alerts = alertMapper.listAlertByStatus(AlertStatus.EXECUTION_SUCCESS); - Assert.assertNotEquals(alerts.size(), 0); - alertMapper.deleteById(alert.getId()); + private Map createAlertMap(Integer count,AlertStatus alertStatus){ + Map alertMap = new HashMap<>(); + + for (int i = 0 ; i < count ;i++){ + Alert alert = createAlert(alertStatus); + alertMap.put(alert.getId(),alert); + } + + return alertMap; + + } + + + /** + * create alert + * @return alert + * @throws Exception + */ + private Alert createAlert(){ + return createAlert(AlertStatus.WAIT_EXECUTION); + } + + /** + * create alert + * @param alertStatus alert status + * @return alert + */ + private Alert createAlert(AlertStatus alertStatus){ + Alert alert = new Alert(); + alert.setShowType(ShowType.TABLE); + alert.setTitle("test alert"); + alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]"); + alert.setAlertType(AlertType.EMAIL); + alert.setAlertStatus(alertStatus); + alert.setLog("success"); + alert.setReceivers("aa@aa.com"); + alert.setReceiversCc("bb@aa.com"); + alert.setCreateTime(DateUtils.getCurrentDate()); + alert.setUpdateTime(DateUtils.getCurrentDate()); + + alertMapper.insert(alert); + return alert; } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 69f1ff0665..796d2b8228 100644 --- a/pom.xml +++ b/pom.xml @@ -674,6 +674,7 @@ **/server/utils/FlinkArgsUtilsTest.java **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java + **/dao/mapper/AlertMapperTest.java From 2a0dab14ee56015f692208a4ef92900d6deec605 Mon Sep 17 00:00:00 2001 From: Tboy Date: Sun, 29 Dec 2019 10:11:41 +0800 Subject: [PATCH 3/7] we should insert alert DB once , and trigger this type of alert 3 times (#1622) --- .../apache/dolphinscheduler/server/master/MasterServer.java | 6 ++---- .../apache/dolphinscheduler/server/worker/WorkerServer.java | 4 +--- .../apache/dolphinscheduler/server/zk/ZKMasterClient.java | 6 ++---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index e8c8b6779e..1f5b16a25a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -152,10 +152,8 @@ public class MasterServer implements IStoppable { @Override public void run() { if (zkMasterClient.getActiveMasterNum() <= 1) { - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - zkMasterClient.getAlertDao().sendServerStopedAlert( - 1, OSUtils.getHost(), "Master-Server"); - } + zkMasterClient.getAlertDao().sendServerStopedAlert( + 1, OSUtils.getHost(), "Master-Server"); } stop("shutdownhook"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index fe9c4991ac..ea8330e18a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -177,9 +177,7 @@ public class WorkerServer implements IStoppable { public void run() { // worker server exit alert if (zkWorkerClient.getActiveMasterNum() <= 1) { - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); - } + alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); } stop("shutdownhook"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index a26a217665..1c7b737f5b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -247,10 +247,8 @@ public class ZKMasterClient extends AbstractZKClient { */ private void alertServerDown(String serverHost, ZKNodeType zkNodeType) { - String serverType = zkNodeType.toString(); - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - alertDao.sendServerStopedAlert(1, serverHost, serverType); - } + String serverType = zkNodeType.toString(); + alertDao.sendServerStopedAlert(1, serverHost, serverType); } /** From b03632bc6de211935a3531cd0ee3a08b14802666 Mon Sep 17 00:00:00 2001 From: samz406 Date: Sun, 29 Dec 2019 10:13:14 +0800 Subject: [PATCH 4/7] User update not check params (#1619) * misspell words * modify common queue TaskQueueZKImplTest.java unit test * extends BaseTaskQueueTest get zkServer * modify zk config * User update not check params * user phone update when noteEmpty --- .../api/service/UsersService.java | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index 92319e71a1..a4664b6c3e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -221,9 +221,13 @@ public class UsersService extends BaseService { return result; } - Date now = new Date(); - if (StringUtils.isNotEmpty(userName)) { + + if (!CheckUtils.checkUserName(userName)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userName); + return result; + } + User tempUser = userMapper.queryByUserNameAccurately(userName); if (tempUser != null && tempUser.getId() != userId) { putMsg(result, Status.USER_NAME_EXIST); @@ -233,14 +237,30 @@ public class UsersService extends BaseService { } if (StringUtils.isNotEmpty(userPassword)) { + if (!CheckUtils.checkPassword(userPassword)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userPassword); + return result; + } user.setUserPassword(EncryptionUtils.getMd5(userPassword)); } if (StringUtils.isNotEmpty(email)) { + if (!CheckUtils.checkEmail(email)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,email); + return result; + } user.setEmail(email); } + + if (StringUtils.isNotEmpty(phone)) { + if (!CheckUtils.checkPhone(phone)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,phone); + return result; + } + user.setPhone(phone); + } user.setQueue(queue); - user.setPhone(phone); + Date now = new Date(); user.setUpdateTime(now); //if user switches the tenant, the user's resources need to be copied to the new tenant @@ -318,10 +338,15 @@ public class UsersService extends BaseService { Map result = new HashMap<>(5); //only admin can operate if (!isAdmin(loginUser)) { + putMsg(result, Status.USER_NO_OPERATION_PERM, id); + return result; + } + //check exist + User tempUser = userMapper.selectById(id); + if (tempUser == null) { putMsg(result, Status.USER_NOT_EXIST, id); return result; } - // delete user User user = userMapper.queryTenantCodeByUserId(id); @@ -357,6 +382,12 @@ public class UsersService extends BaseService { return result; } + //check exist + User tempUser = userMapper.selectById(userId); + if (tempUser == null) { + putMsg(result, Status.USER_NOT_EXIST, userId); + return result; + } //if the selected projectIds are empty, delete all items associated with the user projectUserMapper.deleteProjectRelation(0, userId); @@ -443,6 +474,11 @@ public class UsersService extends BaseService { if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; } + User user = userMapper.selectById(userId); + if(user == null){ + putMsg(result, Status.USER_NOT_EXIST, userId); + return result; + } udfUserMapper.deleteByUserId(userId); @@ -485,6 +521,11 @@ public class UsersService extends BaseService { if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; } + User user = userMapper.selectById(userId); + if(user == null){ + putMsg(result, Status.USER_NOT_EXIST, userId); + return result; + } datasourceUserMapper.deleteByUserId(userId); From 779decc96dd2e832f9cd14b8a0f44b483080683a Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Sun, 29 Dec 2019 10:23:10 +0800 Subject: [PATCH 5/7] =?UTF-8?q?Increase=20script=20text=20box=20to=20zoom?= =?UTF-8?q?=20in=20and=20fix=20log=20loading=20#1547=20=E3=80=81#1247=20(#?= =?UTF-8?q?1604)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Increase script text box to zoom in and fix log loading * Timing operation adds default value for start and end dates --- .../home/pages/dag/_source/formModel/log.vue | 8 +- .../formModel/tasks/_source/scriptBox.vue | 124 ++++++++++++++++++ .../dag/_source/formModel/tasks/shell.vue | 64 ++++++++- .../definition/pages/list/_source/timing.vue | 16 +++ 4 files changed, 202 insertions(+), 10 deletions(-) create mode 100644 dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue index 4c1acff5e6..7ec9cdfa7a 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue @@ -257,8 +257,8 @@ _rtParam () { return { taskInstId: this.stateId || this.logId, - skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '0000' : 0}`), - limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}0000`) + skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '000' : 0}`), + limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}000`) } } }, @@ -316,7 +316,7 @@ } .refresh-log { >i { - font-size: 24px; + font-size: 20px; vertical-align: middle; transform: scale(1); } @@ -353,7 +353,7 @@ font-weight: bold; resize:none; line-height: 1.6; - padding: 6px; + padding: 0px; } } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue new file mode 100644 index 0000000000..c16d18cc38 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 230b43d446..8b45e7c3d1 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -21,10 +21,13 @@
+ + +
@@ -55,6 +58,7 @@ import _ from 'lodash' import i18n from '@/module/i18n' import mListBox from './_source/listBox' + import mScriptBox from './_source/scriptBox' import mResources from './_source/resources' import mLocalParams from './_source/localParams' import disabledState from '@/module/mixin/disabledState' @@ -85,8 +89,37 @@ _onLocalParams (a) { this.localParams = a }, + setEditorVal() { + let self = this + let modal = self.$modal.dialog({ + className: 'scriptModal', + closable: false, + showMask: true, + maskClosable: true, + onClose: function() { + + }, + render (h) { + return h(mScriptBox, { + on: { + getSriptBoxValue (val) { + editor.setValue(val) + }, + closeAble () { + // this.$modal.destroy() + modal.remove() + } + }, + props: { + item: editor.getValue() + } + }) + } + }) + }, /** * return resourceList + * */ _onResourcesData (a) { this.resourceList = a @@ -109,7 +142,6 @@ if (!this.$refs.refLocalParams._verifProp()) { return false } - // storage this.$emit('on-params', { resourceList: this.resourceList, @@ -138,7 +170,6 @@ // Monitor keyboard editor.on('keypress', this.keypress) - editor.setValue(this.rawScript) return editor @@ -176,6 +207,27 @@ editor.off($('.code-shell-mirror'), 'keypress', this.keypress) } }, - components: { mLocalParams, mListBox, mResources } + components: { mLocalParams, mListBox, mResources, mScriptBox } } + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index 6ed78980f1..a15999edbd 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -313,7 +313,23 @@ this.crontab = this.item.crontab } if(this.type == 'timing') { + let date = new Date() + let year = date.getFullYear() + let month = date.getMonth() + 1 + let day = date.getDate() + if (month < 10) { + month = "0" + month; + } + if (day < 10) { + day = "0" + day; + } + let startDate = year + "-" + month + "-" + day + ' ' + '00:00:00' + let endDate = (year+100) + "-" + month + "-" + day + ' ' + '00:00:00' + let times = [] + times[0] = startDate + times[1] = endDate this.crontab = '0 0 * * * ? *' + this.scheduleTime = times } this.receivers = _.cloneDeep(this.receiversD) this.receiversCc = _.cloneDeep(this.receiversCcD) From e61586cd0665f94618d0d0489052dc87a1aea61c Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Sun, 29 Dec 2019 13:12:29 +0800 Subject: [PATCH 6/7] CommandMapperTest UT modify #1465 (#1625) * CommandMapperTest UT modify --- .../dolphinscheduler/dao/entity/Command.java | 74 ++++- .../dao/entity/CommandCount.java | 40 ++- .../dao/mapper/AlertMapperTest.java | 3 + .../dao/mapper/CommandMapperTest.java | 264 ++++++++++++------ pom.xml | 1 + 5 files changed, 292 insertions(+), 90 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java index a180a433bc..2d47b7874a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java @@ -28,7 +28,6 @@ import java.util.Date; /** * command */ -@Data @TableName("t_ds_command") public class Command { @@ -265,6 +264,79 @@ public class Command { this.workerGroupId = workerGroupId; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Command command = (Command) o; + + if (id != command.id) { + return false; + } + if (processDefinitionId != command.processDefinitionId) { + return false; + } + if (executorId != command.executorId) { + return false; + } + if (workerGroupId != command.workerGroupId) { + return false; + } + if (commandType != command.commandType) { + return false; + } + if (commandParam != null ? !commandParam.equals(command.commandParam) : command.commandParam != null) { + return false; + } + if (taskDependType != command.taskDependType) { + return false; + } + if (failureStrategy != command.failureStrategy) { + return false; + } + if (warningType != command.warningType) { + return false; + } + if (warningGroupId != null ? !warningGroupId.equals(command.warningGroupId) : command.warningGroupId != null) { + return false; + } + if (scheduleTime != null ? !scheduleTime.equals(command.scheduleTime) : command.scheduleTime != null) { + return false; + } + if (startTime != null ? !startTime.equals(command.startTime) : command.startTime != null) { + return false; + } + if (processInstancePriority != command.processInstancePriority) { + return false; + } + return !(updateTime != null ? !updateTime.equals(command.updateTime) : command.updateTime != null); + + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + (commandType != null ? commandType.hashCode() : 0); + result = 31 * result + processDefinitionId; + result = 31 * result + executorId; + result = 31 * result + (commandParam != null ? commandParam.hashCode() : 0); + result = 31 * result + (taskDependType != null ? taskDependType.hashCode() : 0); + result = 31 * result + (failureStrategy != null ? failureStrategy.hashCode() : 0); + result = 31 * result + (warningType != null ? warningType.hashCode() : 0); + result = 31 * result + (warningGroupId != null ? warningGroupId.hashCode() : 0); + result = 31 * result + (scheduleTime != null ? scheduleTime.hashCode() : 0); + result = 31 * result + (startTime != null ? startTime.hashCode() : 0); + result = 31 * result + (processInstancePriority != null ? processInstancePriority.hashCode() : 0); + result = 31 * result + (updateTime != null ? updateTime.hashCode() : 0); + result = 31 * result + workerGroupId; + return result; + } + @Override public String toString() { return "Command{" + diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java index 158169b727..df99b54dbe 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java @@ -33,13 +33,6 @@ public class CommandCount { private int count; - @Override - public String toString(){ - return "command count:" + - " commandType: "+ commandType.toString() + - " count: "+ count; - } - public CommandType getCommandType() { return commandType; } @@ -55,4 +48,37 @@ public class CommandCount { public void setCount(int count) { this.count = count; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + CommandCount that = (CommandCount) o; + + if (count != that.count) { + return false; + } + return commandType == that.commandType; + + } + + @Override + public int hashCode() { + int result = commandType != null ? commandType.hashCode() : 0; + result = 31 * result + count; + return result; + } + + @Override + public String toString() { + return "CommandCount{" + + "commandType=" + commandType + + ", count=" + count + + '}'; + } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java index 5f10a7586d..4e298e3aa3 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java @@ -35,6 +35,9 @@ import java.util.*; import static org.hamcrest.Matchers.*; import static org.junit.Assert.*; +/** + * alert mapper test + */ @RunWith(SpringRunner.class) @SpringBootTest @Transactional diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 2af44576b5..c35ce7e8ce 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.CommandCount; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; @@ -25,13 +26,25 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; + +/** + * command mapper test + */ @RunWith(SpringRunner.class) @SpringBootTest +@Transactional +@Rollback(true) public class CommandMapperTest { @@ -41,71 +54,85 @@ public class CommandMapperTest { @Autowired ProcessDefinitionMapper processDefinitionMapper; + /** - * insert - * @return Command + * test insert */ - private Command insertOne(){ - //insertOne - Command command = new Command(); - command.setCommandType(CommandType.START_PROCESS); - command.setProcessDefinitionId(1); - command.setExecutorId(4); - command.setProcessInstancePriority(Priority.MEDIUM); - command.setFailureStrategy(FailureStrategy.CONTINUE); - command.setWorkerGroupId(-1); - command.setWarningGroupId(1); - command.setUpdateTime(new Date()); - commandMapper.insert(command); - return command; + @Test + public void testInsert(){ + Command command = createCommand(); + assertNotNull(command.getId()); + assertThat(command.getId(),greaterThan(0)); } + /** - * test update + * test select by id */ @Test - public void testUpdate(){ - //insertOne - Command command = insertOne(); - //update - command.setStartTime(new Date()); - int update = commandMapper.updateById(command); - Assert.assertEquals(update, 1); - commandMapper.deleteById(command.getId()); + public void testSelectById() { + Command expectedCommand = createCommand(); + //query + Command actualCommand = commandMapper.selectById(expectedCommand.getId()); + + assertEquals(expectedCommand, actualCommand); } /** - * test delete + * test update */ @Test - public void testDelete(){ + public void testUpdate(){ + + Command expectedCommand = createCommand(); + + // update the command time if current command if recover from waiting + expectedCommand.setUpdateTime(DateUtils.getCurrentDate()); + + commandMapper.updateById(expectedCommand); + + Command actualCommand = commandMapper.selectById(expectedCommand.getId()); + + assertEquals(expectedCommand,actualCommand); - Command Command = insertOne(); - int delete = commandMapper.deleteById(Command.getId()); - Assert.assertEquals(delete, 1); } /** - * test query + * test delete */ @Test - public void testQuery() { - Command command = insertOne(); - //query - List commands = commandMapper.selectList(null); - Assert.assertNotEquals(commands.size(), 0); - commandMapper.deleteById(command.getId()); + public void testDelete(){ + Command expectedCommand = createCommand(); + + commandMapper.deleteById(expectedCommand.getId()); + + Command actualCommand = commandMapper.selectById(expectedCommand.getId()); + + assertNull(actualCommand); } + + /** * test query all */ @Test public void testGetAll() { - Command command = insertOne(); - List commands = commandMapper.selectList(null); - Assert.assertNotEquals(commands.size(), 0); - commandMapper.deleteById(command.getId()); + Integer count = 10; + + Map commandMap = createCommandMap(count); + + + List actualCommands = commandMapper.selectList(null); + + assertThat(actualCommands.size(), greaterThanOrEqualTo(count)); + + for (Command actualCommand : actualCommands){ + Command expectedCommand = commandMap.get(actualCommand.getId()); + if (expectedCommand != null){ + assertEquals(expectedCommand,actualCommand); + } + } } /** @@ -113,28 +140,14 @@ public class CommandMapperTest { */ @Test public void testGetOneToRun() { - ProcessDefinition processDefinition = new ProcessDefinition(); - processDefinition.setReleaseState(ReleaseState.ONLINE); - processDefinition.setName("ut test"); - processDefinition.setProjectId(1); - processDefinition.setFlag(Flag.YES); - processDefinitionMapper.insert(processDefinition); - Command command = new Command(); - command.setCommandType(CommandType.START_PROCESS); - command.setProcessDefinitionId(processDefinition.getId()); - command.setExecutorId(4); - command.setProcessInstancePriority(Priority.MEDIUM); - command.setFailureStrategy(FailureStrategy.CONTINUE); - command.setWorkerGroupId(-1); - command.setWarningGroupId(1); - command.setUpdateTime(new Date()); - commandMapper.insert(command); + ProcessDefinition processDefinition = createProcessDefinition(); + + Command expectedCommand = createCommand(CommandType.START_PROCESS,processDefinition.getId()); - Command command2 = commandMapper.getOneToRun(); - Assert.assertNotEquals(command2, null); - commandMapper.deleteById(command.getId()); - processDefinitionMapper.deleteById(processDefinition.getId()); + Command actualCommand = commandMapper.getOneToRun(); + + assertEquals(expectedCommand, actualCommand); } /** @@ -142,35 +155,122 @@ public class CommandMapperTest { */ @Test public void testCountCommandState() { - Command command = insertOne(); + Integer count = 10; + + ProcessDefinition processDefinition = createProcessDefinition(); + + CommandCount expectedCommandCount = createCommandMap(count, CommandType.START_PROCESS, processDefinition.getId()); + + Integer[] projectIdArray = {processDefinition.getProjectId()}; + + Date startTime = DateUtils.stringToDate("2019-12-29 00:10:00"); + + Date endTime = DateUtils.stringToDate("2019-12-29 23:59:59"); + + List actualCommandCounts = commandMapper.countCommandState(0, startTime, endTime, projectIdArray); + + assertThat(actualCommandCounts.size(),greaterThanOrEqualTo(1)); + + Boolean flag = false; + for (CommandCount actualCommandCount : actualCommandCounts){ + if (actualCommandCount.getCommandType().equals(expectedCommandCount.getCommandType())){ + assertEquals(expectedCommandCount,actualCommandCount); + flag = true; + } + } + + assertTrue(flag); + } - //insertOne + + /** + * create command map + * @param count map count + * @param commandType comman type + * @param processDefinitionId process definition id + * @return command map + */ + private CommandCount createCommandMap( + Integer count, + CommandType commandType, + Integer processDefinitionId){ + + CommandCount commandCount = new CommandCount(); + + for (int i = 0 ;i < count ;i++){ + createCommand(commandType,processDefinitionId); + } + commandCount.setCommandType(commandType); + commandCount.setCount(count); + + return commandCount; + } + + /** + * create process definition + * @return process definition + */ + private ProcessDefinition createProcessDefinition(){ ProcessDefinition processDefinition = new ProcessDefinition(); - processDefinition.setName("def 1"); - processDefinition.setProjectId(1010); - processDefinition.setUserId(101); - processDefinition.setUpdateTime(new Date()); - processDefinition.setCreateTime(new Date()); + processDefinition.setReleaseState(ReleaseState.ONLINE); + processDefinition.setName("ut test"); + processDefinition.setProjectId(1); + processDefinition.setFlag(Flag.YES); + processDefinitionMapper.insert(processDefinition); - command.setProcessDefinitionId(processDefinition.getId()); - commandMapper.updateById(command); + return processDefinition; + } + /** + * create command map + * @param count map count + * @return command map + */ + private Map createCommandMap(Integer count){ + Map commandMap = new HashMap<>(); + + for (int i = 0; i < count ;i++){ + Command command = createCommand(); + commandMap.put(command.getId(),command); + } + return commandMap; + } - List commandCounts = commandMapper.countCommandState( - 4, null, null, new Integer[0] - ); - Integer[] projectIdArray = new Integer[2]; - projectIdArray[0] = processDefinition.getProjectId(); - projectIdArray[1] = 200; - List commandCounts2 = commandMapper.countCommandState( - 4, null, null, projectIdArray - ); + /** + * create command + * @return + */ + private Command createCommand(){ + return createCommand(CommandType.START_PROCESS,1); + } + + /** + * create command + * @return Command + */ + private Command createCommand(CommandType commandType,Integer processDefinitionId){ + + Command command = new Command(); + command.setCommandType(commandType); + command.setProcessDefinitionId(processDefinitionId); + command.setExecutorId(4); + command.setCommandParam("test command param"); + command.setTaskDependType(TaskDependType.TASK_ONLY); + command.setFailureStrategy(FailureStrategy.CONTINUE); + command.setWarningType(WarningType.ALL); + command.setWarningGroupId(1); + command.setScheduleTime(DateUtils.stringToDate("2019-12-29 12:10:00")); + command.setProcessInstancePriority(Priority.MEDIUM); + command.setStartTime(DateUtils.stringToDate("2019-12-29 10:10:00")); + command.setUpdateTime(DateUtils.stringToDate("2019-12-29 10:10:00")); + command.setWorkerGroupId(-1); + commandMapper.insert(command); - commandMapper.deleteById(command.getId()); - processDefinitionMapper.deleteById(processDefinition.getId()); - Assert.assertNotEquals(commandCounts.size(), 0); - Assert.assertNotEquals(commandCounts2.size(), 0); + return command; } + + + } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 796d2b8228..7936bf7dc8 100644 --- a/pom.xml +++ b/pom.xml @@ -675,6 +675,7 @@ **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/AlertMapperTest.java + **/dao/mapper/CommandMapperTest.java From d3d5726aebdeab98e10abbff725ca7be9be1a911 Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Sun, 29 Dec 2019 13:15:29 +0800 Subject: [PATCH 7/7] Add github action to analyze code with sonarcloud (#1624) * Create .sonarcloud.properties * add License description --- .github/workflows/SonarCloud.yml | 51 ++++++++++++++++++++++++++++++++ README.md | 3 +- README_zh_CN.md | 2 +- 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/SonarCloud.yml diff --git a/.github/workflows/SonarCloud.yml b/.github/workflows/SonarCloud.yml new file mode 100644 index 0000000000..fd5b0585db --- /dev/null +++ b/.github/workflows/SonarCloud.yml @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: SonarCloud +on: [push, pull_request] +jobs: + sonarCloudTrigger: + name: SonarCloud Trigger + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - uses: actions/cache@v1 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube + restore-keys: | + ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube + ${{ runner.os }}-maven- + - name: Maven clean + run: mvn clean + - name: Run SonarCloud analyse + run: > + mvn clean --batch-mode + org.jacoco:jacoco-maven-plugin:prepare-agent + verify + org.sonarsource.scanner.maven:sonar-maven-plugin:sonar + -Dmaven.test.skip=true + -Dsonar.host.url=https://sonarcloud.io + -Dsonar.organization=apache + -Dsonar.projectKey=apache-dolphinscheduler + -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/README.md b/README.md index aee0e3b2e3..bc7dba684f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ Dolphin Scheduler Official Website [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) -![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) + > Dolphin Scheduler for Big Data diff --git a/README_zh_CN.md b/README_zh_CN.md index f8855aca6c..8141763c33 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -4,7 +4,7 @@ Dolphin Scheduler Official Website [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) -![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) > Dolphin Scheduler for Big Data