From c49626b877d587157fc76be58f9349ae27e94832 Mon Sep 17 00:00:00 2001 From: muzhongjiang Date: Sun, 12 Jul 2020 22:20:14 +0800 Subject: [PATCH 1/5] Word spelling modification,Comment modification,Log level modification (#3180) * add state * fixed bug "jackson enum conversion : InvalidFormatException" * Word spelling modification Comment modification Word spelling modification,Comment modification,Log level modification * Update EmailManager.java * Update FlinkParameters.java * Update SqlTask.java Co-authored-by: mzjnumber1@163.com Co-authored-by: dailidong --- .../alert/manager/EmailManager.java | 15 ++++++++------- .../alert/plugin/EmailAlertPlugin.java | 14 +++++++------- .../alert/runner/AlertSender.java | 10 +++++----- .../common/task/flink/FlinkParameters.java | 5 +---- .../server/worker/task/sql/SqlTask.java | 16 +++++++++------- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java index 96feb7f09e..22f4b7a27f 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java @@ -27,28 +27,29 @@ import java.util.Map; public class EmailManager { /** * email send - * @param receviersList the receiver list - * @param receviersCcList the cc List + * @param receiversList the receiver list + * @param receiversCcList the cc List * @param title the title * @param content the content * @param showType the showType * @return the send result */ - public Map send(List receviersList,List receviersCcList,String title,String content,String showType){ + public Map send(List receiversList,List receiversCcList,String title,String content,String showType){ - return MailUtils.sendMails(receviersList, receviersCcList, title, content, showType); + return MailUtils.sendMails(receiversList, receiversCcList, title, content, showType); } /** * msg send - * @param receviersList the receiver list + * @param receiversList the receiver list * @param title the title * @param content the content * @param showType the showType * @return the send result */ - public Map send(List receviersList,String title,String content,String showType){ + public Map send(List receiversList,String title,String content,String showType){ - return MailUtils.sendMails(receviersList,title, content, showType); + return MailUtils.sendMails(receiversList,title, content, showType); } + } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java index d06e12d941..fbc600f39e 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java @@ -71,32 +71,32 @@ public class EmailAlertPlugin implements AlertPlugin { AlertData alert = info.getAlertData(); - List receviersList = (List) info.getProp(Constants.PLUGIN_DEFAULT_EMAIL_RECEIVERS); + List receiversList = (List) info.getProp(Constants.PLUGIN_DEFAULT_EMAIL_RECEIVERS); // receiving group list // custom receiver String receivers = alert.getReceivers(); if (StringUtils.isNotEmpty(receivers)) { String[] splits = receivers.split(","); - receviersList.addAll(Arrays.asList(splits)); + receiversList.addAll(Arrays.asList(splits)); } - List receviersCcList = new ArrayList<>(); + List receiversCcList = new ArrayList<>(); // Custom Copier String receiversCc = alert.getReceiversCc(); if (StringUtils.isNotEmpty(receiversCc)) { String[] splits = receiversCc.split(","); - receviersCcList.addAll(Arrays.asList(splits)); + receiversCcList.addAll(Arrays.asList(splits)); } - if (CollectionUtils.isEmpty(receviersList) && CollectionUtils.isEmpty(receviersCcList)) { + if (CollectionUtils.isEmpty(receiversList) && CollectionUtils.isEmpty(receiversCcList)) { logger.warn("alert send error : At least one receiver address required"); retMaps.put(Constants.STATUS, "false"); retMaps.put(Constants.MESSAGE, "execution failure,At least one receiver address required."); return retMaps; } - retMaps = emailManager.send(receviersList, receviersCcList, alert.getTitle(), alert.getContent(), + retMaps = emailManager.send(receiversList, receiversCcList, alert.getTitle(), alert.getContent(), alert.getShowType()); //send flag @@ -124,7 +124,7 @@ public class EmailAlertPlugin implements AlertPlugin { logger.error(e.getMessage(), e); } } - + if (DingTalkUtils.isEnableDingTalk) { logger.info("Ding Talk is enable."); dingTalkManager.send(info); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java index 122f1bf02d..1bae9c7724 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java @@ -60,9 +60,9 @@ public class AlertSender { users = alertDao.listUserByAlertgroupId(alert.getAlertGroupId()); // receiving group list - List receviersList = new ArrayList<>(); + List receiversList = new ArrayList<>(); for (User user : users) { - receviersList.add(user.getEmail()); + receiversList.add(user.getEmail()); } AlertData alertData = new AlertData(); @@ -78,17 +78,17 @@ public class AlertSender { AlertInfo alertInfo = new AlertInfo(); alertInfo.setAlertData(alertData); - alertInfo.addProp("receivers", receviersList); + alertInfo.addProp("receivers", receiversList); AlertPlugin emailPlugin = pluginManager.findOne(Constants.PLUGIN_DEFAULT_EMAIL_ID); retMaps = emailPlugin.process(alertInfo); if (retMaps == null) { alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, "alert send error", alert.getId()); - logger.info("alert send error : return value is null"); + logger.error("alert send error : return value is null"); } else if (!Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS)))) { alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, String.valueOf(retMaps.get(Constants.MESSAGE)), alert.getId()); - logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); + logger.error("alert send error : {}", retMaps.get(Constants.MESSAGE)); } else { alertDao.updateAlert(AlertStatus.EXECUTION_SUCCESS, (String) retMaps.get(Constants.MESSAGE), alert.getId()); logger.info("alert send success"); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java index b89a920620..231dd33146 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java @@ -19,13 +19,12 @@ package org.apache.dolphinscheduler.common.task.flink; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; import java.util.ArrayList; import java.util.List; /** - * spark parameters + * flink parameters */ public class FlinkParameters extends AbstractParameters { @@ -226,6 +225,4 @@ public class FlinkParameters extends AbstractParameters { } return resourceList; } - - } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 5511c6acd6..0afeb8a3c8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -176,6 +176,7 @@ public class SqlTask extends AbstractTask { logger.info("SQL title : {}",title); sqlParameters.setTitle(title); } + //new //replace variable TIME with $[YYYYmmddd...] in sql when history run job and batch complement job sql = ParameterUtils.replaceScheduleTime(sql, taskExecutionContext.getScheduleTime()); @@ -324,6 +325,7 @@ public class SqlTask extends AbstractTask { } } } + /** * create connection * @@ -423,34 +425,34 @@ public class SqlTask extends AbstractTask { List users = alertDao.queryUserByAlertGroupId(taskExecutionContext.getSqlTaskExecutionContext().getWarningGroupId()); // receiving group list - List receviersList = new ArrayList<>(); + List receiversList = new ArrayList<>(); for(User user:users){ - receviersList.add(user.getEmail().trim()); + receiversList.add(user.getEmail().trim()); } // custom receiver String receivers = sqlParameters.getReceivers(); if (StringUtils.isNotEmpty(receivers)){ String[] splits = receivers.split(COMMA); for (String receiver : splits){ - receviersList.add(receiver.trim()); + receiversList.add(receiver.trim()); } } // copy list - List receviersCcList = new ArrayList<>(); + List receiversCcList = new ArrayList<>(); // Custom Copier String receiversCc = sqlParameters.getReceiversCc(); if (StringUtils.isNotEmpty(receiversCc)){ String[] splits = receiversCc.split(COMMA); for (String receiverCc : splits){ - receviersCcList.add(receiverCc.trim()); + receiversCcList.add(receiverCc.trim()); } } String showTypeName = sqlParameters.getShowType().replace(COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ - Map mailResult = MailUtils.sendMails(receviersList, - receviersCcList, title, content, ShowType.valueOf(showTypeName).getDescp()); + Map mailResult = MailUtils.sendMails(receiversList, + receiversCcList, title, content, ShowType.valueOf(showTypeName).getDescp()); if(!(boolean) mailResult.get(STATUS)){ throw new RuntimeException("send mail failed!"); } From 933a67db8dde3585a2bb5a1ab2b8db67910b3d9d Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Sun, 12 Jul 2020 22:25:25 +0800 Subject: [PATCH 2/5] [Improvement][Server]Code Clean (#3196) * [*][Server]Code Clean redundant code removal * redundant code removal * redundant code removal --- .../server/registry/ZookeeperNodeManager.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java index 278da60867..f039fb5532 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java @@ -131,16 +131,12 @@ public class ZookeeperNodeManager implements InitializingBean { if (event.getType() == TreeCacheEvent.Type.NODE_ADDED) { logger.info("worker group node : {} added.", path); String group = parseGroup(path); - Set workerNodes = workerGroupNodes.getOrDefault(group, new HashSet<>()); - Set previousNodes = new HashSet<>(workerNodes); Set currentNodes = registryCenter.getWorkerGroupNodesDirectly(group); logger.info("currentNodes : {}", currentNodes); syncWorkerGroupNodes(group, currentNodes); } else if (event.getType() == TreeCacheEvent.Type.NODE_REMOVED) { logger.info("worker group node : {} down.", path); String group = parseGroup(path); - Set workerNodes = workerGroupNodes.getOrDefault(group, new HashSet<>()); - Set previousNodes = new HashSet<>(workerNodes); Set currentNodes = registryCenter.getWorkerGroupNodesDirectly(group); syncWorkerGroupNodes(group, currentNodes); alertDao.sendServerStopedAlert(1, path, "WORKER"); @@ -175,12 +171,10 @@ public class ZookeeperNodeManager implements InitializingBean { try { if (event.getType() == TreeCacheEvent.Type.NODE_ADDED) { logger.info("master node : {} added.", path); - Set previousNodes = new HashSet<>(masterNodes); Set currentNodes = registryCenter.getMasterNodesDirectly(); syncMasterNodes(currentNodes); } else if (event.getType() == TreeCacheEvent.Type.NODE_REMOVED) { logger.info("master node : {} down.", path); - Set previousNodes = new HashSet<>(masterNodes); Set currentNodes = registryCenter.getMasterNodesDirectly(); syncMasterNodes(currentNodes); alertDao.sendServerStopedAlert(1, path, "MASTER"); From 5868af89478cda9a0f87a82340c6bc17800d5ba6 Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Mon, 13 Jul 2020 08:48:59 +0800 Subject: [PATCH 3/5] [Hotfix-3131][api] Fix the new tenant already exists prompt (#3132) * Bugfix: Fix the new tenant already exists prompt * Feature: Add test cases * Update TenantServiceTest.java Co-authored-by: dailidong Co-authored-by: qiaozhanwei --- .../dolphinscheduler/api/enums/Status.java | 2 +- .../api/service/TenantService.java | 6 ++-- .../api/controller/TenantControllerTest.java | 16 ++++++++++ .../api/service/TenantServiceTest.java | 30 ++++++++++++------- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 62881796a2..ec0b8f0a76 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -35,7 +35,7 @@ public enum Status { USER_NAME_NULL(10004,"user name is null", "用户名不能为空"), HDFS_OPERATION_ERROR(10006, "hdfs operation error", "hdfs操作错误"), TASK_INSTANCE_NOT_FOUND(10008, "task instance not found", "任务实例不存在"), - TENANT_NAME_EXIST(10009, "tenant code already exists", "租户编码不能为空"), + TENANT_NAME_EXIST(10009, "tenant code {0} already exists", "租户编码[{0}]已存在"), USER_NOT_EXIST(10010, "user {0} not exists", "用户[{0}]不存在"), ALERT_GROUP_NOT_EXIST(10011, "alarm group not found", "告警组不存在"), ALERT_GROUP_EXIST(10012, "alarm group already exists", "告警组名称已存在"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java index 2fded4d32f..a78c951d34 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java @@ -327,11 +327,11 @@ public class TenantService extends BaseService{ * @return true if tenant code can user, otherwise return false */ public Result verifyTenantCode(String tenantCode) { - Result result=new Result(); + Result result = new Result(); if (checkTenantExists(tenantCode)) { logger.error("tenant {} has exist, can't create again.", tenantCode); - putMsg(result, Status.TENANT_NAME_EXIST); - }else{ + putMsg(result, Status.TENANT_NAME_EXIST, tenantCode); + } else { putMsg(result, Status.SUCCESS); } return result; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java index 578d6edee1..7cf622ab9f 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java @@ -120,7 +120,23 @@ public class TenantControllerTest extends AbstractControllerTest{ } + @Test + public void testVerifyTenantCodeExists() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("tenantCode", "tenantCode"); + MvcResult mvcResult = mockMvc.perform(get("/tenant/verify-tenant-code") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertEquals(Status.TENANT_NAME_EXIST.getCode(), result.getCode().intValue()); + logger.info(mvcResult.getResponse().getContentAsString()); + + } @Test public void testQueryTenantlist() throws Exception { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index d6fb6b219c..f7f506b69b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -16,8 +16,11 @@ */ package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; @@ -41,10 +44,10 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.i18n.LocaleContextHolder; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(MockitoJUnitRunner.class) public class TenantServiceTest { @@ -61,8 +64,8 @@ public class TenantServiceTest { @Mock private UserMapper userMapper; - private String tenantCode ="TenantServiceTest"; - private String tenantName ="TenantServiceTest"; + private String tenantCode = "TenantServiceTest"; + private String tenantName = "TenantServiceTest"; @Test @@ -85,6 +88,7 @@ public class TenantServiceTest { result = tenantService.createTenant(loginUser, "test", "test", 1, "TenantServiceTest"); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } catch (Exception e) { logger.error("create tenant error",e); Assert.assertTrue(false); @@ -192,11 +196,17 @@ public class TenantServiceTest { // tenantCode not exist Result result = tenantService.verifyTenantCode("s00000000000l887888885554444sfjdskfjslakslkdf"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); // tenantCode exist result = tenantService.verifyTenantCode(getTenant().getTenantCode()); + String resultString; + if (Locale.SIMPLIFIED_CHINESE.getLanguage().equals(LocaleContextHolder.getLocale().getLanguage())) { + resultString = "租户编码[TenantServiceTest]已存在"; + } else { + resultString = "tenant code TenantServiceTest already exists"; + } logger.info(result.toString()); - Assert.assertEquals(Status.TENANT_NAME_EXIST.getMsg(),result.getMsg()); + Assert.assertEquals(resultString, result.getMsg()); } @@ -261,4 +271,4 @@ public class TenantServiceTest { } -} \ No newline at end of file +} From 6d43c21d80a3b210a7e01df9fa73d4f076698a58 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Mon, 13 Jul 2020 08:51:07 +0800 Subject: [PATCH 4/5] [bug fix]Click the cancel button to prevent the input box from losing focus #3116 (#3115) * [bugfix]Replace favicon icon with png * Cancel the initial value of the node cache * [bug fix]Click the cancel button to prevent the input box from losing focus Co-authored-by: dailidong --- .../conf/home/pages/dag/_source/formModel/formModel.vue | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 54293272df..1e2cbd3ba6 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -260,7 +260,7 @@
- {{$t('Cancel')}} + {{$t('Cancel')}} {{spinnerLoading ? 'Loading...' : $t('Confirm add')}}
@@ -580,6 +580,7 @@ } this.isContentBox = false // flag Whether to delete a node this.$destroy() + this.$emit('close', { item: { type: this.cacheBackfillItem.type, @@ -675,7 +676,11 @@ this.isContentBox = true }, mounted () { - + let self = this + $("#cancelBtn").mousedown(function(event){ + event.preventDefault(); + self.close() + }); }, updated () { }, From 4bf187ce389d9d56291cf2be7958c1a1433fef50 Mon Sep 17 00:00:00 2001 From: BoYiZhang <39816903+BoYiZhang@users.noreply.github.com> Date: Mon, 13 Jul 2020 11:08:41 +0800 Subject: [PATCH 5/5] update CLICK_SAVE_WORKFLOW_BUTTON xpath --- .../dolphinscheduler/locator/project/CreateWorkflowLocator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java index 496bbc4863..c94191bfb9 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java @@ -126,7 +126,7 @@ public class CreateWorkflowLocator { * save workflow */ //click save workflow button - public static final By CLICK_SAVE_WORKFLOW_BUTTON = By.xpath("//button/span"); + public static final By CLICK_SAVE_WORKFLOW_BUTTON = By.xpath("//button[3]/span"); //input workflow name public static final By INPUT_WORKFLOW_NAME = By.xpath("//input");