From d5fe55ad56bd902523a0868a7d4f5d2bed3e8caa Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 30 Dec 2019 18:44:03 +0800 Subject: [PATCH 01/20] modify UdfFuncServiceTest UT --- dolphinscheduler-api/pom.xml | 18 ++ .../api/service/UdfFuncServiceTest.java | 196 ++++++++++++++++-- pom.xml | 1 + 3 files changed, 201 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 06a444aa6d..014799b0b6 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -250,5 +250,23 @@ ${servlet-api.version} + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index 814a9ee7cd..9ec24bbb50 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -16,43 +16,211 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; +import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest(PropertyUtils.class) public class UdfFuncServiceTest { private static final Logger logger = LoggerFactory.getLogger(UdfFuncServiceTest.class); - @Autowired + @InjectMocks private UdfFuncService udfFuncService; + @Mock + private ResourceMapper resourceMapper; + @Mock + private UdfFuncMapper udfFuncMapper; + @Mock + private UDFUserMapper udfUserMapper; + + + @Before + public void setUp() { + PowerMockito.mockStatic(PropertyUtils.class); + } + + @Test + public void testCreateUdfFunction(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + //hdfs not start + Result result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + //resource not exist + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + // success + PowerMockito.when(resourceMapper.selectById(1)).thenReturn(getResource()); + result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + } @Test - public void queryUdfFuncListPaging(){ + public void testQueryUdfFuncDetail(){ + + PowerMockito.when(udfFuncMapper.selectById(1)).thenReturn(getUdfFunc()); + //resource not exist + Map result = udfFuncService.queryUdfFuncDetail(2); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST,result.get(Constants.STATUS)); + // success + result = udfFuncService.queryUdfFuncDetail(1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testUpdateUdfFunc(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + PowerMockito.when(udfFuncMapper.selectUdfById(1)).thenReturn(getUdfFunc()); + PowerMockito.when(resourceMapper.selectById(1)).thenReturn(getResource()); + + //UDF_FUNCTION_NOT_EXIST + Map result = udfFuncService.updateUdfFunc(12, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.UDF_FUNCTION_NOT_EXIST,result.get(Constants.STATUS)); + + //HDFS_NOT_STARTUP + result = udfFuncService.updateUdfFunc(1, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP,result.get(Constants.STATUS)); + + //RESOURCE_NOT_EXIST + PowerMockito.when(udfFuncMapper.selectUdfById(11)).thenReturn(getUdfFunc()); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = udfFuncService.updateUdfFunc(11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 12); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST,result.get(Constants.STATUS)); + + //success + result = udfFuncService.updateUdfFunc(11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + + @Test + public void testQueryUdfFuncListPaging(){ + + IPage page = new Page<>(1,10); + page.setTotal(1L); + page.setRecords(getList()); + Mockito.when(udfFuncMapper.queryUdfFuncPaging(Mockito.any(Page.class), Mockito.eq(0),Mockito.eq("test"))).thenReturn(page); + Map result = udfFuncService.queryUdfFuncListPaging(getLoginUser(),"test",1,10); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + } + + @Test + public void testQueryResourceList(){ + Mockito.when(udfFuncMapper.getUdfFuncByType(1, 1)).thenReturn(getList()); + Map result = udfFuncService.queryResourceList(getLoginUser(),1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List udfFuncList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncList)); + } + + @Test + public void testDelete(){ + Result result= udfFuncService.delete(122); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + } + + @Test + public void testVerifyUdfFuncByName(){ + + //success + Mockito.when(udfFuncMapper.queryUdfByIdStr(null, "UdfFuncServiceTest")).thenReturn(getList()); + Result result = udfFuncService.verifyUdfFuncByName("test"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + //exist + result = udfFuncService.verifyUdfFuncByName("UdfFuncServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.UDF_FUNCTION_EXISTS.getMsg(),result.getMsg()); + } + + /** + * create admin user + * @return + */ + private User getLoginUser(){ User loginUser = new User(); - loginUser.setId(-1); - loginUser.setUserType(UserType.GENERAL_USER); + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setId(1); + return loginUser; + } + + /** + * get resourceId + */ + private Resource getResource(){ - Map map = udfFuncService.queryUdfFuncListPaging(loginUser, "", 1, 10); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); + Resource resource = new Resource(); + resource.setId(1); + resource.setAlias("test"); + return resource; + } - PageInfo pageInfo = (PageInfo) map.get("data"); - logger.info(pageInfo.getLists().toString()); + private List getList(){ + List udfFuncList = new ArrayList<>(); + udfFuncList.add(getUdfFunc()); + return udfFuncList; + } + /** + * get UdfFunc id + */ + private UdfFunc getUdfFunc(){ + UdfFunc udfFunc = new UdfFunc(); + udfFunc.setFuncName("UdfFuncServiceTest"); + udfFunc.setClassName("org.apache.dolphinscheduler.api.service.UdfFuncServiceTest"); + udfFunc.setResourceId(0); + udfFunc.setResourceName("UdfFuncServiceTest"); + udfFunc.setCreateTime(new Date()); + udfFunc.setDatabase("database"); + udfFunc.setUpdateTime(new Date()); + udfFunc.setType(UdfType.HIVE); + return udfFunc; } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index fafa7be6ef..5e94b16768 100644 --- a/pom.xml +++ b/pom.xml @@ -675,6 +675,7 @@ **/api/service/WorkerGroupServiceTest.java **/api/service/AlertGroupServiceTest.java **/api/service/ProjectServiceTest.java + **/api/service/UdfFuncServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From 67acd09f9401ea675d0231a610c23a8b58459767 Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 2 Jan 2020 16:53:07 +0800 Subject: [PATCH 02/20] fix get tenantCode my NPE --- .../api/service/ResourcesService.java | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 66bf214608..33d2c96071 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -234,6 +234,12 @@ public class ResourcesService extends BaseService { } } + // query tenant by user id + String tenantCode = getTenantCode(resource.getUserId(),result); + if (StringUtils.isEmpty(tenantCode)){ + return result; + } + //get the file suffix String originResourceName = resource.getAlias(); String suffix = originResourceName.substring(originResourceName.lastIndexOf(".")); @@ -271,10 +277,6 @@ public class ResourcesService extends BaseService { return result; } - // hdfs move - // query tenant by user id - User user = userMapper.queryDetailsById(resource.getUserId()); - String tenantCode = tenantMapper.queryById(user.getTenantId()).getTenantCode(); // get file hdfs path // delete hdfs file by type String originHdfsFileName = ""; @@ -430,10 +432,15 @@ public class ResourcesService extends BaseService { return result; } - String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode(); + Tenant tenant = tenantMapper.queryById(loginUser.getTenantId()); + if (tenant == null){ + putMsg(result, Status.TENANT_NOT_EXIST); + return result; + } String hdfsFilename = ""; // delete hdfs file by type + String tenantCode = tenant.getTenantCode(); hdfsFilename = getHdfsFileName(resource, tenantCode, hdfsFilename); //delete data in database @@ -522,8 +529,11 @@ public class ResourcesService extends BaseService { } } - User user = userMapper.queryDetailsById(resource.getUserId()); - String tenantCode = tenantMapper.queryById(user.getTenantId()).getTenantCode(); + String tenantCode = getTenantCode(resource.getUserId(),result); + if (StringUtils.isEmpty(tenantCode)){ + return result; + } + // hdfs path String hdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, resource.getAlias()); logger.info("resource hdfs path is {} ", hdfsFileName); @@ -644,18 +654,20 @@ public class ResourcesService extends BaseService { if (StringUtils.isNotEmpty(resourceViewSuffixs)) { List strList = Arrays.asList(resourceViewSuffixs.split(",")); if (!strList.contains(nameSuffix)) { - logger.error("resouce suffix {} not support updateProcessInstance, resource id {}", nameSuffix, resourceId); + logger.error("resource suffix {} not support updateProcessInstance, resource id {}", nameSuffix, resourceId); putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW); return result; } } + String tenantCode = getTenantCode(resource.getUserId(),result); + if (StringUtils.isEmpty(tenantCode)){ + return result; + } resource.setSize(content.getBytes().length); resource.setUpdateTime(new Date()); resourcesMapper.updateById(resource); - User user = userMapper.queryDetailsById(resource.getUserId()); - String tenantCode = tenantMapper.queryById(user.getTenantId()).getTenantCode(); result = uploadContentToHdfs(resource.getAlias(), tenantCode, content); if (!result.getCode().equals(Status.SUCCESS.getCode())) { @@ -897,4 +909,27 @@ public class ResourcesService extends BaseService { } } + /** + * get tenantCode by UserId + * + * @param userId user id + * @param result return result + * @return + */ + private String getTenantCode(int userId,Result result){ + + User user = userMapper.queryDetailsById(userId); + if(user == null){ + putMsg(result, Status.USER_NOT_EXIST,userId); + return null; + } + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null){ + putMsg(result, Status.TENANT_NOT_EXIST); + return null; + } + return tenant.getTenantCode(); + } + } From 32604123eb94edb0148b0f38e6947e07894f0ae8 Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 2 Jan 2020 16:53:43 +0800 Subject: [PATCH 03/20] add ResourcesServiceTest --- .../api/service/ResourcesServiceTest.java | 587 +++++++++++++++++- pom.xml | 1 + 2 files changed, 576 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 8e94ccac52..c0caaaaa71 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -16,38 +16,601 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.common.utils.HadoopUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.*; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.mock.web.MockMultipartFile; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest({HadoopUtils.class,PropertyUtils.class, FileUtils.class,org.apache.dolphinscheduler.api.utils.FileUtils.class}) public class ResourcesServiceTest { private static final Logger logger = LoggerFactory.getLogger(ResourcesServiceTest.class); - @Autowired + @InjectMocks private ResourcesService resourcesService; + @Mock + private ResourceMapper resourcesMapper; + @Mock + private TenantMapper tenantMapper; + @Mock + private ResourceUserMapper resourceUserMapper; + @Mock + private HadoopUtils hadoopUtils; + @Mock + private UserMapper userMapper; + @Mock + private UdfFuncMapper udfFunctionMapper; + + @Before + public void setUp() { + + PowerMockito.mockStatic(HadoopUtils.class); + PowerMockito.mockStatic(FileUtils.class); + PowerMockito.mockStatic(org.apache.dolphinscheduler.api.utils.FileUtils.class); + try { + // new HadoopUtils + PowerMockito.whenNew(HadoopUtils.class).withNoArguments().thenReturn(hadoopUtils); + } catch (Exception e) { + e.printStackTrace(); + } + PowerMockito.when(HadoopUtils.getInstance()).thenReturn(hadoopUtils); + PowerMockito.mockStatic(PropertyUtils.class); + } + + @Test + public void testCreateResource(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + User user = new User(); + //HDFS_NOT_STARTUP + Result result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + + //RESOURCE_FILE_IS_EMPTY + MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf",new String().getBytes()); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(),result.getMsg()); + + //RESOURCE_SUFFIX_FORBID_CHANGE + mockMultipartFile = new MockMultipartFile("test.pdf","test.pdf","pdf",new String("test").getBytes()); + PowerMockito.when(FileUtils.suffix("test.pdf")).thenReturn("pdf"); + PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); + result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(),result.getMsg()); + + //UDF_RESOURCE_SUFFIX_NOT_JAR + mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf","ResourcesServiceTest.pdf","pdf",new String("test").getBytes()); + result = resourcesService.createResource(user,"ResourcesServiceTest.pdf","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile); + logger.info(result.toString()); + Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(),result.getMsg()); + + + //UDF_RESOURCE_SUFFIX_NOT_JAR + Mockito.when(tenantMapper.queryById(0)).thenReturn(getTenant()); + Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest.jar", 0, 1)).thenReturn(getResourceList()); + mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.jar","ResourcesServiceTest.jar","pdf",new String("test").getBytes()); + result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); + + //SUCCESS + Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest.jar", 0, 1)).thenReturn(new ArrayList<>()); + result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + + } + + @Test + public void testUpdateResource(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + User user = new User(); + //HDFS_NOT_STARTUP + Result result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + + //RESOURCE_NOT_EXIST + Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = resourcesService.updateResource(user,0,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + + //USER_NO_OPERATION_PERM + result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(),result.getMsg()); + + //SUCCESS + user.setId(1); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest.jar","ResourcesServiceTest.jar",ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + + //RESOURCE_EXIST + Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest1.jar", 0, 0)).thenReturn(getResourceList()); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); + //USER_NOT_EXIST + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + logger.info(result.toString()); + Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); + + //TENANT_NOT_EXIST + Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); + + //RESOURCE_NOT_EXIST + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + PowerMockito.when(HadoopUtils.getHdfsFilename(Mockito.any(), Mockito.any())).thenReturn("test1"); + + try { + Mockito.when(hadoopUtils.exists("test")).thenReturn(true); + } catch (IOException e) { + e.printStackTrace(); + } + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + + //SUCCESS + PowerMockito.when(HadoopUtils.getHdfsFilename(Mockito.any(), Mockito.any())).thenReturn("test"); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + + } + + @Test + public void testQueryResourceListPaging(){ + User loginUser = new User(); + loginUser.setUserType(UserType.ADMIN_USER); + IPage resourcePage = new Page<>(1,10); + resourcePage.setTotal(1); + resourcePage.setRecords(getResourceList()); + Mockito.when(resourcesMapper.queryResourcePaging(Mockito.any(Page.class), + Mockito.eq(0), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage); + Map result = resourcesService.queryResourceListPaging(loginUser,ResourceType.FILE,"test",1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + + } + + @Test + public void testQueryResourceList(){ + User loginUser = new User(); + loginUser.setId(0); + loginUser.setUserType(UserType.ADMIN_USER); + Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0)).thenReturn(getResourceList()); + Map result = resourcesService.queryResourceList(loginUser, ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + List resourceList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(resourceList)); + } @Test - public void querytResourceList(){ + public void testDelete(){ + User loginUser = new User(); - loginUser.setId(-1); - loginUser.setUserType(UserType.GENERAL_USER); + loginUser.setId(0); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + + try { + // HDFS_NOT_STARTUP + Result result = resourcesService.delete(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); + + //RESOURCE_NOT_EXIST + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); + result = resourcesService.delete(loginUser,2); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); + + // USER_NO_OPERATION_PERM + result = resourcesService.delete(loginUser,2); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); + + //TENANT_NOT_EXIST + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setTenantId(2); + result = resourcesService.delete(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); + + //SUCCESS + loginUser.setTenantId(1); + Mockito.when(hadoopUtils.delete(Mockito.anyString(), Mockito.anyBoolean())).thenReturn(true); + result = resourcesService.delete(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + + } catch (Exception e) { + logger.error("delete error",e); + Assert.assertTrue(false); + } + } + + @Test + public void testVerifyResourceName(){ + + User user = new User(); + user.setId(1); + Mockito.when(resourcesMapper.queryResourceList("test", 0, 0)).thenReturn(getResourceList()); + Result result = resourcesService.verifyResourceName("test",ResourceType.FILE,user); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); + + //TENANT_NOT_EXIST + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + result = resourcesService.verifyResourceName("test1",ResourceType.FILE,user); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); + + + //RESOURCE_FILE_EXIST + user.setTenantId(1); + try { + Mockito.when(hadoopUtils.exists("test")).thenReturn(true); + } catch (IOException e) { + logger.error("hadoop error",e); + } + PowerMockito.when(HadoopUtils.getHdfsFilename("123", "test1")).thenReturn("test"); + result = resourcesService.verifyResourceName("test1",ResourceType.FILE,user); + logger.info(result.toString()); + Assert.assertTrue(Status.RESOURCE_FILE_EXIST.getCode()==result.getCode()); + + //SUCCESS + result = resourcesService.verifyResourceName("test2",ResourceType.FILE,user); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + + } + + @Test + public void testReadResource(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + + //HDFS_NOT_STARTUP + Result result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + + //RESOURCE_NOT_EXIST + Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = resourcesService.readResource(2,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + + + //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + + //USER_NOT_EXIST + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); + PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); + result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertTrue(Status.USER_NOT_EXIST.getCode()==result.getCode()); + + + //TENANT_NOT_EXIST + Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); + result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); + + + //RESOURCE_FILE_NOT_EXIST + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + try { + Mockito.when(hadoopUtils.exists(Mockito.anyString())).thenReturn(false); + } catch (IOException e) { + logger.error("hadoop error",e); + } + result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertTrue(Status.RESOURCE_FILE_NOT_EXIST.getCode()==result.getCode()); + + //SUCCESS + try { + Mockito.when(hadoopUtils.exists(null)).thenReturn(true); + Mockito.when(hadoopUtils.catFile(null,1,10)).thenReturn(getContent()); + } catch (IOException e) { + logger.error("hadoop error",e); + } + result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); - Map map = resourcesService.queryResourceList(loginUser, ResourceType.FILE); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); + + } + + @Test + public void testOnlineCreateResource() { + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + PowerMockito.when(HadoopUtils.getHdfsResDir("hdfsdDir")).thenReturn("hdfsDir"); + PowerMockito.when(HadoopUtils.getHdfsUdfDir("udfDir")).thenReturn("udfDir"); + User user = getUser(); + //HDFS_NOT_STARTUP + Result result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content"); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + + //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); + result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content"); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + + //RuntimeException + try { + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content"); + }catch (RuntimeException ex){ + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), ex.getMessage()); + } + + //SUCCESS + Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); + PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); + result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + + + } + + @Test + public void testUpdateResourceContent(){ + + User loginUser = new User(); + loginUser.setId(0); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + + // HDFS_NOT_STARTUP + Result result = resourcesService.updateResourceContent(1,"content"); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); + + //RESOURCE_NOT_EXIST + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); + result = resourcesService.updateResourceContent(2,"content"); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); + + //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); + result = resourcesService.updateResourceContent(1,"content"); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + + //USER_NOT_EXIST + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); + PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); + result = resourcesService.updateResourceContent(1,"content"); + logger.info(result.toString()); + Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); + + + //TENANT_NOT_EXIST + Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); + result = resourcesService.updateResourceContent(1,"content"); + logger.info(result.toString()); + Assert.assertTrue(Status.TENANT_NOT_EXIST.getCode() == result.getCode()); + + //SUCCESS + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); + PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); + result = resourcesService.updateResourceContent(1,"content"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + } + + @Test + public void testDownloadResource(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); + org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class); + try { + //resource null + org.springframework.core.io.Resource resource = resourcesService.downloadResource(1); + Assert.assertNull(resource); + + Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); + PowerMockito.when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any())).thenReturn(resourceMock); + resource = resourcesService.downloadResource(1); + Assert.assertNotNull(resource); + } catch (Exception e) { + logger.error("DownloadResource error",e); + Assert.assertTrue(false); + } + + } + + @Test + public void testUnauthorizedFile(){ + User user = getUser(); + //USER_NO_OPERATION_PERM + Map result = resourcesService.unauthorizedFile(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + + //SUCCESS + user.setUserType(UserType.ADMIN_USER); + Mockito.when(resourcesMapper.queryResourceExceptUserId(1)).thenReturn(getResourceList()); + result = resourcesService.unauthorizedFile(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List resources = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(resources)); + + } + + @Test + public void testUnauthorizedUDFFunction(){ + + User user = getUser(); + //USER_NO_OPERATION_PERM + Map result = resourcesService.unauthorizedUDFFunction(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + + //SUCCESS + user.setUserType(UserType.ADMIN_USER); + Mockito.when(udfFunctionMapper.queryUdfFuncExceptUserId(1)).thenReturn(getUdfFuncList()); + result = resourcesService.unauthorizedUDFFunction(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List udfFuncs = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs)); + } + + + @Test + public void testAuthorizedUDFFunction(){ + User user = getUser(); + //USER_NO_OPERATION_PERM + Map result = resourcesService.authorizedUDFFunction(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + //SUCCESS + user.setUserType(UserType.ADMIN_USER); + Mockito.when(udfFunctionMapper.queryAuthedUdfFunc(1)).thenReturn(getUdfFuncList()); + result = resourcesService.authorizedUDFFunction(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List udfFuncs = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs)); + } + + @Test + public void testAuthorizedFile(){ + + User user = getUser(); + //USER_NO_OPERATION_PERM + Map result = resourcesService.authorizedFile(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + //SUCCESS + user.setUserType(UserType.ADMIN_USER); + Mockito.when(resourcesMapper.queryAuthorizedResourceList(1)).thenReturn(getResourceList()); + result = resourcesService.authorizedFile(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List resources = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(resources)); + } + + + private List getResourceList(){ + + List resources = new ArrayList<>(); + resources.add(getResource()); + return resources; + } + + + private Tenant getTenant() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("123"); + return tenant; + } + + private Resource getResource(){ + + Resource resource = new Resource(); + resource.setUserId(1); + resource.setDescription("ResourcesServiceTest.jar"); + resource.setAlias("ResourcesServiceTest.jar"); + resource.setType(ResourceType.FILE); + return resource; + } + + private UdfFunc getUdfFunc(){ + + UdfFunc udfFunc = new UdfFunc(); + udfFunc.setId(1); + return udfFunc; + } + + private List getUdfFuncList(){ + + List udfFuncs = new ArrayList<>(); + udfFuncs.add(getUdfFunc()); + return udfFuncs; + } + + private User getUser(){ + User user = new User(); + user.setId(1); + user.setTenantId(1); + return user; + } + private List getContent(){ + List contentList = new ArrayList<>(); + contentList.add("test"); + return contentList; } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 6892b387f1..090babb7cb 100644 --- a/pom.xml +++ b/pom.xml @@ -676,6 +676,7 @@ **/api/service/AlertGroupServiceTest.java **/api/service/ProjectServiceTest.java **/api/service/UdfFuncServiceTest.java + **/api/service/ResourcesServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From b0d73fb7897d94e318d63748d87b271f6adf984b Mon Sep 17 00:00:00 2001 From: lilin Date: Sat, 4 Jan 2020 11:40:53 +0800 Subject: [PATCH 04/20] add error log --- .../apache/dolphinscheduler/api/service/ResourcesService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 33d2c96071..8a6ffb8a18 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -920,12 +920,14 @@ public class ResourcesService extends BaseService { User user = userMapper.queryDetailsById(userId); if(user == null){ + logger.error("user {} not exists", userId); putMsg(result, Status.USER_NOT_EXIST,userId); return null; } Tenant tenant = tenantMapper.queryById(user.getTenantId()); if (tenant == null){ + logger.error("tenant not exists"); putMsg(result, Status.TENANT_NOT_EXIST); return null; } From b4a6d5440fe62da1135acc7b3a6134ae890a6f29 Mon Sep 17 00:00:00 2001 From: lilin Date: Wed, 15 Jan 2020 14:11:09 +0800 Subject: [PATCH 05/20] add BaseServiceTest BaseServiceTest UT --- .../api/service/BaseDAGServiceTest.java | 50 +++++++ .../api/service/BaseServiceTest.java | 140 ++++++++++++++++++ pom.xml | 2 + 3 files changed, 192 insertions(+) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java new file mode 100644 index 0000000000..bb6e3882fe --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.common.graph.DAG; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.model.TaskNodeRelation; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class BaseDAGServiceTest { + + @Test + public void testProcessInstance2DAG(){ + + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setProcessInstanceJson("{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-61567\"," + + "\"name\":\"开始\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo '1'\"}," + + "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + + "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + "\"workerGroupId\":-1,\"preTasks\":[]},{\"type\":\"SHELL\",\"id\":\"tasks-6-3ug5ej\",\"name\":\"结束\"," + + "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo '1'\"},\"description\":\"\"," + + "\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + + "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + "\"workerGroupId\":-1,\"preTasks\":[\"开始\"]}],\"tenantId\":-1,\"timeout\":0}"); + + DAG relationDAG = BaseDAGService.processInstance2DAG(processInstance); + + Assert.assertTrue(relationDAG.containsNode("开始")); + + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java new file mode 100644 index 0000000000..02086a8259 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.HadoopUtils; +import org.apache.dolphinscheduler.dao.entity.User; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.mock.web.MockCookie; +import org.springframework.mock.web.MockHttpServletRequest; + +import javax.servlet.http.Cookie; +import java.util.HashMap; +import java.util.Map; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"sun.security.*", "javax.net.*"}) +@PrepareForTest({HadoopUtils.class}) +public class BaseServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(BaseServiceTest.class); + + private BaseService baseService; + + @Mock + private HadoopUtils hadoopUtils; + + @Before + public void setUp() { + baseService = new BaseService(); + } + + @Test + public void testIsAdmin(){ + + User user = new User(); + user.setUserType(UserType.ADMIN_USER); + //ADMIN_USER + boolean isAdmin = baseService.isAdmin(user); + Assert.assertTrue(isAdmin); + //GENERAL_USER + user.setUserType(UserType.GENERAL_USER); + isAdmin = baseService.isAdmin(user); + Assert.assertFalse(isAdmin); + + } + + @Test + public void testPutMsg(){ + + Map result = new HashMap<>(); + baseService.putMsg(result, Status.SUCCESS); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + //has params + baseService.putMsg(result, Status.PROJECT_NOT_FOUNT,"test"); + + } + @Test + public void testPutMsgTwo(){ + + Result result = new Result(); + baseService.putMsg(result, Status.SUCCESS); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + //has params + baseService.putMsg(result,Status.PROJECT_NOT_FOUNT,"test"); + } + @Test + public void testGetCookie(){ + + MockHttpServletRequest request = new MockHttpServletRequest(); + MockCookie mockCookie = new MockCookie("userId","1"); + request.setCookies(mockCookie); + //cookie is not null + Cookie cookie = BaseService.getCookie(request,"userId"); + Assert.assertNotNull(cookie); + //cookie is null + cookie = BaseService.getCookie(request,"userName"); + Assert.assertNull(cookie); + + } + @Test + public void testCreateTenantDirIfNotExists(){ + + PowerMockito.mockStatic(HadoopUtils.class); + PowerMockito.when(HadoopUtils.getInstance()).thenReturn(hadoopUtils); + + try { + baseService.createTenantDirIfNotExists("test"); + } catch (Exception e) { + Assert.assertTrue(false); + logger.error("CreateTenantDirIfNotExists error ",e); + e.printStackTrace(); + } + + } + @Test + public void testHasPerm(){ + + User user = new User(); + user.setId(1); + //create user + boolean hasPerm = baseService.hasPerm(user,1); + Assert.assertTrue(hasPerm); + + //admin + user.setId(2); + user.setUserType(UserType.ADMIN_USER); + hasPerm = baseService.hasPerm(user,1); + Assert.assertTrue(hasPerm); + + } + +} diff --git a/pom.xml b/pom.xml index 4468c72e78..41c75f7d5f 100644 --- a/pom.xml +++ b/pom.xml @@ -694,6 +694,8 @@ **/api/service/ProcessDefinitionServiceTest.java **/api/service/UdfFuncServiceTest.java **/api/service/ResourcesServiceTest.java + **/api/service/BaseServiceTest.java + **/api/service/BaseDAGServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From b34f9745dc63e8e95407f5c0612db13b649b6e4a Mon Sep 17 00:00:00 2001 From: lilin Date: Wed, 15 Jan 2020 14:56:17 +0800 Subject: [PATCH 06/20] update --- .../dolphinscheduler/api/service/ResourcesService.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 77da00e457..09b1d31151 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -235,12 +235,6 @@ public class ResourcesService extends BaseService { return result; } - // query tenant by user id - String tenantCode = getTenantCode(resource.getUserId(),result); - if (StringUtils.isEmpty(tenantCode)){ - return result; - } - //get the file suffix String originResourceName = resource.getAlias(); String suffix = originResourceName.substring(originResourceName.lastIndexOf(".")); From 19bde513504d991443995d0a3246afc86bc537bb Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 16 Jan 2020 15:43:58 +0800 Subject: [PATCH 07/20] fix word spelling --- .../server/master/runner/MasterExecThread.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index a91f8c17e6..49154faece 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -224,14 +224,14 @@ public class MasterExecThread implements Runnable { // execute process ,waiting for end runProcess(); - // process instace failure ，no more complements + // process instace failure ,no more complements if(!processInstance.getState().typeIsSuccess()){ logger.info("process {} state {}, complement not completely!", processInstance.getId(), processInstance.getState()); break; } - // current process instance sucess ，next execute + // current process instance success ,next execute scheduleDate = DateUtils.getSomeDay(scheduleDate, 1); if(scheduleDate.after(endDate)){ // all success @@ -541,7 +541,7 @@ public class MasterExecThread implements Runnable { private DependResult isTaskDepsComplete(String taskName) { Collection startNodes = dag.getBeginNode(); - // ff the vertex returns true directly + // if the vertex returns true directly if(startNodes.contains(taskName)){ return DependResult.SUCCESS; } From 55923016b49c4e9c0a425a2ea0be890a1e5635de Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 16 Jan 2020 15:57:16 +0800 Subject: [PATCH 08/20] Revert "fix word spelling" This reverts commit 19bde513504d991443995d0a3246afc86bc537bb. --- .../server/master/runner/MasterExecThread.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 49154faece..a91f8c17e6 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -224,14 +224,14 @@ public class MasterExecThread implements Runnable { // execute process ,waiting for end runProcess(); - // process instace failure ,no more complements + // process instace failure ，no more complements if(!processInstance.getState().typeIsSuccess()){ logger.info("process {} state {}, complement not completely!", processInstance.getId(), processInstance.getState()); break; } - // current process instance success ,next execute + // current process instance sucess ，next execute scheduleDate = DateUtils.getSomeDay(scheduleDate, 1); if(scheduleDate.after(endDate)){ // all success @@ -541,7 +541,7 @@ public class MasterExecThread implements Runnable { private DependResult isTaskDepsComplete(String taskName) { Collection startNodes = dag.getBeginNode(); - // if the vertex returns true directly + // ff the vertex returns true directly if(startNodes.contains(taskName)){ return DependResult.SUCCESS; } From 9ebe177e1113b845a50336cb1c1221b0265dce6a Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 16 Jan 2020 16:55:45 +0800 Subject: [PATCH 09/20] fix Misspelled words --- .../server/master/runner/MasterExecThread.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 6c147e2628..84b1114b84 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -254,7 +254,7 @@ public class MasterExecThread implements Runnable { break; } - // current process instance sucess ，next execute + // current process instance success ,next execute if(null == iterator){ // loop by day scheduleDate = DateUtils.getSomeDay(scheduleDate, 1); @@ -575,7 +575,7 @@ public class MasterExecThread implements Runnable { private DependResult isTaskDepsComplete(String taskName) { Collection startNodes = dag.getBeginNode(); - // ff the vertex returns true directly + // if the vertex returns true directly if(startNodes.contains(taskName)){ return DependResult.SUCCESS; } From e00ef4915d50db3d594e89da8bc0efa9b4e25ba7 Mon Sep 17 00:00:00 2001 From: lilin Date: Sun, 9 Feb 2020 16:32:10 +0800 Subject: [PATCH 10/20] modify loggerService UT by Mock --- .../api/service/LoggerService.java | 5 ++ .../api/service/LoggerServiceTest.java | 88 +++++++++++++++---- pom.xml | 1 + 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java index 61dc1a7193..2587290fd3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java @@ -84,7 +84,12 @@ public class LoggerService { if (taskInstance == null){ throw new RuntimeException("task instance is null"); } + String host = taskInstance.getHost(); + if(StringUtils.isEmpty(host)){ + throw new RuntimeException("task instance host is null"); + } + LogClient logClient = new LogClient(host, Constants.RPC_PORT); return logClient.getLogBytes(taskInstance.getLogPath()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 798e064d79..cec7102ce1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -16,37 +16,95 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.log.LogClient; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.ProcessDao; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest({LoggerService.class}) public class LoggerServiceTest { + private static final Logger logger = LoggerFactory.getLogger(LoggerServiceTest.class); - @Autowired + @InjectMocks private LoggerService loggerService; + @Mock + private ProcessDao processDao; + @Mock + private LogClient logClient; - @Test - public void queryDataSourceList(){ + @Before + public void setUp() { + + try { + PowerMockito.whenNew(LogClient.class).withAnyArguments().thenReturn(logClient); + } catch (Exception e) { + logger.error("setUp error: {}",e.getMessage()); + } + } - User loginUser = new User(); - loginUser.setId(27); - loginUser.setUserType(UserType.GENERAL_USER); + @Test + public void testQueryDataSourceList(){ - Result result = loggerService.queryLog(-1, 0, 100); + TaskInstance taskInstance = new TaskInstance(); + Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); + Result result = loggerService.queryLog(2,1,1); + //TASK_INSTANCE_NOT_FOUND + Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(),result.getCode().intValue()); + //HOST NOT FOUND + result = loggerService.queryLog(1,1,1); Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(),result.getCode().intValue()); + + //SUCCESS + taskInstance.setHost("127.0.0.1"); + taskInstance.setLogPath("/temp/log"); + Mockito.when(logClient.rollViewLog("/temp/log",1,1 )).thenReturn("test"); + Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); + result = loggerService.queryLog(1,1,1); + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); } + + @Test + public void testGetLogBytes(){ + + TaskInstance taskInstance = new TaskInstance(); + Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); + + //task instance is null + try{ + loggerService.getLogBytes(2); + }catch (Exception e){ + logger.error("testGetLogBytes error: {}","task instance is null"); + } + + //task instance host is null + try{ + loggerService.getLogBytes(1); + }catch (Exception e){ + logger.error("testGetLogBytes error: {}","task instance host is null"); + } + + //success + Mockito.when(logClient.getLogBytes("/temp/log")).thenReturn(new byte[]{}); + taskInstance.setHost("127.0.0.1"); + taskInstance.setLogPath("/temp/log"); + byte [] result = loggerService.getLogBytes(1); + Assert.assertEquals(0,result.length); + } + } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 875577c672..1b88c77418 100644 --- a/pom.xml +++ b/pom.xml @@ -699,6 +699,7 @@ **/api/service/ExecutorService2Test.java **/api/service/BaseServiceTest.java **/api/service/BaseDAGServiceTest.java + **/api/service/LoggerServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From b3976da5b022886b9d7089b856118e75d7884707 Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 10 Feb 2020 11:35:16 +0800 Subject: [PATCH 11/20] modify LoggerServiceTest ut --- .../dolphinscheduler/api/service/LoggerServiceTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index cec7102ce1..193ecb4129 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -88,14 +88,16 @@ public class LoggerServiceTest { //task instance is null try{ loggerService.getLogBytes(2); - }catch (Exception e){ + }catch (RuntimeException e){ + Assert.assertTrue(true); logger.error("testGetLogBytes error: {}","task instance is null"); } //task instance host is null try{ loggerService.getLogBytes(1); - }catch (Exception e){ + }catch (RuntimeException e){ + Assert.assertTrue(true); logger.error("testGetLogBytes error: {}","task instance host is null"); } From c447bb489f5da45ca234a236551474778b24f2e3 Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Mon, 10 Feb 2020 13:52:32 +0800 Subject: [PATCH 12/20] add stringutils ut (#1921) * add stringutils ut --- .../alert/utils/MailUtils.java | 8 +- .../ProcessDefinitionController.java | 16 +-- .../controller/ProcessInstanceController.java | 10 +- .../api/service/ResourcesService.java | 2 +- .../common/utils/StringUtils.java | 125 +----------------- .../common/utils/StringUtilsTest.java | 66 +++++++++ 6 files changed, 89 insertions(+), 138 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 7ebe6a7863..99efdc8a6a 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -90,14 +90,14 @@ public class MailUtils { public static Map sendMails(Collection receivers, Collection receiversCc, String title, String content, ShowType showType) { Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); - + // if there is no receivers && no receiversCc, no need to process if (CollectionUtils.isEmpty(receivers) && CollectionUtils.isEmpty(receiversCc)) { return retMap; } receivers.removeIf(StringUtils::isEmpty); - + if (showType == ShowType.TABLE || showType == ShowType.TEXT){ // send email HtmlEmail email = new HtmlEmail(); @@ -335,7 +335,7 @@ public class MailUtils { */ private static void handleException(Collection receivers, Map retMap, Exception e) { logger.error("Send email to {} failed {}", receivers, e); - retMap.put(Constants.MESSAGE, "Send email to {" + StringUtils.join(receivers, ",") + "} failed，" + e.toString()); + retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed，" + e.toString()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java index de9cc12a36..c07ecf9ca7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java @@ -53,7 +53,7 @@ public class ProcessDefinitionController extends BaseController{ /** * create process definition - * + * * @param loginUser login user * @param projectName project name * @param name process definition name @@ -96,7 +96,7 @@ public class ProcessDefinitionController extends BaseController{ /** * verify process definition name unique - * + * * @param loginUser login user * @param projectName project name * @param name name @@ -328,9 +328,9 @@ public class ProcessDefinitionController extends BaseController{ /** - * + * * get tasks list by process definition id - * + * * * @param loginUser login user * @param projectName project name @@ -442,7 +442,7 @@ public class ProcessDefinitionController extends BaseController{ loginUser.getUserName(), projectName, processDefinitionIds); Map result = new HashMap<>(5); - List deleteFailedIdList = new ArrayList(); + List deleteFailedIdList = new ArrayList<>(); if(StringUtils.isNotEmpty(processDefinitionIds)){ String[] processDefinitionIdArray = processDefinitionIds.split(","); @@ -451,17 +451,17 @@ public class ProcessDefinitionController extends BaseController{ try { Map deleteResult = processDefinitionService.deleteProcessDefinitionById(loginUser, projectName, processDefinitionId); if(!Status.SUCCESS.equals(deleteResult.get(Constants.STATUS))){ - deleteFailedIdList.add(processDefinitionId); + deleteFailedIdList.add(strProcessDefinitionId); logger.error((String)deleteResult.get(Constants.MSG)); } } catch (Exception e) { - deleteFailedIdList.add(processDefinitionId); + deleteFailedIdList.add(strProcessDefinitionId); } } } if(!deleteFailedIdList.isEmpty()){ - putMsg(result, Status.BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR,StringUtils.join(deleteFailedIdList,",")); + putMsg(result, Status.BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR, String.join(",", deleteFailedIdList)); }else{ putMsg(result, Status.SUCCESS); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index 542aad5c33..150c647f99 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -58,7 +58,7 @@ public class ProcessInstanceController extends BaseController{ /** * query process instance list paging - * + * * @param loginUser login user * @param projectName project name * @param pageNo page number @@ -372,7 +372,7 @@ public class ProcessInstanceController extends BaseController{ // task queue ITaskQueue tasksQueue = TaskQueueFactory.getTaskQueueInstance(); Map result = new HashMap<>(5); - List deleteFailedIdList = new ArrayList(); + List deleteFailedIdList = new ArrayList<>(); if(StringUtils.isNotEmpty(processInstanceIds)){ String[] processInstanceIdArray = processInstanceIds.split(","); @@ -381,16 +381,16 @@ public class ProcessInstanceController extends BaseController{ try { Map deleteResult = processInstanceService.deleteProcessInstanceById(loginUser, projectName, processInstanceId,tasksQueue); if(!Status.SUCCESS.equals(deleteResult.get(Constants.STATUS))){ - deleteFailedIdList.add(processInstanceId); + deleteFailedIdList.add(strProcessInstanceId); logger.error((String)deleteResult.get(Constants.MSG)); } } catch (Exception e) { - deleteFailedIdList.add(processInstanceId); + deleteFailedIdList.add(strProcessInstanceId); } } } if(deleteFailedIdList.size() > 0){ - putMsg(result, Status.BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR,StringUtils.join(deleteFailedIdList,",")); + putMsg(result, Status.BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR, String.join(",", deleteFailedIdList)); }else{ putMsg(result, Status.SUCCESS); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 09b1d31151..29a16447e1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -539,7 +539,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.SUCCESS); Map map = new HashMap<>(); map.put(ALIAS, resource.getAlias()); - map.put(CONTENT, StringUtils.join(content, "\n")); + map.put(CONTENT, String.join("\n", content)); result.setData(map); }else{ logger.error("read file {} not exist in hdfs", hdfsFileName); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java index 12b75fb0e5..af2817a8d7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java @@ -16,17 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; - -import java.nio.charset.StandardCharsets; -import java.util.Iterator; -import java.util.Objects; -import java.util.regex.Pattern; - - public class StringUtils { - - public static final int INDEX_NOT_FOUND = -1; - public static final String EMPTY = ""; public static boolean isEmpty(final CharSequence cs) { @@ -37,119 +27,14 @@ public class StringUtils { return !isEmpty(cs); } - public static boolean isBlank(CharSequence cs){ - int strLen; - if (cs == null || (strLen = cs.length()) == 0) { + public static boolean isBlank(String s){ + if (isEmpty(s)) { return true; } - for (int i = 0; i < strLen; i++) { - if (Character.isWhitespace(cs.charAt(i)) == false) { - return false; - } - } - return true; - } - - public static boolean isNotBlank(CharSequence str){ - return !isBlank(str); - } - - public static String substringBefore(final String str, final String separator) { - if (isBlank(str) || separator == null) { - return str; - } - if (separator.isEmpty()) { - return EMPTY; - } - final int pos = str.indexOf(separator); - if (pos == INDEX_NOT_FOUND) { - return str; - } - return str.substring(0, pos); - } - - public static String substringAfter(final String str, final String separator) { - if (isBlank(str)) { - return str; - } - if (separator == null) { - return EMPTY; - } - final int pos = str.indexOf(separator); - if (pos == INDEX_NOT_FOUND) { - return EMPTY; - } - return str.substring(pos + separator.length()); + return s.trim().length() == 0; } - public static String substringAfterLast(final String str, final String separator) { - if (isEmpty(str)) { - return str; - } - if (isEmpty(separator)) { - return EMPTY; - } - final int pos = str.lastIndexOf(separator); - if (pos == INDEX_NOT_FOUND || pos == str.length() - separator.length()) { - return EMPTY; - } - return str.substring(pos + separator.length()); - } - - public static String getUtf8String(byte[] bytes){ - return new String(bytes, StandardCharsets.UTF_8); - } - - public static byte[] getUtf8Bytes(String str){ - return str.getBytes(StandardCharsets.UTF_8); - } - - public static boolean hasChinese(String str) { - if (str == null) { - return false; - } - Pattern pattern = Pattern.compile("[\\u4E00-\\u9FBF]+"); - return pattern.matcher(str).find(); - } - - public static boolean hasSpace(String str) { - if (str == null) { - return false; - } - int len = str.length(); - for (int i = 0; i < len; i++) { - if (str.charAt(i) == ' ') { - return true; - } - } - return false; - } - - public static String join(final Iterable iterable, final String separator){ - Iterator iterator = iterable.iterator(); - if (iterator == null) { - return null; - } - if (!iterator.hasNext()) { - return EMPTY; - } - final Object first = iterator.next(); - if (!iterable.iterator().hasNext()) { - return Objects.toString(first, ""); - } - final StringBuilder buf = new StringBuilder(64); - if (first != null) { - buf.append(first); - } - while (iterator.hasNext()) { - if (separator != null) { - buf.append(separator); - } - final Object obj = iterator.next(); - if (obj != null) { - buf.append(obj); - } - } - return buf.toString(); + public static boolean isNotBlank(String s){ + return !isBlank(s); } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java new file mode 100644 index 0000000000..947e7310db --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; + +public class StringUtilsTest { + @Test + public void testIsNotEmpty() { + //null string + boolean b = StringUtils.isNotEmpty(null); + Assert.assertFalse(b); + + //"" string + b = StringUtils.isNotEmpty(""); + Assert.assertFalse(b); + + //" " string + b = StringUtils.isNotEmpty(" "); + Assert.assertTrue(b); + + //"test" string + b = StringUtils.isNotEmpty("test"); + Assert.assertTrue(b); + } + + @Test + public void testIsNotBlank() { + //null string + boolean b = StringUtils.isNotBlank(null); + Assert.assertFalse(b); + + //"" string + b = StringUtils.isNotBlank(""); + Assert.assertFalse(b); + + //" " string + b = StringUtils.isNotBlank(" "); + Assert.assertFalse(b); + + //" test " string + b = StringUtils.isNotBlank(" test "); + Assert.assertTrue(b); + + //"test" string + b = StringUtils.isNotBlank("test"); + Assert.assertTrue(b); + } +} From bcc734713bbb397e117722b6c62a4a62994923f7 Mon Sep 17 00:00:00 2001 From: zhukai Date: Mon, 10 Feb 2020 14:15:20 +0800 Subject: [PATCH 13/20] Newfeature for #1675. (#1908) Continue to finish the rest works, add the cache feature for dependence,mr,python,sub_process,procedure and shell. --- .../pages/dag/_source/formModel/formModel.vue | 20 +++++++++-- .../dag/_source/formModel/tasks/dependent.vue | 17 +++++++++- .../dag/_source/formModel/tasks/flink.vue | 2 +- .../pages/dag/_source/formModel/tasks/mr.vue | 29 ++++++++++++++++ .../dag/_source/formModel/tasks/procedure.vue | 25 +++++++++++--- .../dag/_source/formModel/tasks/python.vue | 32 ++++++++++++++--- .../dag/_source/formModel/tasks/shell.vue | 34 ++++++++++++++++--- .../dag/_source/formModel/tasks/spark.vue | 2 +- .../_source/formModel/tasks/sub_process.vue | 10 ++++-- 9 files changed, 151 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 682dd5b51a..e05de8e880 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -121,6 +121,7 @@ @@ -128,6 +129,7 @@ @@ -136,6 +138,7 @@ @@ -167,6 +170,7 @@ @@ -174,6 +178,7 @@ @@ -181,6 +186,7 @@ @@ -248,6 +254,8 @@ resourcesList: [], // dependence dependence: {}, + // cache dependence + cacheDependence: {}, // Current node params data params: {}, // Running sign @@ -283,6 +291,12 @@ _onDependent (o) { this.dependence = Object.assign(this.dependence, {}, o) }, + /** + * cache dependent + */ + _onCacheDependent (o) { + this.cacheDependence = Object.assign(this.cacheDependence, {}, o) + }, /** * Task timeout alarm */ @@ -356,9 +370,10 @@ type: this.taskType, id: this.id, name: this.name, + params: this.params, description: this.description, runFlag: this.runFlag, - dependence: this.dependence, + dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, timeout: this.timeout, @@ -522,6 +537,7 @@ this.params = o.params || {} this.dependence = o.dependence || {} + this.cacheDependence = o.dependence || {} } this.isContentBox = true @@ -551,7 +567,7 @@ name: this.name, description: this.description, runFlag: this.runFlag, - dependence: this.dependence, + dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, timeout: this.timeout, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/dependent.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/dependent.vue index cca9ec7003..79d127a108 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/dependent.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/dependent.vue @@ -131,6 +131,9 @@ setTimeout(() => { this.isLoading = false }, 600) + }, + cacheDependence (val) { + this.$emit('on-cache-dependent', val) } }, beforeCreate () { @@ -151,7 +154,19 @@ }, destroyed () { }, - computed: {}, + computed: { + cacheDependence () { + return { + relation: this.relation, + dependTaskList: _.map(this.dependTaskList, v => { + return { + relation: v.relation, + dependItemList: _.map(v.dependItemList, v1 => _.omit(v1, ['depTasksList', 'state', 'dateValueList'])) + } + }) + } + } + }, components: { mListBox, mDependItemList } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index f0d69ec420..03e53fe5e5 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -379,7 +379,7 @@ // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - this.mainJar = o.params.mainJar.res || '' + this.mainJar = o.params.mainJar && o.params.mainJar.res ? o.params.mainJar.res : '' this.deployMode = o.params.deployMode || '' this.slot = o.params.slot || 1 this.taskManager = o.params.taskManager || '2' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index fa73e9bb89..706a35f4fe 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -91,6 +91,7 @@ @@ -127,6 +128,8 @@ mainJarList: [], // Resource(list) resourceList: [], + // Cache ResourceList + cacheResourceList: [], // Custom parameter localParams: [], // Command line argument @@ -156,6 +159,12 @@ _onResourcesData (a) { this.resourceList = a }, + /** + * cache resourceList + */ + _onCacheResourcesData (a) { + this.cacheResourceList = a + }, /** * verification */ @@ -220,6 +229,25 @@ if (type === 'PYTHON') { this.mainClass = '' } + }, + //Watch the cacheParams + cacheParams (val) { + this.$emit('on-cache-params', val); + } + }, + computed: { + cacheParams () { + return { + mainClass: this.mainClass, + mainJar: { + res: this.mainJar + }, + resourceList: this.cacheResourceList, + localParams: this.localParams, + mainArgs: this.mainArgs, + others: this.others, + programType: this.programType + } } }, created () { @@ -238,6 +266,7 @@ let resourceList = o.params.resourceList || [] if (resourceList.length) { this.resourceList = resourceList + this.cacheResourceList = resourceList } // backfill localParams diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue index d55f18a2cc..e84f37d7a9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue @@ -70,7 +70,9 @@ // Data source type type: '', // data source - datasource: '' + datasource: '', + // Return to the selected data source + rtDatasource: '' } }, mixins: [disabledState], @@ -83,7 +85,7 @@ */ _onDsData (o) { this.type = o.type - this.datasource = o.datasource + this.rtDatasource = o.datasource }, /** * return udp @@ -112,14 +114,29 @@ // storage this.$emit('on-params', { type: this.type, - datasource: this.datasource, + datasource: this.rtDatasource, method: this.method, localParams: this.localParams }) return true } }, - watch: {}, + watch: { + //Watch the cacheParams + cacheParams (val) { + this.$emit('on-cache-params', val); + } + }, + computed: { + cacheParams () { + return { + type: this.type, + datasource: this.rtDatasource, + method: this.method, + localParams: this.localParams + } + } + }, created () { let o = this.backfillItem diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 39a7cd858b..e565b4a6bd 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -31,6 +31,7 @@ @@ -69,7 +70,9 @@ // Custom parameter localParams: [], // resource(list) - resourceList: [] + resourceList: [], + // Cache ResourceList + cacheResourceList: [] } }, mixins: [disabledState], @@ -89,6 +92,12 @@ _onResourcesData (a) { this.resourceList = a }, + /** + * cache resourceList + */ + _onCacheResourcesData (a) { + this.cacheResourceList = a + }, /** * verification */ @@ -142,18 +151,33 @@ return editor } }, - watch: {}, + watch: { + //Watch the cacheParams + cacheParams (val) { + this.$emit('on-cache-params', val); + } + }, + computed: { + cacheParams () { + return { + resourceList: this.cacheResourceList, + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' + } + } + }, created () { let o = this.backfillItem // Non-null objects represent backfill if (!_.isEmpty(o)) { - this.rawScript = o.params.rawScript + this.rawScript = o.params.rawScript || '' // backfill resourceList let resourceList = o.params.resourceList || [] if (resourceList.length) { this.resourceList = resourceList + this.cacheResourceList = resourceList } // backfill localParams @@ -174,4 +198,4 @@ }, components: { mLocalParams, mListBox, mResources } } - \ No newline at end of file + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 17184eed75..ad40c586b9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -37,6 +37,7 @@ @@ -75,7 +76,9 @@ // Custom parameter localParams: [], // resource(list) - resourceList: [] + resourceList: [], + // Cache ResourceList + cacheResourceList: [] } }, mixins: [disabledState], @@ -119,11 +122,17 @@ }, /** * return resourceList - * + * */ _onResourcesData (a) { this.resourceList = a }, + /** + * cache resourceList + */ + _onCacheResourcesData (a) { + this.cacheResourceList = a + }, /** * verification */ @@ -175,18 +184,33 @@ return editor } }, - watch: {}, + watch: { + //Watch the cacheParams + cacheParams (val) { + this.$emit('on-cache-params', val); + } + }, + computed: { + cacheParams () { + return { + resourceList: this.cacheResourceList, + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' + } + } + }, created () { let o = this.backfillItem // Non-null objects represent backfill if (!_.isEmpty(o)) { - this.rawScript = o.params.rawScript + this.rawScript = o.params.rawScript || '' // backfill resourceList let resourceList = o.params.resourceList || [] if (resourceList.length) { this.resourceList = resourceList + this.cacheResourceList = resourceList } // backfill localParams @@ -229,5 +253,5 @@ right: -12px; top: -16px; } - + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue index 66a89a4944..feef19856c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue @@ -412,7 +412,7 @@ // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - this.mainJar = o.params.mainJar.res || '' + this.mainJar = o.params.mainJar && o.params.mainJar.res ? o.params.mainJar.res : '' this.deployMode = o.params.deployMode || '' this.driverCores = o.params.driverCores || 1 this.driverMemory = o.params.driverMemory || '512M' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sub_process.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sub_process.vue index ee03513249..477038f18f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sub_process.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sub_process.vue @@ -86,7 +86,13 @@ return _.filter(this.processDefinitionList, v => id === v.id)[0].code } }, - watch: {}, + watch: { + wdiCurr (val) { + this.$emit('on-cache-params', { + processDefinitionId: this.wdiCurr + }) + } + }, created () { let processListS = _.cloneDeep(this.store.state.dag.processListS) let id = this.router.history.current.params.id || null @@ -115,4 +121,4 @@ mounted () { } } - \ No newline at end of file + From 69fa28357b025655ab5b0d6d2d5536e2e6add558 Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 10 Feb 2020 15:33:37 +0800 Subject: [PATCH 14/20] modify @RunWith(MockitoJUnitRunner.class) --- .../api/service/LoggerServiceTest.java | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 193ecb4129..31034e5993 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -17,24 +17,21 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.log.LogClient; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; +import org.mockito.junit.MockitoJUnitRunner; import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@RunWith(PowerMockRunner.class) +@RunWith(MockitoJUnitRunner.class) @PrepareForTest({LoggerService.class}) public class LoggerServiceTest { @@ -44,18 +41,7 @@ public class LoggerServiceTest { private LoggerService loggerService; @Mock private ProcessDao processDao; - @Mock - private LogClient logClient; - @Before - public void setUp() { - - try { - PowerMockito.whenNew(LogClient.class).withAnyArguments().thenReturn(logClient); - } catch (Exception e) { - logger.error("setUp error: {}",e.getMessage()); - } - } @Test public void testQueryDataSourceList(){ @@ -73,7 +59,6 @@ public class LoggerServiceTest { //SUCCESS taskInstance.setHost("127.0.0.1"); taskInstance.setLogPath("/temp/log"); - Mockito.when(logClient.rollViewLog("/temp/log",1,1 )).thenReturn("test"); Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); result = loggerService.queryLog(1,1,1); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); @@ -102,11 +87,10 @@ public class LoggerServiceTest { } //success - Mockito.when(logClient.getLogBytes("/temp/log")).thenReturn(new byte[]{}); taskInstance.setHost("127.0.0.1"); taskInstance.setLogPath("/temp/log"); - byte [] result = loggerService.getLogBytes(1); - Assert.assertEquals(0,result.length); + loggerService.getLogBytes(1); + } } \ No newline at end of file From d937a6ae9421b4df20a524de0ca8e93524e96d12 Mon Sep 17 00:00:00 2001 From: Yelli Date: Mon, 10 Feb 2020 16:37:51 +0800 Subject: [PATCH 15/20] Add modify user name for process definition (#1919) * class overrides equals() and should therefore also override hashCode() * #1862 add modify user in process difinition list * #1862 add pg-1.2.2 ddl.sql * modify ScriptRunnerTest * add updateProessDifinition UT * modify updateProcessDifinition UT * modify updateProcessDifinition UT * modify mysql 1.2.2 ddl.sql&dml.sql * add scope test to mysql in pom * modify pg-1.2.2 ddl.sql --- .../api/service/ProcessDefinitionService.java | 2 + .../service/ProcessDefinitionServiceTest.java | 27 +++++++++++ .../common/utils/ScriptRunner.java | 4 +- .../common/utils/ScriptRunnerTest.java | 4 +- .../dao/entity/ProcessDefinition.java | 48 ++++++++++++------- .../definition/pages/list/_source/list.vue | 7 +++ .../src/js/module/i18n/locale/en_US.js | 1 + .../src/js/module/i18n/locale/zh_CN.js | 1 + pom.xml | 1 + sql/dolphinscheduler-postgre.sql | 1 + sql/dolphinscheduler_mysql.sql | 1 + .../mysql/dolphinscheduler_ddl.sql | 37 ++++++++++++++ .../mysql/dolphinscheduler_dml.sql | 16 +++++++ .../postgresql/dolphinscheduler_ddl.sql | 34 +++++++++++++ .../postgresql/dolphinscheduler_dml.sql | 16 +++++++ 15 files changed, 180 insertions(+), 20 deletions(-) create mode 100644 sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_dml.sql create mode 100644 sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_dml.sql diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 5fe708ce8c..8a762d7557 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -143,6 +143,7 @@ public class ProcessDefinitionService extends BaseDAGService { processDefine.setConnects(connects); processDefine.setTimeout(processData.getTimeout()); processDefine.setTenantId(processData.getTenantId()); + processDefine.setModifyBy(loginUser.getUserName()); //custom global params List globalParamsList = processData.getGlobalParams(); @@ -308,6 +309,7 @@ public class ProcessDefinitionService extends BaseDAGService { processDefine.setConnects(connects); processDefine.setTimeout(processData.getTimeout()); processDefine.setTenantId(processData.getTenantId()); + processDefine.setModifyBy(loginUser.getUserName()); //custom global params List globalParamsList = new ArrayList<>(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index b30a919598..a4b07e1835 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; import org.apache.http.entity.ContentType; @@ -77,6 +78,9 @@ public class ProcessDefinitionServiceTest { @Mock private WorkerGroupMapper workerGroupMapper; + @Mock + private ProcessDao processDao; + private String sqlDependentJson = "{\"globalParams\":[]," + "\"tasks\":[{\"type\":\"SQL\",\"id\":\"tasks-27297\",\"name\":\"sql\"," + "\"params\":{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from test\"," + @@ -422,6 +426,27 @@ public class ProcessDefinitionServiceTest { Assert.assertTrue(deleteFlag); } + @Test + public void testUpdateProcessDefinition () { + User loginUser = new User(); + loginUser.setId(1); + loginUser.setUserType(UserType.ADMIN_USER); + + Map result = new HashMap<>(5); + putMsg(result, Status.SUCCESS); + + String projectName = "project_test1"; + Project project = getProject(projectName); + + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + Mockito.when(processDao.findProcessDefineById(1)).thenReturn(getProcessDefinition()); + + Map updateResult = processDefinitionService.updateProcessDefinition(loginUser, projectName, 1, "test", + sqlDependentJson, "", "", ""); + + Assert.assertEquals(Status.UPDATE_PROCESS_DEFINITION_ERROR, updateResult.get(Constants.STATUS)); + } /** * get mock datasource @@ -443,6 +468,8 @@ public class ProcessDefinitionServiceTest { processDefinition.setId(46); processDefinition.setName("testProject"); processDefinition.setProjectId(2); + processDefinition.setTenantId(1); + processDefinition.setDescription(""); return processDefinition; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java index bbc937c89f..f92839bfe5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java @@ -169,13 +169,13 @@ public class ScriptRunner { if (stopOnError && rs != null) { ResultSetMetaData md = rs.getMetaData(); int cols = md.getColumnCount(); - for (int i = 0; i < cols; i++) { + for (int i = 1; i < cols; i++) { String name = md.getColumnLabel(i); logger.info("{} \t", name); } logger.info(""); while (rs.next()) { - for (int i = 0; i < cols; i++) { + for (int i = 1; i < cols; i++) { String value = rs.getString(i); logger.info("{} \t", value); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java index 0eb1cce950..155d52ab75 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java @@ -48,7 +48,7 @@ public class ScriptRunnerTest { Mockito.when(st.getResultSet()).thenReturn(rs); ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class); Mockito.when(rs.getMetaData()).thenReturn(md); - Mockito.when(md.getColumnCount()).thenReturn(1); + Mockito.when(md.getColumnCount()).thenReturn(2); Mockito.when(rs.next()).thenReturn(true, false); ScriptRunner s = new ScriptRunner(conn, true, true); if (dbName.isEmpty()) { @@ -56,7 +56,7 @@ public class ScriptRunnerTest { } else { s.runScript(new StringReader("select 1;"), dbName); } - Mockito.verify(md).getColumnLabel(0); + Mockito.verify(md).getColumnLabel(1); } catch(Exception e) { Assert.assertNotNull(e); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index cd0494ecc6..6e7ea7d64f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -29,6 +29,7 @@ import com.baomidou.mybatisplus.core.toolkit.StringUtils; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; @@ -158,6 +159,11 @@ public class ProcessDefinition { */ private int tenantId; + /** + * modify user name + */ + private String modifyBy; + public String getName() { return name; @@ -337,6 +343,30 @@ public class ProcessDefinition { this.timeout = timeout; } + public int getTenantId() { + return tenantId; + } + + public void setTenantId(int tenantId) { + this.tenantId = tenantId; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getModifyBy() { + return modifyBy; + } + + public void setModifyBy(String modifyBy) { + this.modifyBy = modifyBy; + } + @Override public String toString() { return "ProcessDefinition{" + @@ -346,6 +376,7 @@ public class ProcessDefinition { ", releaseState=" + releaseState + ", projectId=" + projectId + ", processDefinitionJson='" + processDefinitionJson + '\'' + + ", description='" + description + '\'' + ", globalParams='" + globalParams + '\'' + ", globalParamList=" + globalParamList + ", globalParamMap=" + globalParamMap + @@ -362,22 +393,7 @@ public class ProcessDefinition { ", scheduleReleaseState=" + scheduleReleaseState + ", timeout=" + timeout + ", tenantId=" + tenantId + + ", modifyBy='" + modifyBy + '\'' + '}'; } - - public int getTenantId() { - return tenantId; - } - - public void setTenantId(int tenantId) { - this.tenantId = tenantId; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue index 5550864176..53939f3f7b 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue @@ -40,6 +40,9 @@ {{$t('Description')}} + + {{$t('Modify User')}} + {{$t('Timing state')}} @@ -72,6 +75,10 @@ {{item.description}} - + + {{item.modifyBy}} + - + {{$t('offline')}} {{$t('online')}} diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 6e8c113b30..0402d7e398 100644 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -518,4 +518,5 @@ export default { 'SpeedRecord': 'speed(record count)', '0 means unlimited by byte': '0 means unlimited', '0 means unlimited by count': '0 means unlimited', + 'Modify User': 'Modify User' } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 52a98773a6..95eb4a1081 100644 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -518,4 +518,5 @@ export default { 'SpeedRecord': '限流(记录数)', '0 means unlimited by byte': 'KB，0代表不限制', '0 means unlimited by count': '0代表不限制', + 'Modify User': '修改用户' } diff --git a/pom.xml b/pom.xml index 875577c672..307db31eee 100644 --- a/pom.xml +++ b/pom.xml @@ -343,6 +343,7 @@ mysql mysql-connector-java ${mysql.connector.version} + test org.slf4j diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index b3c61ebce4..c68fd17be1 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -319,6 +319,7 @@ CREATE TABLE t_ds_process_definition ( timeout int DEFAULT '0' , tenant_id int NOT NULL DEFAULT '-1' , update_time timestamp DEFAULT NULL , + modify_by varchar(36) DEFAULT '' , PRIMARY KEY (id) ) ; diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index fec2771fff..ea0f9cb022 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -366,6 +366,7 @@ CREATE TABLE `t_ds_process_definition` ( `timeout` int(11) DEFAULT '0' COMMENT 'time out', `tenant_id` int(11) NOT NULL DEFAULT '-1' COMMENT 'tenant id', `update_time` datetime DEFAULT NULL COMMENT 'update time', + `modify_by` varchar(36) DEFAULT '' COMMENT 'modify user', PRIMARY KEY (`id`), KEY `process_definition_index` (`project_id`,`id`) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..9fe246a8c2 --- /dev/null +++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); +-- uc_dolphin_T_t_ds_process_definition_A_modify_by +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_process_definition_A_modify_by; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_process_definition_A_modify_by() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_process_definition' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='modify_by') + THEN + ALTER TABLE t_ds_process_definition ADD `modify_by` varchar(36) DEFAULT '' COMMENT 'modify user'; + END IF; + END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_process_definition_A_modify_by; +DROP PROCEDURE uc_dolphin_T_t_ds_process_definition_A_modify_by; diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..7fc12900e4 --- /dev/null +++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +-- uc_dolphin_T_t_ds_process_definition_A_modify_by +delimiter d// +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_process_definition_A_modify_by() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_process_definition' + AND COLUMN_NAME ='modify_by') + THEN + ALTER TABLE t_ds_process_definition ADD COLUMN modify_by varchar(36) DEFAULT ''; + END IF; +END; +$$ LANGUAGE plpgsql; +d// + +delimiter ; +SELECT uc_dolphin_T_t_ds_process_definition_A_modify_by(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_definition_A_modify_by(); + diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_dml.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file From 17162341c3af7277dbac584beff6abf733a796d9 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Tue, 11 Feb 2020 00:18:06 +0800 Subject: [PATCH 16/20] add ResInfoTest.java ut --- .../common/utils/ResInfo.java | 12 ++++----- .../common/utils/ResInfoTest.java} | 27 +++++++++++++++---- 2 files changed, 28 insertions(+), 11 deletions(-) rename dolphinscheduler-common/src/{main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java => test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java} (54%) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java index e0cea1d7e0..aa8d44fa42 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java @@ -15,7 +15,6 @@ * limitations under the License. */ package org.apache.dolphinscheduler.common.utils; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.model.Server; @@ -135,14 +134,15 @@ public class ResInfo { * @return heartbeat info to Server */ public static Server parseHeartbeatForZKInfo(String heartBeatInfo){ - Server masterServer = null; + if (StringUtils.isEmpty(heartBeatInfo)) { + return null; + } String[] masterArray = heartBeatInfo.split(Constants.COMMA); - if(masterArray == null || - masterArray.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ - return masterServer; + if(masterArray.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ + return null; } - masterServer = new Server(); + Server masterServer = new Server(); masterServer.setHost(masterArray[0]); masterServer.setPort(Integer.parseInt(masterArray[1])); masterServer.setResInfo(getResInfoJson(Double.parseDouble(masterArray[2]), diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java similarity index 54% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java rename to dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java index 163c56bbcc..1e1b154d93 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java @@ -14,13 +14,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.dolphinscheduler.common.utils; +import org.junit.Assert; +import org.junit.Test; +import java.util.Date; +import org.apache.dolphinscheduler.common.model.Server; + +public class ResInfoTest { + @Test + public void testGetHeartBeatInfo() { + String info = ResInfo.getHeartBeatInfo(new Date()); + Assert.assertEquals(info.split(",").length, 7); + } -public class ArrayUtils { + @Test + public void testParseHeartbeatForZKInfo() { + //normal info + String info = ResInfo.getHeartBeatInfo(new Date()); + Server s = ResInfo.parseHeartbeatForZKInfo(info); + Assert.assertNotNull(s); + Assert.assertNotNull(s.getResInfo()); - public static boolean isEmpty(final int[] array) { - return array == null || array.length == 0; - } + //null param + s = ResInfo.parseHeartbeatForZKInfo(null); + Assert.assertNull(s); + } } From 1172abee44b2b894b10b88233919959d383acb13 Mon Sep 17 00:00:00 2001 From: lilin Date: Tue, 11 Feb 2020 09:00:56 +0800 Subject: [PATCH 17/20] add comment --- .../apache/dolphinscheduler/api/service/LoggerServiceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 31034e5993..c6ab6f8e74 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -89,6 +89,8 @@ public class LoggerServiceTest { //success taskInstance.setHost("127.0.0.1"); taskInstance.setLogPath("/temp/log"); + //if use @RunWith(PowerMockRunner.class) mock object,sonarcloud will not calculate the coverage, + // so no assert will be added here loggerService.getLogBytes(1); } From 980b5ae5ea4ee9e5751f15066401aec4118ee14f Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Tue, 11 Feb 2020 09:21:59 +0800 Subject: [PATCH 18/20] swap param --- .../org/apache/dolphinscheduler/common/utils/ResInfoTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java index 1e1b154d93..0d15d9615a 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java @@ -25,7 +25,7 @@ public class ResInfoTest { @Test public void testGetHeartBeatInfo() { String info = ResInfo.getHeartBeatInfo(new Date()); - Assert.assertEquals(info.split(",").length, 7); + Assert.assertEquals(7, info.split(",").length, ); } @Test From 0ed6156b221ef96d53917bb1ae1dece41f7aaa4c Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Tue, 11 Feb 2020 09:26:43 +0800 Subject: [PATCH 19/20] swap param --- .../org/apache/dolphinscheduler/common/utils/ResInfoTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java index 0d15d9615a..e4318965b7 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java @@ -25,7 +25,7 @@ public class ResInfoTest { @Test public void testGetHeartBeatInfo() { String info = ResInfo.getHeartBeatInfo(new Date()); - Assert.assertEquals(7, info.split(",").length, ); + Assert.assertEquals(7, info.split(",").length); } @Test From d3f8bb0a2b80f2a0348e2396c537f41be480cf84 Mon Sep 17 00:00:00 2001 From: Tboy Date: Tue, 11 Feb 2020 21:55:14 +0800 Subject: [PATCH 20/20] Refactor architecture (#1936) * move datasource classes to dao module * fix send4LetterWord bug --- .../api/service/DataSourceService.java | 3 +-- .../api/utils/FourLetterWordMain.java | 18 ++++++++++++------ .../dao/datasource}/BaseDataSource.java | 2 +- .../dao/datasource}/ClickHouseDataSource.java | 2 +- .../dao/datasource}/DB2ServerDataSource.java | 2 +- .../dao/datasource}/DataSourceFactory.java | 4 ++-- .../dao/datasource}/HiveDataSource.java | 6 ++++-- .../dao/datasource}/MySQLDataSource.java | 2 +- .../dao/datasource}/OracleDataSource.java | 2 +- .../dao/datasource}/PostgreDataSource.java | 2 +- .../dao/datasource}/SQLServerDataSource.java | 2 +- .../dao/datasource}/SparkDataSource.java | 2 +- .../server/worker/task/datax/DataxTask.java | 4 ++-- .../worker/task/processdure/ProcedureTask.java | 4 ++-- .../server/worker/task/sql/SqlTask.java | 4 ++-- .../worker/task/datax/DataxTaskTest.java | 4 ++-- 16 files changed, 35 insertions(+), 28 deletions(-) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/BaseDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/ClickHouseDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/DB2ServerDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/DataSourceFactory.java (98%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/HiveDataSource.java (93%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/MySQLDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/OracleDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/PostgreDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/SQLServerDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/SparkDataSource.java (97%) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 5d33b46bd2..f6d8903dd8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -21,10 +21,9 @@ import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.job.db.*; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.datasource.*; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.User; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java index b04e773aea..d4c11c769a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java @@ -60,21 +60,22 @@ public class FourLetterWordMain { public static String send4LetterWord(String host, int port, String cmd, int timeout) throws IOException { LOG.info("connecting to {} {}", host, port); + Socket sock = new Socket(); InetSocketAddress hostaddress= host != null ? new InetSocketAddress(host, port) : new InetSocketAddress(InetAddress.getByName(null), port); - - try (Socket sock = new Socket(); - OutputStream outstream = sock.getOutputStream(); - BufferedReader reader = - new BufferedReader( - new InputStreamReader(sock.getInputStream()))) { + BufferedReader reader = null; + try { sock.setSoTimeout(timeout); sock.connect(hostaddress, timeout); + OutputStream outstream = sock.getOutputStream(); outstream.write(cmd.getBytes()); outstream.flush(); // this replicates NC - close the output stream before reading sock.shutdownOutput(); + reader = + new BufferedReader( + new InputStreamReader(sock.getInputStream())); StringBuilder sb = new StringBuilder(); String line; while((line = reader.readLine()) != null) { @@ -83,6 +84,11 @@ public class FourLetterWordMain { return sb.toString(); } catch (SocketTimeoutException e) { throw new IOException("Exception while executing four letter word: " + cmd, e); + } finally { + sock.close(); + if (reader != null) { + reader.close(); + } } } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/BaseDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index 41a9b3a566..a46e5aabcc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; /** * data source base class diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java index fe76497ff8..cfa739ba25 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java index 44ee200c5d..3c2366b5b0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DataSourceFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java similarity index 98% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DataSourceFactory.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java index 48ec319eaa..9571f9c9f6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DataSourceFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java similarity index 93% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java index cddedd1f73..0a8f527536 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java @@ -14,14 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.*; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; /** * data source of hive diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java index fa149e67e2..6e2fbe3dd8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java index c3dc3a96df..cefaf879b5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java index 4989e7681e..176cba2587 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java index 8554992efc..07770c06a7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java index 5d10c63e5d..81a5ac6f04 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index e731ae9879..ef941cd062 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -39,14 +39,14 @@ import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.job.db.BaseDataSource; -import org.apache.dolphinscheduler.common.job.db.DataSourceFactory; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; +import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.utils.DataxUtils; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index 3898357f85..fb881453e9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -22,13 +22,13 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; -import org.apache.dolphinscheduler.common.job.db.BaseDataSource; -import org.apache.dolphinscheduler.common.job.db.DataSourceFactory; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; +import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 23bda089f8..aae11f5530 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -27,8 +27,6 @@ import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.common.enums.UdfType; -import org.apache.dolphinscheduler.common.job.db.BaseDataSource; -import org.apache.dolphinscheduler.common.job.db.DataSourceFactory; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.sql.SqlBinds; @@ -36,6 +34,8 @@ import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sql.SqlType; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AlertDao; +import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; +import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.UdfFunc; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index 4024dc36dd..bd7f27530a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -25,8 +25,8 @@ import java.util.List; import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.job.db.BaseDataSource; -import org.apache.dolphinscheduler.common.job.db.DataSourceFactory; +import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; +import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.utils.DataxUtils;