From d5fe55ad56bd902523a0868a7d4f5d2bed3e8caa Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 30 Dec 2019 18:44:03 +0800 Subject: [PATCH 01/43] modify UdfFuncServiceTest UT --- dolphinscheduler-api/pom.xml | 18 ++ .../api/service/UdfFuncServiceTest.java | 196 ++++++++++++++++-- pom.xml | 1 + 3 files changed, 201 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 06a444aa6d..014799b0b6 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -250,5 +250,23 @@ ${servlet-api.version} + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index 814a9ee7cd..9ec24bbb50 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -16,43 +16,211 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; +import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest(PropertyUtils.class) public class UdfFuncServiceTest { private static final Logger logger = LoggerFactory.getLogger(UdfFuncServiceTest.class); - @Autowired + @InjectMocks private UdfFuncService udfFuncService; + @Mock + private ResourceMapper resourceMapper; + @Mock + private UdfFuncMapper udfFuncMapper; + @Mock + private UDFUserMapper udfUserMapper; + + + @Before + public void setUp() { + PowerMockito.mockStatic(PropertyUtils.class); + } + + @Test + public void testCreateUdfFunction(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + //hdfs not start + Result result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + //resource not exist + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + // success + PowerMockito.when(resourceMapper.selectById(1)).thenReturn(getResource()); + result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + } @Test - public void queryUdfFuncListPaging(){ + public void testQueryUdfFuncDetail(){ + + PowerMockito.when(udfFuncMapper.selectById(1)).thenReturn(getUdfFunc()); + //resource not exist + Map result = udfFuncService.queryUdfFuncDetail(2); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST,result.get(Constants.STATUS)); + // success + result = udfFuncService.queryUdfFuncDetail(1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testUpdateUdfFunc(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + PowerMockito.when(udfFuncMapper.selectUdfById(1)).thenReturn(getUdfFunc()); + PowerMockito.when(resourceMapper.selectById(1)).thenReturn(getResource()); + + //UDF_FUNCTION_NOT_EXIST + Map result = udfFuncService.updateUdfFunc(12, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.UDF_FUNCTION_NOT_EXIST,result.get(Constants.STATUS)); + + //HDFS_NOT_STARTUP + result = udfFuncService.updateUdfFunc(1, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP,result.get(Constants.STATUS)); + + //RESOURCE_NOT_EXIST + PowerMockito.when(udfFuncMapper.selectUdfById(11)).thenReturn(getUdfFunc()); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = udfFuncService.updateUdfFunc(11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 12); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST,result.get(Constants.STATUS)); + + //success + result = udfFuncService.updateUdfFunc(11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + + @Test + public void testQueryUdfFuncListPaging(){ + + IPage page = new Page<>(1,10); + page.setTotal(1L); + page.setRecords(getList()); + Mockito.when(udfFuncMapper.queryUdfFuncPaging(Mockito.any(Page.class), Mockito.eq(0),Mockito.eq("test"))).thenReturn(page); + Map result = udfFuncService.queryUdfFuncListPaging(getLoginUser(),"test",1,10); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + } + + @Test + public void testQueryResourceList(){ + Mockito.when(udfFuncMapper.getUdfFuncByType(1, 1)).thenReturn(getList()); + Map result = udfFuncService.queryResourceList(getLoginUser(),1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List udfFuncList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncList)); + } + + @Test + public void testDelete(){ + Result result= udfFuncService.delete(122); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + } + + @Test + public void testVerifyUdfFuncByName(){ + + //success + Mockito.when(udfFuncMapper.queryUdfByIdStr(null, "UdfFuncServiceTest")).thenReturn(getList()); + Result result = udfFuncService.verifyUdfFuncByName("test"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + //exist + result = udfFuncService.verifyUdfFuncByName("UdfFuncServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.UDF_FUNCTION_EXISTS.getMsg(),result.getMsg()); + } + + /** + * create admin user + * @return + */ + private User getLoginUser(){ User loginUser = new User(); - loginUser.setId(-1); - loginUser.setUserType(UserType.GENERAL_USER); + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setId(1); + return loginUser; + } + + /** + * get resourceId + */ + private Resource getResource(){ - Map map = udfFuncService.queryUdfFuncListPaging(loginUser, "", 1, 10); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); + Resource resource = new Resource(); + resource.setId(1); + resource.setAlias("test"); + return resource; + } - PageInfo pageInfo = (PageInfo) map.get("data"); - logger.info(pageInfo.getLists().toString()); + private List getList(){ + List udfFuncList = new ArrayList<>(); + udfFuncList.add(getUdfFunc()); + return udfFuncList; + } + /** + * get UdfFunc id + */ + private UdfFunc getUdfFunc(){ + UdfFunc udfFunc = new UdfFunc(); + udfFunc.setFuncName("UdfFuncServiceTest"); + udfFunc.setClassName("org.apache.dolphinscheduler.api.service.UdfFuncServiceTest"); + udfFunc.setResourceId(0); + udfFunc.setResourceName("UdfFuncServiceTest"); + udfFunc.setCreateTime(new Date()); + udfFunc.setDatabase("database"); + udfFunc.setUpdateTime(new Date()); + udfFunc.setType(UdfType.HIVE); + return udfFunc; } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index fafa7be6ef..5e94b16768 100644 --- a/pom.xml +++ b/pom.xml @@ -675,6 +675,7 @@ **/api/service/WorkerGroupServiceTest.java **/api/service/AlertGroupServiceTest.java **/api/service/ProjectServiceTest.java + **/api/service/UdfFuncServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From 67acd09f9401ea675d0231a610c23a8b58459767 Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 2 Jan 2020 16:53:07 +0800 Subject: [PATCH 02/43] fix get tenantCode my NPE --- .../api/service/ResourcesService.java | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 66bf214608..33d2c96071 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -234,6 +234,12 @@ public class ResourcesService extends BaseService { } } + // query tenant by user id + String tenantCode = getTenantCode(resource.getUserId(),result); + if (StringUtils.isEmpty(tenantCode)){ + return result; + } + //get the file suffix String originResourceName = resource.getAlias(); String suffix = originResourceName.substring(originResourceName.lastIndexOf(".")); @@ -271,10 +277,6 @@ public class ResourcesService extends BaseService { return result; } - // hdfs move - // query tenant by user id - User user = userMapper.queryDetailsById(resource.getUserId()); - String tenantCode = tenantMapper.queryById(user.getTenantId()).getTenantCode(); // get file hdfs path // delete hdfs file by type String originHdfsFileName = ""; @@ -430,10 +432,15 @@ public class ResourcesService extends BaseService { return result; } - String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode(); + Tenant tenant = tenantMapper.queryById(loginUser.getTenantId()); + if (tenant == null){ + putMsg(result, Status.TENANT_NOT_EXIST); + return result; + } String hdfsFilename = ""; // delete hdfs file by type + String tenantCode = tenant.getTenantCode(); hdfsFilename = getHdfsFileName(resource, tenantCode, hdfsFilename); //delete data in database @@ -522,8 +529,11 @@ public class ResourcesService extends BaseService { } } - User user = userMapper.queryDetailsById(resource.getUserId()); - String tenantCode = tenantMapper.queryById(user.getTenantId()).getTenantCode(); + String tenantCode = getTenantCode(resource.getUserId(),result); + if (StringUtils.isEmpty(tenantCode)){ + return result; + } + // hdfs path String hdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, resource.getAlias()); logger.info("resource hdfs path is {} ", hdfsFileName); @@ -644,18 +654,20 @@ public class ResourcesService extends BaseService { if (StringUtils.isNotEmpty(resourceViewSuffixs)) { List strList = Arrays.asList(resourceViewSuffixs.split(",")); if (!strList.contains(nameSuffix)) { - logger.error("resouce suffix {} not support updateProcessInstance, resource id {}", nameSuffix, resourceId); + logger.error("resource suffix {} not support updateProcessInstance, resource id {}", nameSuffix, resourceId); putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW); return result; } } + String tenantCode = getTenantCode(resource.getUserId(),result); + if (StringUtils.isEmpty(tenantCode)){ + return result; + } resource.setSize(content.getBytes().length); resource.setUpdateTime(new Date()); resourcesMapper.updateById(resource); - User user = userMapper.queryDetailsById(resource.getUserId()); - String tenantCode = tenantMapper.queryById(user.getTenantId()).getTenantCode(); result = uploadContentToHdfs(resource.getAlias(), tenantCode, content); if (!result.getCode().equals(Status.SUCCESS.getCode())) { @@ -897,4 +909,27 @@ public class ResourcesService extends BaseService { } } + /** + * get tenantCode by UserId + * + * @param userId user id + * @param result return result + * @return + */ + private String getTenantCode(int userId,Result result){ + + User user = userMapper.queryDetailsById(userId); + if(user == null){ + putMsg(result, Status.USER_NOT_EXIST,userId); + return null; + } + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null){ + putMsg(result, Status.TENANT_NOT_EXIST); + return null; + } + return tenant.getTenantCode(); + } + } From 32604123eb94edb0148b0f38e6947e07894f0ae8 Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 2 Jan 2020 16:53:43 +0800 Subject: [PATCH 03/43] add ResourcesServiceTest --- .../api/service/ResourcesServiceTest.java | 587 +++++++++++++++++- pom.xml | 1 + 2 files changed, 576 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 8e94ccac52..c0caaaaa71 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -16,38 +16,601 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.common.utils.HadoopUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.*; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.mock.web.MockMultipartFile; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest({HadoopUtils.class,PropertyUtils.class, FileUtils.class,org.apache.dolphinscheduler.api.utils.FileUtils.class}) public class ResourcesServiceTest { private static final Logger logger = LoggerFactory.getLogger(ResourcesServiceTest.class); - @Autowired + @InjectMocks private ResourcesService resourcesService; + @Mock + private ResourceMapper resourcesMapper; + @Mock + private TenantMapper tenantMapper; + @Mock + private ResourceUserMapper resourceUserMapper; + @Mock + private HadoopUtils hadoopUtils; + @Mock + private UserMapper userMapper; + @Mock + private UdfFuncMapper udfFunctionMapper; + + @Before + public void setUp() { + + PowerMockito.mockStatic(HadoopUtils.class); + PowerMockito.mockStatic(FileUtils.class); + PowerMockito.mockStatic(org.apache.dolphinscheduler.api.utils.FileUtils.class); + try { + // new HadoopUtils + PowerMockito.whenNew(HadoopUtils.class).withNoArguments().thenReturn(hadoopUtils); + } catch (Exception e) { + e.printStackTrace(); + } + PowerMockito.when(HadoopUtils.getInstance()).thenReturn(hadoopUtils); + PowerMockito.mockStatic(PropertyUtils.class); + } + + @Test + public void testCreateResource(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + User user = new User(); + //HDFS_NOT_STARTUP + Result result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + + //RESOURCE_FILE_IS_EMPTY + MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf",new String().getBytes()); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(),result.getMsg()); + + //RESOURCE_SUFFIX_FORBID_CHANGE + mockMultipartFile = new MockMultipartFile("test.pdf","test.pdf","pdf",new String("test").getBytes()); + PowerMockito.when(FileUtils.suffix("test.pdf")).thenReturn("pdf"); + PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); + result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(),result.getMsg()); + + //UDF_RESOURCE_SUFFIX_NOT_JAR + mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf","ResourcesServiceTest.pdf","pdf",new String("test").getBytes()); + result = resourcesService.createResource(user,"ResourcesServiceTest.pdf","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile); + logger.info(result.toString()); + Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(),result.getMsg()); + + + //UDF_RESOURCE_SUFFIX_NOT_JAR + Mockito.when(tenantMapper.queryById(0)).thenReturn(getTenant()); + Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest.jar", 0, 1)).thenReturn(getResourceList()); + mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.jar","ResourcesServiceTest.jar","pdf",new String("test").getBytes()); + result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); + + //SUCCESS + Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest.jar", 0, 1)).thenReturn(new ArrayList<>()); + result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + + } + + @Test + public void testUpdateResource(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + User user = new User(); + //HDFS_NOT_STARTUP + Result result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + + //RESOURCE_NOT_EXIST + Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = resourcesService.updateResource(user,0,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + + //USER_NO_OPERATION_PERM + result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(),result.getMsg()); + + //SUCCESS + user.setId(1); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest.jar","ResourcesServiceTest.jar",ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + + //RESOURCE_EXIST + Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest1.jar", 0, 0)).thenReturn(getResourceList()); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); + //USER_NOT_EXIST + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + logger.info(result.toString()); + Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); + + //TENANT_NOT_EXIST + Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); + + //RESOURCE_NOT_EXIST + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + PowerMockito.when(HadoopUtils.getHdfsFilename(Mockito.any(), Mockito.any())).thenReturn("test1"); + + try { + Mockito.when(hadoopUtils.exists("test")).thenReturn(true); + } catch (IOException e) { + e.printStackTrace(); + } + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + + //SUCCESS + PowerMockito.when(HadoopUtils.getHdfsFilename(Mockito.any(), Mockito.any())).thenReturn("test"); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + + } + + @Test + public void testQueryResourceListPaging(){ + User loginUser = new User(); + loginUser.setUserType(UserType.ADMIN_USER); + IPage resourcePage = new Page<>(1,10); + resourcePage.setTotal(1); + resourcePage.setRecords(getResourceList()); + Mockito.when(resourcesMapper.queryResourcePaging(Mockito.any(Page.class), + Mockito.eq(0), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage); + Map result = resourcesService.queryResourceListPaging(loginUser,ResourceType.FILE,"test",1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + + } + + @Test + public void testQueryResourceList(){ + User loginUser = new User(); + loginUser.setId(0); + loginUser.setUserType(UserType.ADMIN_USER); + Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0)).thenReturn(getResourceList()); + Map result = resourcesService.queryResourceList(loginUser, ResourceType.FILE); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + List resourceList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(resourceList)); + } @Test - public void querytResourceList(){ + public void testDelete(){ + User loginUser = new User(); - loginUser.setId(-1); - loginUser.setUserType(UserType.GENERAL_USER); + loginUser.setId(0); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + + try { + // HDFS_NOT_STARTUP + Result result = resourcesService.delete(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); + + //RESOURCE_NOT_EXIST + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); + result = resourcesService.delete(loginUser,2); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); + + // USER_NO_OPERATION_PERM + result = resourcesService.delete(loginUser,2); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); + + //TENANT_NOT_EXIST + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setTenantId(2); + result = resourcesService.delete(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); + + //SUCCESS + loginUser.setTenantId(1); + Mockito.when(hadoopUtils.delete(Mockito.anyString(), Mockito.anyBoolean())).thenReturn(true); + result = resourcesService.delete(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + + } catch (Exception e) { + logger.error("delete error",e); + Assert.assertTrue(false); + } + } + + @Test + public void testVerifyResourceName(){ + + User user = new User(); + user.setId(1); + Mockito.when(resourcesMapper.queryResourceList("test", 0, 0)).thenReturn(getResourceList()); + Result result = resourcesService.verifyResourceName("test",ResourceType.FILE,user); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); + + //TENANT_NOT_EXIST + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + result = resourcesService.verifyResourceName("test1",ResourceType.FILE,user); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); + + + //RESOURCE_FILE_EXIST + user.setTenantId(1); + try { + Mockito.when(hadoopUtils.exists("test")).thenReturn(true); + } catch (IOException e) { + logger.error("hadoop error",e); + } + PowerMockito.when(HadoopUtils.getHdfsFilename("123", "test1")).thenReturn("test"); + result = resourcesService.verifyResourceName("test1",ResourceType.FILE,user); + logger.info(result.toString()); + Assert.assertTrue(Status.RESOURCE_FILE_EXIST.getCode()==result.getCode()); + + //SUCCESS + result = resourcesService.verifyResourceName("test2",ResourceType.FILE,user); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + + } + + @Test + public void testReadResource(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + + //HDFS_NOT_STARTUP + Result result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + + //RESOURCE_NOT_EXIST + Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = resourcesService.readResource(2,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + + + //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + + //USER_NOT_EXIST + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); + PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); + result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertTrue(Status.USER_NOT_EXIST.getCode()==result.getCode()); + + + //TENANT_NOT_EXIST + Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); + result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); + + + //RESOURCE_FILE_NOT_EXIST + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + try { + Mockito.when(hadoopUtils.exists(Mockito.anyString())).thenReturn(false); + } catch (IOException e) { + logger.error("hadoop error",e); + } + result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertTrue(Status.RESOURCE_FILE_NOT_EXIST.getCode()==result.getCode()); + + //SUCCESS + try { + Mockito.when(hadoopUtils.exists(null)).thenReturn(true); + Mockito.when(hadoopUtils.catFile(null,1,10)).thenReturn(getContent()); + } catch (IOException e) { + logger.error("hadoop error",e); + } + result = resourcesService.readResource(1,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); - Map map = resourcesService.queryResourceList(loginUser, ResourceType.FILE); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); + + } + + @Test + public void testOnlineCreateResource() { + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + PowerMockito.when(HadoopUtils.getHdfsResDir("hdfsdDir")).thenReturn("hdfsDir"); + PowerMockito.when(HadoopUtils.getHdfsUdfDir("udfDir")).thenReturn("udfDir"); + User user = getUser(); + //HDFS_NOT_STARTUP + Result result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content"); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + + //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); + result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content"); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + + //RuntimeException + try { + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content"); + }catch (RuntimeException ex){ + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), ex.getMessage()); + } + + //SUCCESS + Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); + PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); + result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + + + } + + @Test + public void testUpdateResourceContent(){ + + User loginUser = new User(); + loginUser.setId(0); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + + // HDFS_NOT_STARTUP + Result result = resourcesService.updateResourceContent(1,"content"); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); + + //RESOURCE_NOT_EXIST + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); + result = resourcesService.updateResourceContent(2,"content"); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); + + //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); + result = resourcesService.updateResourceContent(1,"content"); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + + //USER_NOT_EXIST + PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); + PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); + result = resourcesService.updateResourceContent(1,"content"); + logger.info(result.toString()); + Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); + + + //TENANT_NOT_EXIST + Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); + result = resourcesService.updateResourceContent(1,"content"); + logger.info(result.toString()); + Assert.assertTrue(Status.TENANT_NOT_EXIST.getCode() == result.getCode()); + + //SUCCESS + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); + PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); + result = resourcesService.updateResourceContent(1,"content"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + } + + @Test + public void testDownloadResource(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); + org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class); + try { + //resource null + org.springframework.core.io.Resource resource = resourcesService.downloadResource(1); + Assert.assertNull(resource); + + Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); + PowerMockito.when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any())).thenReturn(resourceMock); + resource = resourcesService.downloadResource(1); + Assert.assertNotNull(resource); + } catch (Exception e) { + logger.error("DownloadResource error",e); + Assert.assertTrue(false); + } + + } + + @Test + public void testUnauthorizedFile(){ + User user = getUser(); + //USER_NO_OPERATION_PERM + Map result = resourcesService.unauthorizedFile(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + + //SUCCESS + user.setUserType(UserType.ADMIN_USER); + Mockito.when(resourcesMapper.queryResourceExceptUserId(1)).thenReturn(getResourceList()); + result = resourcesService.unauthorizedFile(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List resources = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(resources)); + + } + + @Test + public void testUnauthorizedUDFFunction(){ + + User user = getUser(); + //USER_NO_OPERATION_PERM + Map result = resourcesService.unauthorizedUDFFunction(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + + //SUCCESS + user.setUserType(UserType.ADMIN_USER); + Mockito.when(udfFunctionMapper.queryUdfFuncExceptUserId(1)).thenReturn(getUdfFuncList()); + result = resourcesService.unauthorizedUDFFunction(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List udfFuncs = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs)); + } + + + @Test + public void testAuthorizedUDFFunction(){ + User user = getUser(); + //USER_NO_OPERATION_PERM + Map result = resourcesService.authorizedUDFFunction(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + //SUCCESS + user.setUserType(UserType.ADMIN_USER); + Mockito.when(udfFunctionMapper.queryAuthedUdfFunc(1)).thenReturn(getUdfFuncList()); + result = resourcesService.authorizedUDFFunction(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List udfFuncs = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs)); + } + + @Test + public void testAuthorizedFile(){ + + User user = getUser(); + //USER_NO_OPERATION_PERM + Map result = resourcesService.authorizedFile(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + //SUCCESS + user.setUserType(UserType.ADMIN_USER); + Mockito.when(resourcesMapper.queryAuthorizedResourceList(1)).thenReturn(getResourceList()); + result = resourcesService.authorizedFile(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List resources = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(resources)); + } + + + private List getResourceList(){ + + List resources = new ArrayList<>(); + resources.add(getResource()); + return resources; + } + + + private Tenant getTenant() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("123"); + return tenant; + } + + private Resource getResource(){ + + Resource resource = new Resource(); + resource.setUserId(1); + resource.setDescription("ResourcesServiceTest.jar"); + resource.setAlias("ResourcesServiceTest.jar"); + resource.setType(ResourceType.FILE); + return resource; + } + + private UdfFunc getUdfFunc(){ + + UdfFunc udfFunc = new UdfFunc(); + udfFunc.setId(1); + return udfFunc; + } + + private List getUdfFuncList(){ + + List udfFuncs = new ArrayList<>(); + udfFuncs.add(getUdfFunc()); + return udfFuncs; + } + + private User getUser(){ + User user = new User(); + user.setId(1); + user.setTenantId(1); + return user; + } + private List getContent(){ + List contentList = new ArrayList<>(); + contentList.add("test"); + return contentList; } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 6892b387f1..090babb7cb 100644 --- a/pom.xml +++ b/pom.xml @@ -676,6 +676,7 @@ **/api/service/AlertGroupServiceTest.java **/api/service/ProjectServiceTest.java **/api/service/UdfFuncServiceTest.java + **/api/service/ResourcesServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From b0d73fb7897d94e318d63748d87b271f6adf984b Mon Sep 17 00:00:00 2001 From: lilin Date: Sat, 4 Jan 2020 11:40:53 +0800 Subject: [PATCH 04/43] add error log --- .../apache/dolphinscheduler/api/service/ResourcesService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 33d2c96071..8a6ffb8a18 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -920,12 +920,14 @@ public class ResourcesService extends BaseService { User user = userMapper.queryDetailsById(userId); if(user == null){ + logger.error("user {} not exists", userId); putMsg(result, Status.USER_NOT_EXIST,userId); return null; } Tenant tenant = tenantMapper.queryById(user.getTenantId()); if (tenant == null){ + logger.error("tenant not exists"); putMsg(result, Status.TENANT_NOT_EXIST); return null; } From b4a6d5440fe62da1135acc7b3a6134ae890a6f29 Mon Sep 17 00:00:00 2001 From: lilin Date: Wed, 15 Jan 2020 14:11:09 +0800 Subject: [PATCH 05/43] add BaseServiceTest BaseServiceTest UT --- .../api/service/BaseDAGServiceTest.java | 50 +++++++ .../api/service/BaseServiceTest.java | 140 ++++++++++++++++++ pom.xml | 2 + 3 files changed, 192 insertions(+) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java new file mode 100644 index 0000000000..bb6e3882fe --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.common.graph.DAG; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.model.TaskNodeRelation; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class BaseDAGServiceTest { + + @Test + public void testProcessInstance2DAG(){ + + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setProcessInstanceJson("{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-61567\"," + + "\"name\":\"开始\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo '1'\"}," + + "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + + "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + "\"workerGroupId\":-1,\"preTasks\":[]},{\"type\":\"SHELL\",\"id\":\"tasks-6-3ug5ej\",\"name\":\"结束\"," + + "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo '1'\"},\"description\":\"\"," + + "\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + + "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + "\"workerGroupId\":-1,\"preTasks\":[\"开始\"]}],\"tenantId\":-1,\"timeout\":0}"); + + DAG relationDAG = BaseDAGService.processInstance2DAG(processInstance); + + Assert.assertTrue(relationDAG.containsNode("开始")); + + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java new file mode 100644 index 0000000000..02086a8259 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.HadoopUtils; +import org.apache.dolphinscheduler.dao.entity.User; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.mock.web.MockCookie; +import org.springframework.mock.web.MockHttpServletRequest; + +import javax.servlet.http.Cookie; +import java.util.HashMap; +import java.util.Map; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"sun.security.*", "javax.net.*"}) +@PrepareForTest({HadoopUtils.class}) +public class BaseServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(BaseServiceTest.class); + + private BaseService baseService; + + @Mock + private HadoopUtils hadoopUtils; + + @Before + public void setUp() { + baseService = new BaseService(); + } + + @Test + public void testIsAdmin(){ + + User user = new User(); + user.setUserType(UserType.ADMIN_USER); + //ADMIN_USER + boolean isAdmin = baseService.isAdmin(user); + Assert.assertTrue(isAdmin); + //GENERAL_USER + user.setUserType(UserType.GENERAL_USER); + isAdmin = baseService.isAdmin(user); + Assert.assertFalse(isAdmin); + + } + + @Test + public void testPutMsg(){ + + Map result = new HashMap<>(); + baseService.putMsg(result, Status.SUCCESS); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + //has params + baseService.putMsg(result, Status.PROJECT_NOT_FOUNT,"test"); + + } + @Test + public void testPutMsgTwo(){ + + Result result = new Result(); + baseService.putMsg(result, Status.SUCCESS); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + //has params + baseService.putMsg(result,Status.PROJECT_NOT_FOUNT,"test"); + } + @Test + public void testGetCookie(){ + + MockHttpServletRequest request = new MockHttpServletRequest(); + MockCookie mockCookie = new MockCookie("userId","1"); + request.setCookies(mockCookie); + //cookie is not null + Cookie cookie = BaseService.getCookie(request,"userId"); + Assert.assertNotNull(cookie); + //cookie is null + cookie = BaseService.getCookie(request,"userName"); + Assert.assertNull(cookie); + + } + @Test + public void testCreateTenantDirIfNotExists(){ + + PowerMockito.mockStatic(HadoopUtils.class); + PowerMockito.when(HadoopUtils.getInstance()).thenReturn(hadoopUtils); + + try { + baseService.createTenantDirIfNotExists("test"); + } catch (Exception e) { + Assert.assertTrue(false); + logger.error("CreateTenantDirIfNotExists error ",e); + e.printStackTrace(); + } + + } + @Test + public void testHasPerm(){ + + User user = new User(); + user.setId(1); + //create user + boolean hasPerm = baseService.hasPerm(user,1); + Assert.assertTrue(hasPerm); + + //admin + user.setId(2); + user.setUserType(UserType.ADMIN_USER); + hasPerm = baseService.hasPerm(user,1); + Assert.assertTrue(hasPerm); + + } + +} diff --git a/pom.xml b/pom.xml index 4468c72e78..41c75f7d5f 100644 --- a/pom.xml +++ b/pom.xml @@ -694,6 +694,8 @@ **/api/service/ProcessDefinitionServiceTest.java **/api/service/UdfFuncServiceTest.java **/api/service/ResourcesServiceTest.java + **/api/service/BaseServiceTest.java + **/api/service/BaseDAGServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From b34f9745dc63e8e95407f5c0612db13b649b6e4a Mon Sep 17 00:00:00 2001 From: lilin Date: Wed, 15 Jan 2020 14:56:17 +0800 Subject: [PATCH 06/43] update --- .../dolphinscheduler/api/service/ResourcesService.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 77da00e457..09b1d31151 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -235,12 +235,6 @@ public class ResourcesService extends BaseService { return result; } - // query tenant by user id - String tenantCode = getTenantCode(resource.getUserId(),result); - if (StringUtils.isEmpty(tenantCode)){ - return result; - } - //get the file suffix String originResourceName = resource.getAlias(); String suffix = originResourceName.substring(originResourceName.lastIndexOf(".")); From 19bde513504d991443995d0a3246afc86bc537bb Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 16 Jan 2020 15:43:58 +0800 Subject: [PATCH 07/43] fix word spelling --- .../server/master/runner/MasterExecThread.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index a91f8c17e6..49154faece 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -224,14 +224,14 @@ public class MasterExecThread implements Runnable { // execute process ,waiting for end runProcess(); - // process instace failure ，no more complements + // process instace failure ,no more complements if(!processInstance.getState().typeIsSuccess()){ logger.info("process {} state {}, complement not completely!", processInstance.getId(), processInstance.getState()); break; } - // current process instance sucess ，next execute + // current process instance success ,next execute scheduleDate = DateUtils.getSomeDay(scheduleDate, 1); if(scheduleDate.after(endDate)){ // all success @@ -541,7 +541,7 @@ public class MasterExecThread implements Runnable { private DependResult isTaskDepsComplete(String taskName) { Collection startNodes = dag.getBeginNode(); - // ff the vertex returns true directly + // if the vertex returns true directly if(startNodes.contains(taskName)){ return DependResult.SUCCESS; } From 55923016b49c4e9c0a425a2ea0be890a1e5635de Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 16 Jan 2020 15:57:16 +0800 Subject: [PATCH 08/43] Revert "fix word spelling" This reverts commit 19bde513504d991443995d0a3246afc86bc537bb. --- .../server/master/runner/MasterExecThread.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 49154faece..a91f8c17e6 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -224,14 +224,14 @@ public class MasterExecThread implements Runnable { // execute process ,waiting for end runProcess(); - // process instace failure ,no more complements + // process instace failure ，no more complements if(!processInstance.getState().typeIsSuccess()){ logger.info("process {} state {}, complement not completely!", processInstance.getId(), processInstance.getState()); break; } - // current process instance success ,next execute + // current process instance sucess ，next execute scheduleDate = DateUtils.getSomeDay(scheduleDate, 1); if(scheduleDate.after(endDate)){ // all success @@ -541,7 +541,7 @@ public class MasterExecThread implements Runnable { private DependResult isTaskDepsComplete(String taskName) { Collection startNodes = dag.getBeginNode(); - // if the vertex returns true directly + // ff the vertex returns true directly if(startNodes.contains(taskName)){ return DependResult.SUCCESS; } From 9ebe177e1113b845a50336cb1c1221b0265dce6a Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 16 Jan 2020 16:55:45 +0800 Subject: [PATCH 09/43] fix Misspelled words --- .../server/master/runner/MasterExecThread.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 6c147e2628..84b1114b84 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -254,7 +254,7 @@ public class MasterExecThread implements Runnable { break; } - // current process instance sucess ，next execute + // current process instance success ,next execute if(null == iterator){ // loop by day scheduleDate = DateUtils.getSomeDay(scheduleDate, 1); @@ -575,7 +575,7 @@ public class MasterExecThread implements Runnable { private DependResult isTaskDepsComplete(String taskName) { Collection startNodes = dag.getBeginNode(); - // ff the vertex returns true directly + // if the vertex returns true directly if(startNodes.contains(taskName)){ return DependResult.SUCCESS; } From 18750dfc0881180a068edb92a5fbde6d66ac6ef6 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 6 Feb 2020 18:52:44 +0800 Subject: [PATCH 10/43] remove jasper-runtime jar, not need anymore (#1900) * update README about DolphinScheduler * Update issue templates * update * regularize api pom xml update rpc maven compile to 1.8 * regularize api pom xml * change commons.lang3.StringUtils to common.utils.StringUtils * update pom.xml * update * correct equals method * jasper-runtime is needed when api server start * jasper-runtime jar is needed when api server start * combine logback config of master/worker/alert/api server to one logback.xml * remove tomcat runtime jar * add UT * add license * remove jasper-runtime jar, not need anymore Co-authored-by: DS --- dolphinscheduler-api/pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index fd56fb8d9f..bac74da0b0 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -27,10 +27,6 @@ ${project.artifactId} jar - - 5.5.23 - - org.apache.dolphinscheduler @@ -182,12 +178,6 @@ hadoop-aws - - tomcat - jasper-runtime - ${jasper-runtime.version} - - javax.servlet servlet-api From b3b075d9157fc865cc8a090a8f31142bdfa8223f Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Thu, 6 Feb 2020 19:32:31 +0800 Subject: [PATCH 11/43] Fix bug: Use try-with-resources or close this "Statement" in a "finally" clause. (#1702) * #1701 Fix bug: Use try-with-resources or close this "Statement" in a "finally" clause. * fix some indent * refix code smell --- .../common/utils/ScriptRunner.java | 152 ++++++++---------- 1 file changed, 65 insertions(+), 87 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java index 529052c7cf..bbc937c89f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java @@ -16,13 +16,17 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.LineNumberReader; import java.io.Reader; -import java.sql.*; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /* * Slightly modified version of the com.ibatis.common.jdbc.ScriptRunner class @@ -94,9 +98,7 @@ public class ScriptRunner { } finally { connection.setAutoCommit(originalAutoCommit); } - } catch (IOException e) { - throw e; - } catch (SQLException e) { + } catch (IOException | SQLException e) { throw e; } catch (Exception e) { throw new RuntimeException("Error running script. Cause: " + e, e); @@ -114,9 +116,7 @@ public class ScriptRunner { } finally { connection.setAutoCommit(originalAutoCommit); } - } catch (IOException e) { - throw e; - } catch (SQLException e) { + } catch (IOException | SQLException e) { throw e; } catch (Exception e) { throw new RuntimeException("Error running script. Cause: " + e, e); @@ -161,44 +161,34 @@ public class ScriptRunner { || fullLineDelimiter && trimmedLine.equals(getDelimiter())) { command.append(line.substring(0, line.lastIndexOf(getDelimiter()))); command.append(" "); - Statement statement = conn.createStatement(); - boolean hasResults = false; - logger.info("sql:"+command.toString()); - if (stopOnError) { - hasResults = statement.execute(command.toString()); - } else { - try { - statement.execute(command.toString()); - } catch (SQLException e) { - logger.error(e.getMessage(),e); - throw e; - } - } - - ResultSet rs = statement.getResultSet(); - if (hasResults && rs != null) { - ResultSetMetaData md = rs.getMetaData(); - int cols = md.getColumnCount(); - for (int i = 0; i < cols; i++) { - String name = md.getColumnLabel(i); - logger.info(name + "\t"); - } - logger.info(""); - while (rs.next()) { - for (int i = 0; i < cols; i++) { - String value = rs.getString(i); - logger.info(value + "\t"); - } - logger.info(""); - } - } + logger.info("sql: {}", command); + try (Statement statement = conn.createStatement()) { + statement.execute(command.toString()); + try (ResultSet rs = statement.getResultSet()) { + if (stopOnError && rs != null) { + ResultSetMetaData md = rs.getMetaData(); + int cols = md.getColumnCount(); + for (int i = 0; i < cols; i++) { + String name = md.getColumnLabel(i); + logger.info("{} \t", name); + } + logger.info(""); + while (rs.next()) { + for (int i = 0; i < cols; i++) { + String value = rs.getString(i); + logger.info("{} \t", value); + } + logger.info(""); + } + } + } + } catch (SQLException e) { + logger.error("SQLException", e); + throw e; + } + command = null; - try { - statement.close(); - } catch (Exception e) { - // Ignore to workaround a bug in Jakarta DBCP - } Thread.yield(); } else { command.append(line); @@ -207,11 +197,11 @@ public class ScriptRunner { } } catch (SQLException e) { - logger.error("Error executing: " + command.toString()); + logger.error("Error executing: {}", command); throw e; } catch (IOException e) { e.fillInStackTrace(); - logger.error("Error executing: " + command.toString()); + logger.error("Error executing: {}", command); throw e; } } @@ -243,46 +233,35 @@ public class ScriptRunner { || fullLineDelimiter && trimmedLine.equals(getDelimiter())) { command.append(line.substring(0, line.lastIndexOf(getDelimiter()))); command.append(" "); - Statement statement = conn.createStatement(); - sql = command.toString().replaceAll("\\{\\{APPDB\\}\\}", dbName); - boolean hasResults = false; - logger.info("sql : " + sql); - if (stopOnError) { - hasResults = statement.execute(sql); - } else { - try { - statement.execute(sql); - } catch (SQLException e) { - logger.error(e.getMessage(),e); - throw e; - } - } - - ResultSet rs = statement.getResultSet(); - if (hasResults && rs != null) { - ResultSetMetaData md = rs.getMetaData(); - int cols = md.getColumnCount(); - for (int i = 0; i < cols; i++) { - String name = md.getColumnLabel(i); - logger.info(name + "\t"); - } - logger.info(""); - while (rs.next()) { - for (int i = 0; i < cols; i++) { - String value = rs.getString(i); - logger.info(value + "\t"); - } - logger.info(""); - } - } + logger.info("sql : {}", sql); + + try (Statement statement = conn.createStatement()) { + statement.execute(sql); + try (ResultSet rs = statement.getResultSet()) { + if (stopOnError && rs != null) { + ResultSetMetaData md = rs.getMetaData(); + int cols = md.getColumnCount(); + for (int i = 0; i < cols; i++) { + String name = md.getColumnLabel(i); + logger.info("{} \t", name); + } + logger.info(""); + while (rs.next()) { + for (int i = 0; i < cols; i++) { + String value = rs.getString(i); + logger.info("{} \t", value); + } + logger.info(""); + } + } + } + } catch (SQLException e) { + logger.error("SQLException", e); + throw e; + } command = null; - try { - statement.close(); - } catch (Exception e) { - // Ignore to workaround a bug in Jakarta DBCP - } Thread.yield(); } else { command.append(line); @@ -291,11 +270,10 @@ public class ScriptRunner { } } catch (SQLException e) { - logger.error("Error executing: " + sql); throw e; } catch (IOException e) { e.fillInStackTrace(); - logger.error("Error executing: " + sql); + logger.error("Error executing: {}", sql); throw e; } } From 662233e0dd5b09d6099bddeeb8d5812dac05ab19 Mon Sep 17 00:00:00 2001 From: zhangzy Date: Fri, 7 Feb 2020 10:15:42 +0800 Subject: [PATCH 12/43] =?UTF-8?q?Remove=20ScheduleUtil=EF=BC=8Cuse=20the?= =?UTF-8?q?=20existing=20CronUtils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CronUtils is a public quartz tools, then the project dolphinscheduler-api can remove dolphinscheduler-server --- dolphinscheduler-api/pom.xml | 4 - .../api/service/ExecutorService.java | 14 +++- .../master/runner/MasterExecThread.java | 15 +++- .../server/utils/ScheduleUtils.java | 79 ------------------- .../server/utils/ScheduleUtilsTest.java | 44 ----------- pom.xml | 1 - 6 files changed, 23 insertions(+), 134 deletions(-) delete mode 100644 dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ScheduleUtils.java delete mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ScheduleUtilsTest.java diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index bac74da0b0..c10f443384 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -43,10 +43,6 @@ org.apache.dolphinscheduler dolphinscheduler-dao - - org.apache.dolphinscheduler - dolphinscheduler-server - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index c1689c5bec..7c0a8637ad 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -30,7 +30,8 @@ import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; -import org.apache.dolphinscheduler.server.utils.ScheduleUtils; +import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; +import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -512,8 +513,15 @@ public class ExecutorService extends BaseService{ List listDate = new LinkedList<>(); if(!CollectionUtils.isEmpty(schedules)){ for (Schedule item : schedules) { - List list = ScheduleUtils.getRecentTriggerTime(item.getCrontab(), start, end); - listDate.addAll(list); + CronExpression cronExpression = null; + try { + cronExpression = CronUtils.parse2CronExpression(item.getCrontab()); + List list = CronUtils.getSelfFireDateList(start, end, cronExpression); + listDate.addAll(list); + } catch (ParseException e) { + logger.error(e.getMessage(), e); + continue; + } } } if(!CollectionUtils.isEmpty(listDate)){ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 2d0e99a867..1e876239de 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -33,14 +33,16 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.utils.DagHelper; +import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.utils.AlertManager; -import org.apache.dolphinscheduler.server.utils.ScheduleUtils; +import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.text.ParseException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -213,8 +215,15 @@ public class MasterExecThread implements Runnable { List listDate = Lists.newLinkedList(); if(!CollectionUtils.isEmpty(schedules)){ for (Schedule schedule : schedules) { - List list = ScheduleUtils.getRecentTriggerTime(schedule.getCrontab(), startDate, endDate); - listDate.addAll(list); + CronExpression cronExpression = null; + try { + cronExpression = CronUtils.parse2CronExpression(schedule.getCrontab()); + List list = CronUtils.getSelfFireDateList(startDate, endDate, cronExpression); + listDate.addAll(list); + } catch (ParseException e) { + logger.error(e.getMessage(), e); + continue; + } } } // get first fire date diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ScheduleUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ScheduleUtils.java deleted file mode 100644 index 11730b9545..0000000000 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ScheduleUtils.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.server.utils; - -import org.quartz.impl.triggers.CronTriggerImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.text.ParseException; -import java.util.Date; -import java.util.LinkedList; -import java.util.List; - -/** - * ScheduleUtils - */ -public class ScheduleUtils { - - private static final Logger logger = LoggerFactory.getLogger(ScheduleUtils.class); - - /** - * Get the execution time of the time interval - * @param cron - * @param from - * @param to - * @return - */ - public static List getRecentTriggerTime(String cron, Date from, Date to) { - return getRecentTriggerTime(cron, Integer.MAX_VALUE, from, to); - } - - /** - * Get the execution time of the time interval - * @param cron - * @param size - * @param from - * @param to - * @return - */ - public static List getRecentTriggerTime(String cron, int size, Date from, Date to) { - List list = new LinkedList(); - if(to.before(from)){ - logger.error("schedule date from:{} must before date to:{}!", from, to); - return list; - } - try { - CronTriggerImpl trigger = new CronTriggerImpl(); - trigger.setCronExpression(cron); - trigger.setStartTime(from); - trigger.setEndTime(to); - trigger.computeFirstFireTime(null); - for (int i = 0; i < size; i++) { - Date schedule = trigger.getNextFireTime(); - if(null == schedule){ - break; - } - list.add(schedule); - trigger.triggered(null); - } - } catch (ParseException e) { - logger.error("cron:{} error:{}", cron, e.getMessage()); - } - return java.util.Collections.unmodifiableList(list); - } -} \ No newline at end of file diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ScheduleUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ScheduleUtilsTest.java deleted file mode 100644 index 4fbbdab70f..0000000000 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ScheduleUtilsTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.utils; - -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.junit.Test; -import java.util.Date; -import static org.junit.Assert.assertEquals; - -/** - * Test ScheduleUtils - */ -public class ScheduleUtilsTest { - - /** - * Test the getRecentTriggerTime method - */ - @Test - public void testGetRecentTriggerTime() { - Date from = DateUtils.stringToDate("2020-01-01 00:00:00"); - Date to = DateUtils.stringToDate("2020-01-31 01:00:00"); - // test date - assertEquals(0, ScheduleUtils.getRecentTriggerTime("0 0 0 * * ? ", to, from).size()); - // test error cron - assertEquals(0, ScheduleUtils.getRecentTriggerTime("0 0 0 * *", from, to).size()); - // test cron - assertEquals(31, ScheduleUtils.getRecentTriggerTime("0 0 0 * * ? ", from, to).size()); - } -} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 59278f73cd..d1141eed96 100644 --- a/pom.xml +++ b/pom.xml @@ -706,7 +706,6 @@ **/server/utils/SparkArgsUtilsTest.java **/server/utils/FlinkArgsUtilsTest.java **/server/utils/ParamUtilsTest.java - **/server/utils/ScheduleUtilsTest.java **/server/master/MasterExecThreadTest.java **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java From 9d944d3f76dafb421f352e0afaa88312f9321ec8 Mon Sep 17 00:00:00 2001 From: zhangzy Date: Fri, 7 Feb 2020 15:47:07 +0800 Subject: [PATCH 13/43] Optimize duplicate code Use public methods to simplify duplicate code --- .../api/service/ExecutorService.java | 11 +---------- .../dao/utils/cron/CronUtils.java | 17 +++++++++++++++++ .../server/master/runner/MasterExecThread.java | 12 +----------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 7c0a8637ad..257f15d580 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; -import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -513,15 +512,7 @@ public class ExecutorService extends BaseService{ List listDate = new LinkedList<>(); if(!CollectionUtils.isEmpty(schedules)){ for (Schedule item : schedules) { - CronExpression cronExpression = null; - try { - cronExpression = CronUtils.parse2CronExpression(item.getCrontab()); - List list = CronUtils.getSelfFireDateList(start, end, cronExpression); - listDate.addAll(list); - } catch (ParseException e) { - logger.error(e.getMessage(), e); - continue; - } + listDate.addAll(CronUtils.getSelfFireDateList(start, end, item.getCrontab())); } } if(!CollectionUtils.isEmpty(listDate)){ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java index 8649462110..fc5681356d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java @@ -156,6 +156,23 @@ public class CronUtils { return dateList; } + /** + * gets all scheduled times for a period of time based on self dependency + * @param startTime startTime + * @param endTime endTime + * @param cron cron + * @return date list + */ + public static List getSelfFireDateList(Date startTime, Date endTime, String cron) { + CronExpression cronExpression = null; + try { + cronExpression = CronUtils.parse2CronExpression(cron); + }catch (ParseException e){ + logger.error(e.getMessage(), e); + return Collections.EMPTY_LIST; + } + return getSelfFireDateList(startTime, endTime, cronExpression); + } /** * get expiration time diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 1e876239de..2b1ff4d23f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -36,13 +36,11 @@ import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.utils.AlertManager; -import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.text.ParseException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -215,15 +213,7 @@ public class MasterExecThread implements Runnable { List listDate = Lists.newLinkedList(); if(!CollectionUtils.isEmpty(schedules)){ for (Schedule schedule : schedules) { - CronExpression cronExpression = null; - try { - cronExpression = CronUtils.parse2CronExpression(schedule.getCrontab()); - List list = CronUtils.getSelfFireDateList(startDate, endDate, cronExpression); - listDate.addAll(list); - } catch (ParseException e) { - logger.error(e.getMessage(), e); - continue; - } + listDate.addAll(CronUtils.getSelfFireDateList(startDate, endDate, schedule.getCrontab())); } } // get first fire date From fccbcc8d7df592cc591568f6e7b328abdbce7602 Mon Sep 17 00:00:00 2001 From: Rubik <39549317+wenhemin@users.noreply.github.com> Date: Fri, 7 Feb 2020 16:13:50 +0800 Subject: [PATCH 14/43] fix: The constructor has passed in an taskAppId, no need to get from taskAppId. (#1906) --- .../server/worker/task/AbstractCommandExecutor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 0c62f0648a..04098215dd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -147,9 +147,6 @@ public abstract class AbstractCommandExecutor { // get process id int pid = getProcessId(process); - // task instance id - int taskInstId = Integer.parseInt(taskAppId.split("_")[2]); - processDao.updatePidByTaskInstId(taskInstId, pid, ""); logger.info("process start, process id is: {}", pid); From a09d2d9d0a49127f0c7ba7eb86cef54ca5928d81 Mon Sep 17 00:00:00 2001 From: zhangzy Date: Fri, 7 Feb 2020 16:43:24 +0800 Subject: [PATCH 15/43] add CronUtils ut --- .../dao/utils/cron/CronUtils.java | 2 +- .../dolphinscheduler/dao/cron/CronUtilsTest.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java index fc5681356d..8a9087a33c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java @@ -166,7 +166,7 @@ public class CronUtils { public static List getSelfFireDateList(Date startTime, Date endTime, String cron) { CronExpression cronExpression = null; try { - cronExpression = CronUtils.parse2CronExpression(cron); + cronExpression = parse2CronExpression(cron); }catch (ParseException e){ logger.error(e.getMessage(), e); return Collections.EMPTY_LIST; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java index c375143d7b..05ebde9c0f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao.cron; import org.apache.dolphinscheduler.common.enums.CycleEnum; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; import com.cronutils.builder.CronBuilder; import com.cronutils.model.Cron; @@ -31,6 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.text.ParseException; +import java.util.Date; import static com.cronutils.model.field.expression.FieldExpressionFactory.*; @@ -180,4 +182,17 @@ public class CronUtilsTest { } } } + + + @Test + public void test3(){ + Date from = DateUtils.stringToDate("2020-01-01 00:00:00"); + Date to = DateUtils.stringToDate("2020-01-31 01:00:00"); + // test date + Assert.assertEquals(0, CronUtils.getSelfFireDateList(to, from, "0 0 0 * * ? ").size()); + // test error cron + Assert.assertEquals(0, CronUtils.getSelfFireDateList(from, to, "0 0 0 * *").size()); + // test cron + Assert.assertEquals(30, CronUtils.getSelfFireDateList(from, to, "0 0 0 * * ? ").size()); + } } From 515255b9199eecb944b498a1506b0c1b5ff65563 Mon Sep 17 00:00:00 2001 From: zhangzy Date: Fri, 7 Feb 2020 17:18:27 +0800 Subject: [PATCH 16/43] ut --- .../dolphinscheduler/server/master/MasterExecThreadTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index 6f31e66213..d7c3de13a5 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -134,7 +134,7 @@ public class MasterExecThreadTest { method.setAccessible(true); method.invoke(masterExecThread); // one create save, and 15(1 to 31 step 2) for next save, and last day 31 no save - verify(processDao, times(16)).saveProcessInstance(processInstance); + verify(processDao, times(15)).saveProcessInstance(processInstance); }catch (Exception e){ Assert.assertTrue(false); } From 61593cfe6b2a36a287b9e6a60ec3aba516e953e4 Mon Sep 17 00:00:00 2001 From: zhangzy Date: Fri, 7 Feb 2020 17:35:12 +0800 Subject: [PATCH 17/43] ut change --- .../dolphinscheduler/api/service/ExecutorService2Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java index b4f3e7e31f..66c7a3ebab 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java @@ -203,7 +203,7 @@ public class ExecutorService2Test { "", "", RunMode.RUN_MODE_PARALLEL, Priority.LOW, 0, 110); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - verify(processDao, times(16)).createCommand(any(Command.class)); + verify(processDao, times(15)).createCommand(any(Command.class)); }catch (Exception e){ Assert.assertTrue(false); } From 6b586f31fc0416f72363db31039c7d41c4500cbd Mon Sep 17 00:00:00 2001 From: zhangzy Date: Fri, 7 Feb 2020 18:34:38 +0800 Subject: [PATCH 18/43] =?UTF-8?q?pom=20add=20ut=EF=BC=8C=20add=20cronutils?= =?UTF-8?q?=20ut?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dao/cron/CronUtilsTest.java | 36 ++++++++++++++++--- pom.xml | 1 + 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java index 05ebde9c0f..8cf1b6eaf3 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java @@ -70,9 +70,9 @@ public class CronUtilsTest { @Test public void testCronParse() throws ParseException { String strCrontab = "0 1 2 3 * ? *"; - strCrontab = "0/50 0/59 * * * ? *"; - strCrontab = "3/5 * 0/5 * * ? *"; - strCrontab = "1/5 3/5 1/5 3/30 * ? *"; +// strCrontab = "0/50 0/59 * * * ? *"; +// strCrontab = "3/5 * 0/5 * * ? *"; +// strCrontab = "1/5 3/5 1/5 3/30 * ? *"; Cron depCron = CronUtils.parse2Cron(strCrontab); Assert.assertEquals(depCron.retrieve(CronFieldName.SECOND).getExpression().asString(), "0"); @@ -90,11 +90,14 @@ public class CronUtilsTest { @Test public void testScheduleType() throws ParseException { - CycleEnum cycleEnum = CronUtils.getMaxCycle("0 */1 * * * ? *"); + CycleEnum cycleEnum = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 */1 * * * ? *")); Assert.assertEquals(cycleEnum.name(), "MINUTE"); CycleEnum cycleEnum2 = CronUtils.getMaxCycle("0 * * * * ? *"); Assert.assertEquals(cycleEnum2.name(), "MINUTE"); + + CycleEnum cycleEnum3 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 * * * * ? *")); + Assert.assertEquals(cycleEnum3.name(), "MINUTE"); } /** @@ -185,7 +188,12 @@ public class CronUtilsTest { @Test - public void test3(){ + public void parse2Cron(){ + + } + + @Test + public void getSelfFireDateList() throws ParseException{ Date from = DateUtils.stringToDate("2020-01-01 00:00:00"); Date to = DateUtils.stringToDate("2020-01-31 01:00:00"); // test date @@ -194,5 +202,23 @@ public class CronUtilsTest { Assert.assertEquals(0, CronUtils.getSelfFireDateList(from, to, "0 0 0 * *").size()); // test cron Assert.assertEquals(30, CronUtils.getSelfFireDateList(from, to, "0 0 0 * * ? ").size()); + // test other + Assert.assertEquals(30, CronUtils.getFireDateList(from, to, CronUtils.parse2CronExpression("0 0 0 * * ? ")).size()); + Assert.assertEquals(5, CronUtils.getSelfFireDateList(from, to, CronUtils.parse2CronExpression("0 0 0 * * ? "), 5).size()); + } + + @Test + public void getExpirationTime(){ + Date startTime = DateUtils.stringToDate("2020-02-07 18:30:00"); + Date expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.HOUR); + Assert.assertEquals("2020-02-07 19:30:00", DateUtils.dateToString(expirationTime)); + expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.DAY); + Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime)); + expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.WEEK); + Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime)); + expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.MONTH); + Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime)); + expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.YEAR); + Assert.assertEquals("2020-02-07 18:30:00", DateUtils.dateToString(expirationTime)); } } diff --git a/pom.xml b/pom.xml index d1141eed96..875577c672 100644 --- a/pom.xml +++ b/pom.xml @@ -711,6 +711,7 @@ **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/AlertMapperTest.java **/dao/mapper/CommandMapperTest.java + **/dao/cron/CronUtilsTest.java **/alert/template/AlertTemplateFactoryTest.java **/alert/template/impl/DefaultHTMLTemplateTest.java **/server/worker/task/datax/DataxTaskTest.java From 0eec3b8661adc6153409b8cf2c8674b855c8aa65 Mon Sep 17 00:00:00 2001 From: zhangzy Date: Fri, 7 Feb 2020 19:17:47 +0800 Subject: [PATCH 19/43] Remove useless code --- .../dao/cron/CronUtilsTest.java | 38 +++---------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java index 8cf1b6eaf3..1135cf20f5 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java @@ -37,6 +37,7 @@ import java.util.Date; import static com.cronutils.model.field.expression.FieldExpressionFactory.*; /** + * CronUtilsTest */ public class CronUtilsTest { @@ -57,8 +58,9 @@ public class CronUtilsTest { .withSecond(on(0)) .instance(); // Obtain the string expression - String cronAsString = cron.asString(); // 0 */5 * * * ? * Every five minutes(once every 5 minutes) + String cronAsString = cron.asString(); + // 0 */5 * * * ? * Every five minutes(once every 5 minutes) Assert.assertEquals(cronAsString, "0 */5 * * * ? *"); } @@ -70,9 +72,6 @@ public class CronUtilsTest { @Test public void testCronParse() throws ParseException { String strCrontab = "0 1 2 3 * ? *"; -// strCrontab = "0/50 0/59 * * * ? *"; -// strCrontab = "3/5 * 0/5 * * ? *"; -// strCrontab = "1/5 3/5 1/5 3/30 * ? *"; Cron depCron = CronUtils.parse2Cron(strCrontab); Assert.assertEquals(depCron.retrieve(CronFieldName.SECOND).getExpression().asString(), "0"); @@ -89,7 +88,6 @@ public class CronUtilsTest { */ @Test public void testScheduleType() throws ParseException { - CycleEnum cycleEnum = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 */1 * * * ? *")); Assert.assertEquals(cycleEnum.name(), "MINUTE"); @@ -114,26 +112,9 @@ public class CronUtilsTest { .withMinute(every(5)) .withSecond(on(0)) .instance(); - - String cronAsString = cron1.asString(); // 0 */5 * * * ? * once every 5 minutes - //logger.info(cronAsString); - // Obtain the string expression - //String minCrontab = "0 0 * * * ? *"; - //String minCrontab = "0 0 10,14,16 * * ?"; - //String minCrontab = "0 0-5 14 * * ? *"; - //String minCrontab = "0 0 2 ? * SUN *"; - //String minCrontab = "* 0,3 2 SUN * 1#1 *"; - //String minCrontab = "* 0,3 * 1W * ? *"; - //cron = CronUtils.parse2Cron("0 * * * * ? *"); - // month cycle - /*String[] cronArayy = new String[]{"* 0,3 * 1W * ? *","* 0 0 1W * ? *", - "0 0 0 L 3/5 ? *","0 0 0 ? 3/5 2/2 *"};*/ // minute cycle String[] cronArayy = new String[]{"* * * * * ? *","* 0 * * * ? *", "* 5 * * 3/5 ? *","0 0 * * * ? *"}; - // week cycle - /*String[] cronArayy = new String[]{"* * * ? * 2/1 *","0 *//*5 * ? * 2/1 *", - "* * *//*5 ? * 2/1 *"};*/ for(String minCrontab:cronArayy){ if (!org.quartz.CronExpression.isValidExpression(minCrontab)) { throw new RuntimeException(minCrontab+" verify failure, cron expression not valid"); @@ -176,7 +157,6 @@ public class CronUtilsTest { logger.info("dayOfWeekField instanceof And:"+(dayOfWeekField.getExpression() instanceof And)); logger.info("dayOfWeekField instanceof QuestionMark:"+(dayOfWeekField.getExpression() instanceof QuestionMark)); - CycleEnum cycleEnum = CronUtils.getMaxCycle(minCrontab); if(cycleEnum !=null){ logger.info(cycleEnum.name()); @@ -186,22 +166,16 @@ public class CronUtilsTest { } } - - @Test - public void parse2Cron(){ - - } - @Test public void getSelfFireDateList() throws ParseException{ Date from = DateUtils.stringToDate("2020-01-01 00:00:00"); - Date to = DateUtils.stringToDate("2020-01-31 01:00:00"); + Date to = DateUtils.stringToDate("2020-01-31 00:00:00"); // test date Assert.assertEquals(0, CronUtils.getSelfFireDateList(to, from, "0 0 0 * * ? ").size()); // test error cron Assert.assertEquals(0, CronUtils.getSelfFireDateList(from, to, "0 0 0 * *").size()); // test cron - Assert.assertEquals(30, CronUtils.getSelfFireDateList(from, to, "0 0 0 * * ? ").size()); + Assert.assertEquals(29, CronUtils.getSelfFireDateList(from, to, "0 0 0 * * ? ").size()); // test other Assert.assertEquals(30, CronUtils.getFireDateList(from, to, CronUtils.parse2CronExpression("0 0 0 * * ? ")).size()); Assert.assertEquals(5, CronUtils.getSelfFireDateList(from, to, CronUtils.parse2CronExpression("0 0 0 * * ? "), 5).size()); @@ -221,4 +195,4 @@ public class CronUtilsTest { expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.YEAR); Assert.assertEquals("2020-02-07 18:30:00", DateUtils.dateToString(expirationTime)); } -} +} \ No newline at end of file From 3364a8c8a971f17b0483f2c9161b4fe814607a9c Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Sat, 8 Feb 2020 21:22:20 +0800 Subject: [PATCH 20/43] add script runner ut --- .../common/utils/ScriptRunnerTest.java | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java new file mode 100644 index 0000000000..165b3ed50f --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import java.io.StringReader; +import java.sql.*; + +public class ScriptRunnerTest { + @Test + public void testRunScript() { + //connection is null + Exception exception = null; + ScriptRunner s = new ScriptRunner(null, true, true); + try { + s.runScript(new StringReader("select 1")); + } catch (Exception e) { + exception = e; + } + Assert.assertNotNull(exception); + + //connect is not null + try { + Connection conn = Mockito.mock(Connection.class); + Mockito.when(conn.getAutoCommit()).thenReturn(true); + PreparedStatement st = Mockito.mock(PreparedStatement.class); + Mockito.when(conn.createStatement()).thenReturn(st); + ResultSet rs = Mockito.mock(ResultSet.class); + Mockito.when(st.getResultSet()).thenReturn(rs); + ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class); + Mockito.when(rs.getMetaData()).thenReturn(md); + Mockito.when(md.getColumnCount()).thenReturn(1); + Mockito.when(rs.next()).thenReturn(true, false); + ScriptRunner s2 = new ScriptRunner(conn, true, true); + s2.runScript(new StringReader("select 1;")); + Mockito.verify(md).getColumnLabel(0); + } catch(Exception e) { + Assert.assertNotNull(e); + } + } + + @Test + public void testRunScriptWithDbName() { + //connect is not null + try { + Connection conn = Mockito.mock(Connection.class); + Mockito.when(conn.getAutoCommit()).thenReturn(true); + PreparedStatement st = Mockito.mock(PreparedStatement.class); + Mockito.when(conn.createStatement()).thenReturn(st); + ResultSet rs = Mockito.mock(ResultSet.class); + Mockito.when(st.getResultSet()).thenReturn(rs); + ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class); + Mockito.when(rs.getMetaData()).thenReturn(md); + Mockito.when(md.getColumnCount()).thenReturn(1); + Mockito.when(rs.next()).thenReturn(true, false); + ScriptRunner s = new ScriptRunner(conn, true, true); + s.runScript(new StringReader("select 1;"), "test_db"); + Mockito.verify(md).getColumnLabel(0); + } catch(Exception e) { + Assert.assertNotNull(e); + } + } +} From bd10b78e292c12995e17fee32f068075a1c9e268 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Sun, 9 Feb 2020 10:09:31 +0800 Subject: [PATCH 21/43] delete dup code --- .../common/utils/ScriptRunnerTest.java | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java index 165b3ed50f..a36253f8ef 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java @@ -36,41 +36,34 @@ public class ScriptRunnerTest { //connect is not null try { - Connection conn = Mockito.mock(Connection.class); - Mockito.when(conn.getAutoCommit()).thenReturn(true); - PreparedStatement st = Mockito.mock(PreparedStatement.class); - Mockito.when(conn.createStatement()).thenReturn(st); - ResultSet rs = Mockito.mock(ResultSet.class); - Mockito.when(st.getResultSet()).thenReturn(rs); - ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class); - Mockito.when(rs.getMetaData()).thenReturn(md); - Mockito.when(md.getColumnCount()).thenReturn(1); - Mockito.when(rs.next()).thenReturn(true, false); - ScriptRunner s2 = new ScriptRunner(conn, true, true); + ScriptRunner s2 = getScriptRunner(); s2.runScript(new StringReader("select 1;")); - Mockito.verify(md).getColumnLabel(0); } catch(Exception e) { Assert.assertNotNull(e); } } + private ScriptRunner getScriptRunner() throws SQLException { + Connection conn = Mockito.mock(Connection.class); + Mockito.when(conn.getAutoCommit()).thenReturn(true); + PreparedStatement st = Mockito.mock(PreparedStatement.class); + Mockito.when(conn.createStatement()).thenReturn(st); + ResultSet rs = Mockito.mock(ResultSet.class); + Mockito.when(st.getResultSet()).thenReturn(rs); + ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class); + Mockito.when(rs.getMetaData()).thenReturn(md); + Mockito.when(md.getColumnCount()).thenReturn(1); + Mockito.when(rs.next()).thenReturn(true, false); + ScriptRunner s2 = new ScriptRunner(conn, true, true); + Mockito.verify(md).getColumnLabel(0); + } + @Test public void testRunScriptWithDbName() { //connect is not null try { - Connection conn = Mockito.mock(Connection.class); - Mockito.when(conn.getAutoCommit()).thenReturn(true); - PreparedStatement st = Mockito.mock(PreparedStatement.class); - Mockito.when(conn.createStatement()).thenReturn(st); - ResultSet rs = Mockito.mock(ResultSet.class); - Mockito.when(st.getResultSet()).thenReturn(rs); - ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class); - Mockito.when(rs.getMetaData()).thenReturn(md); - Mockito.when(md.getColumnCount()).thenReturn(1); - Mockito.when(rs.next()).thenReturn(true, false); - ScriptRunner s = new ScriptRunner(conn, true, true); + ScriptRunner s = getScriptRunner(); s.runScript(new StringReader("select 1;"), "test_db"); - Mockito.verify(md).getColumnLabel(0); } catch(Exception e) { Assert.assertNotNull(e); } From 2f33a9e2ab33598850766e749e2dabf18bbc8e64 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Sun, 9 Feb 2020 10:20:18 +0800 Subject: [PATCH 22/43] delete dup code --- .../common/utils/ScriptRunnerTest.java | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java index a36253f8ef..0eb1cce950 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java @@ -35,37 +35,36 @@ public class ScriptRunnerTest { Assert.assertNotNull(exception); //connect is not null + runScript(""); + } + + private void runScript(String dbName) { try { - ScriptRunner s2 = getScriptRunner(); - s2.runScript(new StringReader("select 1;")); + Connection conn = Mockito.mock(Connection.class); + Mockito.when(conn.getAutoCommit()).thenReturn(true); + PreparedStatement st = Mockito.mock(PreparedStatement.class); + Mockito.when(conn.createStatement()).thenReturn(st); + ResultSet rs = Mockito.mock(ResultSet.class); + Mockito.when(st.getResultSet()).thenReturn(rs); + ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class); + Mockito.when(rs.getMetaData()).thenReturn(md); + Mockito.when(md.getColumnCount()).thenReturn(1); + Mockito.when(rs.next()).thenReturn(true, false); + ScriptRunner s = new ScriptRunner(conn, true, true); + if (dbName.isEmpty()) { + s.runScript(new StringReader("select 1;")); + } else { + s.runScript(new StringReader("select 1;"), dbName); + } + Mockito.verify(md).getColumnLabel(0); } catch(Exception e) { Assert.assertNotNull(e); } } - private ScriptRunner getScriptRunner() throws SQLException { - Connection conn = Mockito.mock(Connection.class); - Mockito.when(conn.getAutoCommit()).thenReturn(true); - PreparedStatement st = Mockito.mock(PreparedStatement.class); - Mockito.when(conn.createStatement()).thenReturn(st); - ResultSet rs = Mockito.mock(ResultSet.class); - Mockito.when(st.getResultSet()).thenReturn(rs); - ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class); - Mockito.when(rs.getMetaData()).thenReturn(md); - Mockito.when(md.getColumnCount()).thenReturn(1); - Mockito.when(rs.next()).thenReturn(true, false); - ScriptRunner s2 = new ScriptRunner(conn, true, true); - Mockito.verify(md).getColumnLabel(0); - } - @Test public void testRunScriptWithDbName() { //connect is not null - try { - ScriptRunner s = getScriptRunner(); - s.runScript(new StringReader("select 1;"), "test_db"); - } catch(Exception e) { - Assert.assertNotNull(e); - } + runScript("db_test"); } } From e00ef4915d50db3d594e89da8bc0efa9b4e25ba7 Mon Sep 17 00:00:00 2001 From: lilin Date: Sun, 9 Feb 2020 16:32:10 +0800 Subject: [PATCH 23/43] modify loggerService UT by Mock --- .../api/service/LoggerService.java | 5 ++ .../api/service/LoggerServiceTest.java | 88 +++++++++++++++---- pom.xml | 1 + 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java index 61dc1a7193..2587290fd3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java @@ -84,7 +84,12 @@ public class LoggerService { if (taskInstance == null){ throw new RuntimeException("task instance is null"); } + String host = taskInstance.getHost(); + if(StringUtils.isEmpty(host)){ + throw new RuntimeException("task instance host is null"); + } + LogClient logClient = new LogClient(host, Constants.RPC_PORT); return logClient.getLogBytes(taskInstance.getLogPath()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 798e064d79..cec7102ce1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -16,37 +16,95 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.log.LogClient; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.ProcessDao; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest({LoggerService.class}) public class LoggerServiceTest { + private static final Logger logger = LoggerFactory.getLogger(LoggerServiceTest.class); - @Autowired + @InjectMocks private LoggerService loggerService; + @Mock + private ProcessDao processDao; + @Mock + private LogClient logClient; - @Test - public void queryDataSourceList(){ + @Before + public void setUp() { + + try { + PowerMockito.whenNew(LogClient.class).withAnyArguments().thenReturn(logClient); + } catch (Exception e) { + logger.error("setUp error: {}",e.getMessage()); + } + } - User loginUser = new User(); - loginUser.setId(27); - loginUser.setUserType(UserType.GENERAL_USER); + @Test + public void testQueryDataSourceList(){ - Result result = loggerService.queryLog(-1, 0, 100); + TaskInstance taskInstance = new TaskInstance(); + Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); + Result result = loggerService.queryLog(2,1,1); + //TASK_INSTANCE_NOT_FOUND + Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(),result.getCode().intValue()); + //HOST NOT FOUND + result = loggerService.queryLog(1,1,1); Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(),result.getCode().intValue()); + + //SUCCESS + taskInstance.setHost("127.0.0.1"); + taskInstance.setLogPath("/temp/log"); + Mockito.when(logClient.rollViewLog("/temp/log",1,1 )).thenReturn("test"); + Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); + result = loggerService.queryLog(1,1,1); + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); } + + @Test + public void testGetLogBytes(){ + + TaskInstance taskInstance = new TaskInstance(); + Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); + + //task instance is null + try{ + loggerService.getLogBytes(2); + }catch (Exception e){ + logger.error("testGetLogBytes error: {}","task instance is null"); + } + + //task instance host is null + try{ + loggerService.getLogBytes(1); + }catch (Exception e){ + logger.error("testGetLogBytes error: {}","task instance host is null"); + } + + //success + Mockito.when(logClient.getLogBytes("/temp/log")).thenReturn(new byte[]{}); + taskInstance.setHost("127.0.0.1"); + taskInstance.setLogPath("/temp/log"); + byte [] result = loggerService.getLogBytes(1); + Assert.assertEquals(0,result.length); + } + } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 875577c672..1b88c77418 100644 --- a/pom.xml +++ b/pom.xml @@ -699,6 +699,7 @@ **/api/service/ExecutorService2Test.java **/api/service/BaseServiceTest.java **/api/service/BaseDAGServiceTest.java + **/api/service/LoggerServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From b77dddfeefa87be2027c5ddac29350f946b9e629 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sun, 9 Feb 2020 17:56:55 +0800 Subject: [PATCH 24/43] Adapting partial code(file name start with B) to the sonar cloud rules (#1915) * Adapting partial code(file name start with B) to the sonar cloud rules * add @Ignore to BaseTaskQueueTest class --- .../dolphinscheduler/api/controller/BaseController.java | 9 +++------ .../common/utils/placeholder/BusinessTimeUtils.java | 4 +++- .../dolphinscheduler/common/queue/BaseTaskQueueTest.java | 8 ++++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java index 46432569e1..c434398679 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java @@ -157,8 +157,7 @@ public class BaseController { * @return success result code */ public Result success(String msg, Object list) { - Result result = getResult(msg, list); - return result; + return getResult(msg, list); } /** @@ -168,8 +167,7 @@ public class BaseController { * @return success result code */ public Result success(Object list) { - Result result = getResult(Status.SUCCESS.getMsg(), list); - return result; + return getResult(Status.SUCCESS.getMsg(), list); } /** @@ -181,8 +179,7 @@ public class BaseController { * @return success result code */ public Result success(String msg, Map object) { - Result result = getResult(msg, object); - return result; + return getResult(msg, object); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java index d6b4e34ce3..23db4b626b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java @@ -33,7 +33,9 @@ import static org.apache.commons.lang.time.DateUtils.addDays; * business time utils */ public class BusinessTimeUtils { - + private BusinessTimeUtils() { + throw new IllegalStateException("BusinessTimeUtils class"); + } /** * get business time in parameters by different command types * diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/BaseTaskQueueTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/BaseTaskQueueTest.java index 0bd4266bb7..433e4fa30f 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/BaseTaskQueueTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/BaseTaskQueueTest.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.common.queue; import org.apache.dolphinscheduler.common.zk.ZKServer; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.*; /** * base task queue test for only start zk server once */ +@Ignore public class BaseTaskQueueTest { protected static ITaskQueue tasksQueue = null; @@ -40,4 +40,8 @@ public class BaseTaskQueueTest { tasksQueue.delete(); ZKServer.stop(); } + @Test + public void tasksQueueNotNull(){ + Assert.assertNotNull(tasksQueue); + } } From b3976da5b022886b9d7089b856118e75d7884707 Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 10 Feb 2020 11:35:16 +0800 Subject: [PATCH 25/43] modify LoggerServiceTest ut --- .../dolphinscheduler/api/service/LoggerServiceTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index cec7102ce1..193ecb4129 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -88,14 +88,16 @@ public class LoggerServiceTest { //task instance is null try{ loggerService.getLogBytes(2); - }catch (Exception e){ + }catch (RuntimeException e){ + Assert.assertTrue(true); logger.error("testGetLogBytes error: {}","task instance is null"); } //task instance host is null try{ loggerService.getLogBytes(1); - }catch (Exception e){ + }catch (RuntimeException e){ + Assert.assertTrue(true); logger.error("testGetLogBytes error: {}","task instance host is null"); } From c447bb489f5da45ca234a236551474778b24f2e3 Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Mon, 10 Feb 2020 13:52:32 +0800 Subject: [PATCH 26/43] add stringutils ut (#1921) * add stringutils ut --- .../alert/utils/MailUtils.java | 8 +- .../ProcessDefinitionController.java | 16 +-- .../controller/ProcessInstanceController.java | 10 +- .../api/service/ResourcesService.java | 2 +- .../common/utils/StringUtils.java | 125 +----------------- .../common/utils/StringUtilsTest.java | 66 +++++++++ 6 files changed, 89 insertions(+), 138 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 7ebe6a7863..99efdc8a6a 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -90,14 +90,14 @@ public class MailUtils { public static Map sendMails(Collection receivers, Collection receiversCc, String title, String content, ShowType showType) { Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); - + // if there is no receivers && no receiversCc, no need to process if (CollectionUtils.isEmpty(receivers) && CollectionUtils.isEmpty(receiversCc)) { return retMap; } receivers.removeIf(StringUtils::isEmpty); - + if (showType == ShowType.TABLE || showType == ShowType.TEXT){ // send email HtmlEmail email = new HtmlEmail(); @@ -335,7 +335,7 @@ public class MailUtils { */ private static void handleException(Collection receivers, Map retMap, Exception e) { logger.error("Send email to {} failed {}", receivers, e); - retMap.put(Constants.MESSAGE, "Send email to {" + StringUtils.join(receivers, ",") + "} failed，" + e.toString()); + retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed，" + e.toString()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java index de9cc12a36..c07ecf9ca7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java @@ -53,7 +53,7 @@ public class ProcessDefinitionController extends BaseController{ /** * create process definition - * + * * @param loginUser login user * @param projectName project name * @param name process definition name @@ -96,7 +96,7 @@ public class ProcessDefinitionController extends BaseController{ /** * verify process definition name unique - * + * * @param loginUser login user * @param projectName project name * @param name name @@ -328,9 +328,9 @@ public class ProcessDefinitionController extends BaseController{ /** - * + * * get tasks list by process definition id - * + * * * @param loginUser login user * @param projectName project name @@ -442,7 +442,7 @@ public class ProcessDefinitionController extends BaseController{ loginUser.getUserName(), projectName, processDefinitionIds); Map result = new HashMap<>(5); - List deleteFailedIdList = new ArrayList(); + List deleteFailedIdList = new ArrayList<>(); if(StringUtils.isNotEmpty(processDefinitionIds)){ String[] processDefinitionIdArray = processDefinitionIds.split(","); @@ -451,17 +451,17 @@ public class ProcessDefinitionController extends BaseController{ try { Map deleteResult = processDefinitionService.deleteProcessDefinitionById(loginUser, projectName, processDefinitionId); if(!Status.SUCCESS.equals(deleteResult.get(Constants.STATUS))){ - deleteFailedIdList.add(processDefinitionId); + deleteFailedIdList.add(strProcessDefinitionId); logger.error((String)deleteResult.get(Constants.MSG)); } } catch (Exception e) { - deleteFailedIdList.add(processDefinitionId); + deleteFailedIdList.add(strProcessDefinitionId); } } } if(!deleteFailedIdList.isEmpty()){ - putMsg(result, Status.BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR,StringUtils.join(deleteFailedIdList,",")); + putMsg(result, Status.BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR, String.join(",", deleteFailedIdList)); }else{ putMsg(result, Status.SUCCESS); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index 542aad5c33..150c647f99 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -58,7 +58,7 @@ public class ProcessInstanceController extends BaseController{ /** * query process instance list paging - * + * * @param loginUser login user * @param projectName project name * @param pageNo page number @@ -372,7 +372,7 @@ public class ProcessInstanceController extends BaseController{ // task queue ITaskQueue tasksQueue = TaskQueueFactory.getTaskQueueInstance(); Map result = new HashMap<>(5); - List deleteFailedIdList = new ArrayList(); + List deleteFailedIdList = new ArrayList<>(); if(StringUtils.isNotEmpty(processInstanceIds)){ String[] processInstanceIdArray = processInstanceIds.split(","); @@ -381,16 +381,16 @@ public class ProcessInstanceController extends BaseController{ try { Map deleteResult = processInstanceService.deleteProcessInstanceById(loginUser, projectName, processInstanceId,tasksQueue); if(!Status.SUCCESS.equals(deleteResult.get(Constants.STATUS))){ - deleteFailedIdList.add(processInstanceId); + deleteFailedIdList.add(strProcessInstanceId); logger.error((String)deleteResult.get(Constants.MSG)); } } catch (Exception e) { - deleteFailedIdList.add(processInstanceId); + deleteFailedIdList.add(strProcessInstanceId); } } } if(deleteFailedIdList.size() > 0){ - putMsg(result, Status.BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR,StringUtils.join(deleteFailedIdList,",")); + putMsg(result, Status.BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR, String.join(",", deleteFailedIdList)); }else{ putMsg(result, Status.SUCCESS); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 09b1d31151..29a16447e1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -539,7 +539,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.SUCCESS); Map map = new HashMap<>(); map.put(ALIAS, resource.getAlias()); - map.put(CONTENT, StringUtils.join(content, "\n")); + map.put(CONTENT, String.join("\n", content)); result.setData(map); }else{ logger.error("read file {} not exist in hdfs", hdfsFileName); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java index 12b75fb0e5..af2817a8d7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java @@ -16,17 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; - -import java.nio.charset.StandardCharsets; -import java.util.Iterator; -import java.util.Objects; -import java.util.regex.Pattern; - - public class StringUtils { - - public static final int INDEX_NOT_FOUND = -1; - public static final String EMPTY = ""; public static boolean isEmpty(final CharSequence cs) { @@ -37,119 +27,14 @@ public class StringUtils { return !isEmpty(cs); } - public static boolean isBlank(CharSequence cs){ - int strLen; - if (cs == null || (strLen = cs.length()) == 0) { + public static boolean isBlank(String s){ + if (isEmpty(s)) { return true; } - for (int i = 0; i < strLen; i++) { - if (Character.isWhitespace(cs.charAt(i)) == false) { - return false; - } - } - return true; - } - - public static boolean isNotBlank(CharSequence str){ - return !isBlank(str); - } - - public static String substringBefore(final String str, final String separator) { - if (isBlank(str) || separator == null) { - return str; - } - if (separator.isEmpty()) { - return EMPTY; - } - final int pos = str.indexOf(separator); - if (pos == INDEX_NOT_FOUND) { - return str; - } - return str.substring(0, pos); - } - - public static String substringAfter(final String str, final String separator) { - if (isBlank(str)) { - return str; - } - if (separator == null) { - return EMPTY; - } - final int pos = str.indexOf(separator); - if (pos == INDEX_NOT_FOUND) { - return EMPTY; - } - return str.substring(pos + separator.length()); + return s.trim().length() == 0; } - public static String substringAfterLast(final String str, final String separator) { - if (isEmpty(str)) { - return str; - } - if (isEmpty(separator)) { - return EMPTY; - } - final int pos = str.lastIndexOf(separator); - if (pos == INDEX_NOT_FOUND || pos == str.length() - separator.length()) { - return EMPTY; - } - return str.substring(pos + separator.length()); - } - - public static String getUtf8String(byte[] bytes){ - return new String(bytes, StandardCharsets.UTF_8); - } - - public static byte[] getUtf8Bytes(String str){ - return str.getBytes(StandardCharsets.UTF_8); - } - - public static boolean hasChinese(String str) { - if (str == null) { - return false; - } - Pattern pattern = Pattern.compile("[\\u4E00-\\u9FBF]+"); - return pattern.matcher(str).find(); - } - - public static boolean hasSpace(String str) { - if (str == null) { - return false; - } - int len = str.length(); - for (int i = 0; i < len; i++) { - if (str.charAt(i) == ' ') { - return true; - } - } - return false; - } - - public static String join(final Iterable iterable, final String separator){ - Iterator iterator = iterable.iterator(); - if (iterator == null) { - return null; - } - if (!iterator.hasNext()) { - return EMPTY; - } - final Object first = iterator.next(); - if (!iterable.iterator().hasNext()) { - return Objects.toString(first, ""); - } - final StringBuilder buf = new StringBuilder(64); - if (first != null) { - buf.append(first); - } - while (iterator.hasNext()) { - if (separator != null) { - buf.append(separator); - } - final Object obj = iterator.next(); - if (obj != null) { - buf.append(obj); - } - } - return buf.toString(); + public static boolean isNotBlank(String s){ + return !isBlank(s); } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java new file mode 100644 index 0000000000..947e7310db --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; + +public class StringUtilsTest { + @Test + public void testIsNotEmpty() { + //null string + boolean b = StringUtils.isNotEmpty(null); + Assert.assertFalse(b); + + //"" string + b = StringUtils.isNotEmpty(""); + Assert.assertFalse(b); + + //" " string + b = StringUtils.isNotEmpty(" "); + Assert.assertTrue(b); + + //"test" string + b = StringUtils.isNotEmpty("test"); + Assert.assertTrue(b); + } + + @Test + public void testIsNotBlank() { + //null string + boolean b = StringUtils.isNotBlank(null); + Assert.assertFalse(b); + + //"" string + b = StringUtils.isNotBlank(""); + Assert.assertFalse(b); + + //" " string + b = StringUtils.isNotBlank(" "); + Assert.assertFalse(b); + + //" test " string + b = StringUtils.isNotBlank(" test "); + Assert.assertTrue(b); + + //"test" string + b = StringUtils.isNotBlank("test"); + Assert.assertTrue(b); + } +} From bcc734713bbb397e117722b6c62a4a62994923f7 Mon Sep 17 00:00:00 2001 From: zhukai Date: Mon, 10 Feb 2020 14:15:20 +0800 Subject: [PATCH 27/43] Newfeature for #1675. (#1908) Continue to finish the rest works, add the cache feature for dependence,mr,python,sub_process,procedure and shell. --- .../pages/dag/_source/formModel/formModel.vue | 20 +++++++++-- .../dag/_source/formModel/tasks/dependent.vue | 17 +++++++++- .../dag/_source/formModel/tasks/flink.vue | 2 +- .../pages/dag/_source/formModel/tasks/mr.vue | 29 ++++++++++++++++ .../dag/_source/formModel/tasks/procedure.vue | 25 +++++++++++--- .../dag/_source/formModel/tasks/python.vue | 32 ++++++++++++++--- .../dag/_source/formModel/tasks/shell.vue | 34 ++++++++++++++++--- .../dag/_source/formModel/tasks/spark.vue | 2 +- .../_source/formModel/tasks/sub_process.vue | 10 ++++-- 9 files changed, 151 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 682dd5b51a..e05de8e880 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -121,6 +121,7 @@ @@ -128,6 +129,7 @@ @@ -136,6 +138,7 @@ @@ -167,6 +170,7 @@ @@ -174,6 +178,7 @@ @@ -181,6 +186,7 @@ @@ -248,6 +254,8 @@ resourcesList: [], // dependence dependence: {}, + // cache dependence + cacheDependence: {}, // Current node params data params: {}, // Running sign @@ -283,6 +291,12 @@ _onDependent (o) { this.dependence = Object.assign(this.dependence, {}, o) }, + /** + * cache dependent + */ + _onCacheDependent (o) { + this.cacheDependence = Object.assign(this.cacheDependence, {}, o) + }, /** * Task timeout alarm */ @@ -356,9 +370,10 @@ type: this.taskType, id: this.id, name: this.name, + params: this.params, description: this.description, runFlag: this.runFlag, - dependence: this.dependence, + dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, timeout: this.timeout, @@ -522,6 +537,7 @@ this.params = o.params || {} this.dependence = o.dependence || {} + this.cacheDependence = o.dependence || {} } this.isContentBox = true @@ -551,7 +567,7 @@ name: this.name, description: this.description, runFlag: this.runFlag, - dependence: this.dependence, + dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, timeout: this.timeout, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/dependent.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/dependent.vue index cca9ec7003..79d127a108 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/dependent.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/dependent.vue @@ -131,6 +131,9 @@ setTimeout(() => { this.isLoading = false }, 600) + }, + cacheDependence (val) { + this.$emit('on-cache-dependent', val) } }, beforeCreate () { @@ -151,7 +154,19 @@ }, destroyed () { }, - computed: {}, + computed: { + cacheDependence () { + return { + relation: this.relation, + dependTaskList: _.map(this.dependTaskList, v => { + return { + relation: v.relation, + dependItemList: _.map(v.dependItemList, v1 => _.omit(v1, ['depTasksList', 'state', 'dateValueList'])) + } + }) + } + } + }, components: { mListBox, mDependItemList } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index f0d69ec420..03e53fe5e5 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -379,7 +379,7 @@ // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - this.mainJar = o.params.mainJar.res || '' + this.mainJar = o.params.mainJar && o.params.mainJar.res ? o.params.mainJar.res : '' this.deployMode = o.params.deployMode || '' this.slot = o.params.slot || 1 this.taskManager = o.params.taskManager || '2' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index fa73e9bb89..706a35f4fe 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -91,6 +91,7 @@ @@ -127,6 +128,8 @@ mainJarList: [], // Resource(list) resourceList: [], + // Cache ResourceList + cacheResourceList: [], // Custom parameter localParams: [], // Command line argument @@ -156,6 +159,12 @@ _onResourcesData (a) { this.resourceList = a }, + /** + * cache resourceList + */ + _onCacheResourcesData (a) { + this.cacheResourceList = a + }, /** * verification */ @@ -220,6 +229,25 @@ if (type === 'PYTHON') { this.mainClass = '' } + }, + //Watch the cacheParams + cacheParams (val) { + this.$emit('on-cache-params', val); + } + }, + computed: { + cacheParams () { + return { + mainClass: this.mainClass, + mainJar: { + res: this.mainJar + }, + resourceList: this.cacheResourceList, + localParams: this.localParams, + mainArgs: this.mainArgs, + others: this.others, + programType: this.programType + } } }, created () { @@ -238,6 +266,7 @@ let resourceList = o.params.resourceList || [] if (resourceList.length) { this.resourceList = resourceList + this.cacheResourceList = resourceList } // backfill localParams diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue index d55f18a2cc..e84f37d7a9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue @@ -70,7 +70,9 @@ // Data source type type: '', // data source - datasource: '' + datasource: '', + // Return to the selected data source + rtDatasource: '' } }, mixins: [disabledState], @@ -83,7 +85,7 @@ */ _onDsData (o) { this.type = o.type - this.datasource = o.datasource + this.rtDatasource = o.datasource }, /** * return udp @@ -112,14 +114,29 @@ // storage this.$emit('on-params', { type: this.type, - datasource: this.datasource, + datasource: this.rtDatasource, method: this.method, localParams: this.localParams }) return true } }, - watch: {}, + watch: { + //Watch the cacheParams + cacheParams (val) { + this.$emit('on-cache-params', val); + } + }, + computed: { + cacheParams () { + return { + type: this.type, + datasource: this.rtDatasource, + method: this.method, + localParams: this.localParams + } + } + }, created () { let o = this.backfillItem diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 39a7cd858b..e565b4a6bd 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -31,6 +31,7 @@ @@ -69,7 +70,9 @@ // Custom parameter localParams: [], // resource(list) - resourceList: [] + resourceList: [], + // Cache ResourceList + cacheResourceList: [] } }, mixins: [disabledState], @@ -89,6 +92,12 @@ _onResourcesData (a) { this.resourceList = a }, + /** + * cache resourceList + */ + _onCacheResourcesData (a) { + this.cacheResourceList = a + }, /** * verification */ @@ -142,18 +151,33 @@ return editor } }, - watch: {}, + watch: { + //Watch the cacheParams + cacheParams (val) { + this.$emit('on-cache-params', val); + } + }, + computed: { + cacheParams () { + return { + resourceList: this.cacheResourceList, + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' + } + } + }, created () { let o = this.backfillItem // Non-null objects represent backfill if (!_.isEmpty(o)) { - this.rawScript = o.params.rawScript + this.rawScript = o.params.rawScript || '' // backfill resourceList let resourceList = o.params.resourceList || [] if (resourceList.length) { this.resourceList = resourceList + this.cacheResourceList = resourceList } // backfill localParams @@ -174,4 +198,4 @@ }, components: { mLocalParams, mListBox, mResources } } - \ No newline at end of file + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 17184eed75..ad40c586b9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -37,6 +37,7 @@ @@ -75,7 +76,9 @@ // Custom parameter localParams: [], // resource(list) - resourceList: [] + resourceList: [], + // Cache ResourceList + cacheResourceList: [] } }, mixins: [disabledState], @@ -119,11 +122,17 @@ }, /** * return resourceList - * + * */ _onResourcesData (a) { this.resourceList = a }, + /** + * cache resourceList + */ + _onCacheResourcesData (a) { + this.cacheResourceList = a + }, /** * verification */ @@ -175,18 +184,33 @@ return editor } }, - watch: {}, + watch: { + //Watch the cacheParams + cacheParams (val) { + this.$emit('on-cache-params', val); + } + }, + computed: { + cacheParams () { + return { + resourceList: this.cacheResourceList, + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' + } + } + }, created () { let o = this.backfillItem // Non-null objects represent backfill if (!_.isEmpty(o)) { - this.rawScript = o.params.rawScript + this.rawScript = o.params.rawScript || '' // backfill resourceList let resourceList = o.params.resourceList || [] if (resourceList.length) { this.resourceList = resourceList + this.cacheResourceList = resourceList } // backfill localParams @@ -229,5 +253,5 @@ right: -12px; top: -16px; } - + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue index 66a89a4944..feef19856c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue @@ -412,7 +412,7 @@ // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - this.mainJar = o.params.mainJar.res || '' + this.mainJar = o.params.mainJar && o.params.mainJar.res ? o.params.mainJar.res : '' this.deployMode = o.params.deployMode || '' this.driverCores = o.params.driverCores || 1 this.driverMemory = o.params.driverMemory || '512M' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sub_process.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sub_process.vue index ee03513249..477038f18f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sub_process.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sub_process.vue @@ -86,7 +86,13 @@ return _.filter(this.processDefinitionList, v => id === v.id)[0].code } }, - watch: {}, + watch: { + wdiCurr (val) { + this.$emit('on-cache-params', { + processDefinitionId: this.wdiCurr + }) + } + }, created () { let processListS = _.cloneDeep(this.store.state.dag.processListS) let id = this.router.history.current.params.id || null @@ -115,4 +121,4 @@ mounted () { } } - \ No newline at end of file + From 69fa28357b025655ab5b0d6d2d5536e2e6add558 Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 10 Feb 2020 15:33:37 +0800 Subject: [PATCH 28/43] modify @RunWith(MockitoJUnitRunner.class) --- .../api/service/LoggerServiceTest.java | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 193ecb4129..31034e5993 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -17,24 +17,21 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.log.LogClient; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; +import org.mockito.junit.MockitoJUnitRunner; import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@RunWith(PowerMockRunner.class) +@RunWith(MockitoJUnitRunner.class) @PrepareForTest({LoggerService.class}) public class LoggerServiceTest { @@ -44,18 +41,7 @@ public class LoggerServiceTest { private LoggerService loggerService; @Mock private ProcessDao processDao; - @Mock - private LogClient logClient; - @Before - public void setUp() { - - try { - PowerMockito.whenNew(LogClient.class).withAnyArguments().thenReturn(logClient); - } catch (Exception e) { - logger.error("setUp error: {}",e.getMessage()); - } - } @Test public void testQueryDataSourceList(){ @@ -73,7 +59,6 @@ public class LoggerServiceTest { //SUCCESS taskInstance.setHost("127.0.0.1"); taskInstance.setLogPath("/temp/log"); - Mockito.when(logClient.rollViewLog("/temp/log",1,1 )).thenReturn("test"); Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); result = loggerService.queryLog(1,1,1); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); @@ -102,11 +87,10 @@ public class LoggerServiceTest { } //success - Mockito.when(logClient.getLogBytes("/temp/log")).thenReturn(new byte[]{}); taskInstance.setHost("127.0.0.1"); taskInstance.setLogPath("/temp/log"); - byte [] result = loggerService.getLogBytes(1); - Assert.assertEquals(0,result.length); + loggerService.getLogBytes(1); + } } \ No newline at end of file From d937a6ae9421b4df20a524de0ca8e93524e96d12 Mon Sep 17 00:00:00 2001 From: Yelli Date: Mon, 10 Feb 2020 16:37:51 +0800 Subject: [PATCH 29/43] Add modify user name for process definition (#1919) * class overrides equals() and should therefore also override hashCode() * #1862 add modify user in process difinition list * #1862 add pg-1.2.2 ddl.sql * modify ScriptRunnerTest * add updateProessDifinition UT * modify updateProcessDifinition UT * modify updateProcessDifinition UT * modify mysql 1.2.2 ddl.sql&dml.sql * add scope test to mysql in pom * modify pg-1.2.2 ddl.sql --- .../api/service/ProcessDefinitionService.java | 2 + .../service/ProcessDefinitionServiceTest.java | 27 +++++++++++ .../common/utils/ScriptRunner.java | 4 +- .../common/utils/ScriptRunnerTest.java | 4 +- .../dao/entity/ProcessDefinition.java | 48 ++++++++++++------- .../definition/pages/list/_source/list.vue | 7 +++ .../src/js/module/i18n/locale/en_US.js | 1 + .../src/js/module/i18n/locale/zh_CN.js | 1 + pom.xml | 1 + sql/dolphinscheduler-postgre.sql | 1 + sql/dolphinscheduler_mysql.sql | 1 + .../mysql/dolphinscheduler_ddl.sql | 37 ++++++++++++++ .../mysql/dolphinscheduler_dml.sql | 16 +++++++ .../postgresql/dolphinscheduler_ddl.sql | 34 +++++++++++++ .../postgresql/dolphinscheduler_dml.sql | 16 +++++++ 15 files changed, 180 insertions(+), 20 deletions(-) create mode 100644 sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_dml.sql create mode 100644 sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_dml.sql diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 5fe708ce8c..8a762d7557 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -143,6 +143,7 @@ public class ProcessDefinitionService extends BaseDAGService { processDefine.setConnects(connects); processDefine.setTimeout(processData.getTimeout()); processDefine.setTenantId(processData.getTenantId()); + processDefine.setModifyBy(loginUser.getUserName()); //custom global params List globalParamsList = processData.getGlobalParams(); @@ -308,6 +309,7 @@ public class ProcessDefinitionService extends BaseDAGService { processDefine.setConnects(connects); processDefine.setTimeout(processData.getTimeout()); processDefine.setTenantId(processData.getTenantId()); + processDefine.setModifyBy(loginUser.getUserName()); //custom global params List globalParamsList = new ArrayList<>(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index b30a919598..a4b07e1835 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; import org.apache.http.entity.ContentType; @@ -77,6 +78,9 @@ public class ProcessDefinitionServiceTest { @Mock private WorkerGroupMapper workerGroupMapper; + @Mock + private ProcessDao processDao; + private String sqlDependentJson = "{\"globalParams\":[]," + "\"tasks\":[{\"type\":\"SQL\",\"id\":\"tasks-27297\",\"name\":\"sql\"," + "\"params\":{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from test\"," + @@ -422,6 +426,27 @@ public class ProcessDefinitionServiceTest { Assert.assertTrue(deleteFlag); } + @Test + public void testUpdateProcessDefinition () { + User loginUser = new User(); + loginUser.setId(1); + loginUser.setUserType(UserType.ADMIN_USER); + + Map result = new HashMap<>(5); + putMsg(result, Status.SUCCESS); + + String projectName = "project_test1"; + Project project = getProject(projectName); + + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + Mockito.when(processDao.findProcessDefineById(1)).thenReturn(getProcessDefinition()); + + Map updateResult = processDefinitionService.updateProcessDefinition(loginUser, projectName, 1, "test", + sqlDependentJson, "", "", ""); + + Assert.assertEquals(Status.UPDATE_PROCESS_DEFINITION_ERROR, updateResult.get(Constants.STATUS)); + } /** * get mock datasource @@ -443,6 +468,8 @@ public class ProcessDefinitionServiceTest { processDefinition.setId(46); processDefinition.setName("testProject"); processDefinition.setProjectId(2); + processDefinition.setTenantId(1); + processDefinition.setDescription(""); return processDefinition; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java index bbc937c89f..f92839bfe5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java @@ -169,13 +169,13 @@ public class ScriptRunner { if (stopOnError && rs != null) { ResultSetMetaData md = rs.getMetaData(); int cols = md.getColumnCount(); - for (int i = 0; i < cols; i++) { + for (int i = 1; i < cols; i++) { String name = md.getColumnLabel(i); logger.info("{} \t", name); } logger.info(""); while (rs.next()) { - for (int i = 0; i < cols; i++) { + for (int i = 1; i < cols; i++) { String value = rs.getString(i); logger.info("{} \t", value); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java index 0eb1cce950..155d52ab75 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java @@ -48,7 +48,7 @@ public class ScriptRunnerTest { Mockito.when(st.getResultSet()).thenReturn(rs); ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class); Mockito.when(rs.getMetaData()).thenReturn(md); - Mockito.when(md.getColumnCount()).thenReturn(1); + Mockito.when(md.getColumnCount()).thenReturn(2); Mockito.when(rs.next()).thenReturn(true, false); ScriptRunner s = new ScriptRunner(conn, true, true); if (dbName.isEmpty()) { @@ -56,7 +56,7 @@ public class ScriptRunnerTest { } else { s.runScript(new StringReader("select 1;"), dbName); } - Mockito.verify(md).getColumnLabel(0); + Mockito.verify(md).getColumnLabel(1); } catch(Exception e) { Assert.assertNotNull(e); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index cd0494ecc6..6e7ea7d64f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -29,6 +29,7 @@ import com.baomidou.mybatisplus.core.toolkit.StringUtils; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; @@ -158,6 +159,11 @@ public class ProcessDefinition { */ private int tenantId; + /** + * modify user name + */ + private String modifyBy; + public String getName() { return name; @@ -337,6 +343,30 @@ public class ProcessDefinition { this.timeout = timeout; } + public int getTenantId() { + return tenantId; + } + + public void setTenantId(int tenantId) { + this.tenantId = tenantId; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getModifyBy() { + return modifyBy; + } + + public void setModifyBy(String modifyBy) { + this.modifyBy = modifyBy; + } + @Override public String toString() { return "ProcessDefinition{" + @@ -346,6 +376,7 @@ public class ProcessDefinition { ", releaseState=" + releaseState + ", projectId=" + projectId + ", processDefinitionJson='" + processDefinitionJson + '\'' + + ", description='" + description + '\'' + ", globalParams='" + globalParams + '\'' + ", globalParamList=" + globalParamList + ", globalParamMap=" + globalParamMap + @@ -362,22 +393,7 @@ public class ProcessDefinition { ", scheduleReleaseState=" + scheduleReleaseState + ", timeout=" + timeout + ", tenantId=" + tenantId + + ", modifyBy='" + modifyBy + '\'' + '}'; } - - public int getTenantId() { - return tenantId; - } - - public void setTenantId(int tenantId) { - this.tenantId = tenantId; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue index 5550864176..53939f3f7b 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue @@ -40,6 +40,9 @@ {{$t('Description')}} + + {{$t('Modify User')}} + {{$t('Timing state')}} @@ -72,6 +75,10 @@ {{item.description}} - + + {{item.modifyBy}} + - + {{$t('offline')}} {{$t('online')}} diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 6e8c113b30..0402d7e398 100644 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -518,4 +518,5 @@ export default { 'SpeedRecord': 'speed(record count)', '0 means unlimited by byte': '0 means unlimited', '0 means unlimited by count': '0 means unlimited', + 'Modify User': 'Modify User' } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 52a98773a6..95eb4a1081 100644 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -518,4 +518,5 @@ export default { 'SpeedRecord': '限流(记录数)', '0 means unlimited by byte': 'KB，0代表不限制', '0 means unlimited by count': '0代表不限制', + 'Modify User': '修改用户' } diff --git a/pom.xml b/pom.xml index 875577c672..307db31eee 100644 --- a/pom.xml +++ b/pom.xml @@ -343,6 +343,7 @@ mysql mysql-connector-java ${mysql.connector.version} + test org.slf4j diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index b3c61ebce4..c68fd17be1 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -319,6 +319,7 @@ CREATE TABLE t_ds_process_definition ( timeout int DEFAULT '0' , tenant_id int NOT NULL DEFAULT '-1' , update_time timestamp DEFAULT NULL , + modify_by varchar(36) DEFAULT '' , PRIMARY KEY (id) ) ; diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index fec2771fff..ea0f9cb022 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -366,6 +366,7 @@ CREATE TABLE `t_ds_process_definition` ( `timeout` int(11) DEFAULT '0' COMMENT 'time out', `tenant_id` int(11) NOT NULL DEFAULT '-1' COMMENT 'tenant id', `update_time` datetime DEFAULT NULL COMMENT 'update time', + `modify_by` varchar(36) DEFAULT '' COMMENT 'modify user', PRIMARY KEY (`id`), KEY `process_definition_index` (`project_id`,`id`) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..9fe246a8c2 --- /dev/null +++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); +-- uc_dolphin_T_t_ds_process_definition_A_modify_by +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_process_definition_A_modify_by; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_process_definition_A_modify_by() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_process_definition' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='modify_by') + THEN + ALTER TABLE t_ds_process_definition ADD `modify_by` varchar(36) DEFAULT '' COMMENT 'modify user'; + END IF; + END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_process_definition_A_modify_by; +DROP PROCEDURE uc_dolphin_T_t_ds_process_definition_A_modify_by; diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..7fc12900e4 --- /dev/null +++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +-- uc_dolphin_T_t_ds_process_definition_A_modify_by +delimiter d// +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_process_definition_A_modify_by() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_process_definition' + AND COLUMN_NAME ='modify_by') + THEN + ALTER TABLE t_ds_process_definition ADD COLUMN modify_by varchar(36) DEFAULT ''; + END IF; +END; +$$ LANGUAGE plpgsql; +d// + +delimiter ; +SELECT uc_dolphin_T_t_ds_process_definition_A_modify_by(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_definition_A_modify_by(); + diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_dml.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file From 17162341c3af7277dbac584beff6abf733a796d9 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Tue, 11 Feb 2020 00:18:06 +0800 Subject: [PATCH 30/43] add ResInfoTest.java ut --- .../common/utils/ResInfo.java | 12 ++++----- .../common/utils/ResInfoTest.java} | 27 +++++++++++++++---- 2 files changed, 28 insertions(+), 11 deletions(-) rename dolphinscheduler-common/src/{main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java => test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java} (54%) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java index e0cea1d7e0..aa8d44fa42 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java @@ -15,7 +15,6 @@ * limitations under the License. */ package org.apache.dolphinscheduler.common.utils; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.model.Server; @@ -135,14 +134,15 @@ public class ResInfo { * @return heartbeat info to Server */ public static Server parseHeartbeatForZKInfo(String heartBeatInfo){ - Server masterServer = null; + if (StringUtils.isEmpty(heartBeatInfo)) { + return null; + } String[] masterArray = heartBeatInfo.split(Constants.COMMA); - if(masterArray == null || - masterArray.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ - return masterServer; + if(masterArray.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ + return null; } - masterServer = new Server(); + Server masterServer = new Server(); masterServer.setHost(masterArray[0]); masterServer.setPort(Integer.parseInt(masterArray[1])); masterServer.setResInfo(getResInfoJson(Double.parseDouble(masterArray[2]), diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java similarity index 54% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java rename to dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java index 163c56bbcc..1e1b154d93 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java @@ -14,13 +14,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.dolphinscheduler.common.utils; +import org.junit.Assert; +import org.junit.Test; +import java.util.Date; +import org.apache.dolphinscheduler.common.model.Server; + +public class ResInfoTest { + @Test + public void testGetHeartBeatInfo() { + String info = ResInfo.getHeartBeatInfo(new Date()); + Assert.assertEquals(info.split(",").length, 7); + } -public class ArrayUtils { + @Test + public void testParseHeartbeatForZKInfo() { + //normal info + String info = ResInfo.getHeartBeatInfo(new Date()); + Server s = ResInfo.parseHeartbeatForZKInfo(info); + Assert.assertNotNull(s); + Assert.assertNotNull(s.getResInfo()); - public static boolean isEmpty(final int[] array) { - return array == null || array.length == 0; - } + //null param + s = ResInfo.parseHeartbeatForZKInfo(null); + Assert.assertNull(s); + } } From 1172abee44b2b894b10b88233919959d383acb13 Mon Sep 17 00:00:00 2001 From: lilin Date: Tue, 11 Feb 2020 09:00:56 +0800 Subject: [PATCH 31/43] add comment --- .../apache/dolphinscheduler/api/service/LoggerServiceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 31034e5993..c6ab6f8e74 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -89,6 +89,8 @@ public class LoggerServiceTest { //success taskInstance.setHost("127.0.0.1"); taskInstance.setLogPath("/temp/log"); + //if use @RunWith(PowerMockRunner.class) mock object,sonarcloud will not calculate the coverage, + // so no assert will be added here loggerService.getLogBytes(1); } From 980b5ae5ea4ee9e5751f15066401aec4118ee14f Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Tue, 11 Feb 2020 09:21:59 +0800 Subject: [PATCH 32/43] swap param --- .../org/apache/dolphinscheduler/common/utils/ResInfoTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java index 1e1b154d93..0d15d9615a 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java @@ -25,7 +25,7 @@ public class ResInfoTest { @Test public void testGetHeartBeatInfo() { String info = ResInfo.getHeartBeatInfo(new Date()); - Assert.assertEquals(info.split(",").length, 7); + Assert.assertEquals(7, info.split(",").length, ); } @Test From 0ed6156b221ef96d53917bb1ae1dece41f7aaa4c Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Tue, 11 Feb 2020 09:26:43 +0800 Subject: [PATCH 33/43] swap param --- .../org/apache/dolphinscheduler/common/utils/ResInfoTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java index 0d15d9615a..e4318965b7 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ResInfoTest.java @@ -25,7 +25,7 @@ public class ResInfoTest { @Test public void testGetHeartBeatInfo() { String info = ResInfo.getHeartBeatInfo(new Date()); - Assert.assertEquals(7, info.split(",").length, ); + Assert.assertEquals(7, info.split(",").length); } @Test From 0594a144831da4fc6f6cd5516b04bb18bcea92f1 Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Tue, 11 Feb 2020 23:32:21 +0800 Subject: [PATCH 34/43] fix powermock coverage 0 (#1935) * fix powermock coverage 0 --- .github/workflows/ci_ut.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 11660298b9..8013d40f31 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -47,16 +47,17 @@ jobs: - name: Compile run: | export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' - mvn test -B -Dmaven.test.skip=false cobertura:cobertura + mvn test -B -Dmaven.test.skip=false CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) - name: Run SonarCloud Analysis run: > mvn verify --batch-mode org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.1.1688:sonar - -Dsonar.junit.reportPaths=target/cobertura + -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco/jacoco.xml -Dmaven.test.skip=true -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache + -Dsonar.core.codeCoveragePlugin=jacoco -Dsonar.projectKey=apache-dolphinscheduler -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 env: From 735034c72f36905bfa3722a75baeab6e8e511d8c Mon Sep 17 00:00:00 2001 From: lilin Date: Wed, 12 Feb 2020 10:44:44 +0800 Subject: [PATCH 35/43] modify DataAnalysisServiceTest by mock --- .../api/service/DataAnalysisService.java | 12 +- .../api/service/DataAnalysisServiceTest.java | 207 ++++++++++++++++-- pom.xml | 1 + 3 files changed, 197 insertions(+), 23 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java index f4becbe36d..b95782711f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java @@ -108,8 +108,8 @@ public class DataAnalysisService extends BaseService{ List taskInstanceStateCounts = taskInstanceMapper.countTaskInstanceStateByUser(start, end, projectIds); - TaskCountDto taskCountResult = new TaskCountDto(taskInstanceStateCounts); - if (taskInstanceStateCounts != null) { + if (taskInstanceStateCounts != null && !taskInstanceStateCounts.isEmpty()) { + TaskCountDto taskCountResult = new TaskCountDto(taskInstanceStateCounts); result.put(Constants.DATA_LIST, taskCountResult); putMsg(result, Status.SUCCESS); } else { @@ -155,8 +155,8 @@ public class DataAnalysisService extends BaseService{ processInstanceMapper.countInstanceStateByUser(start, end, projectIdArray); - TaskCountDto taskCountResult = new TaskCountDto(processInstanceStateCounts); - if (processInstanceStateCounts != null) { + if (processInstanceStateCounts != null && !processInstanceStateCounts.isEmpty()) { + TaskCountDto taskCountResult = new TaskCountDto(processInstanceStateCounts); result.put(Constants.DATA_LIST, taskCountResult); putMsg(result, Status.SUCCESS); } else { @@ -340,7 +340,7 @@ public class DataAnalysisService extends BaseService{ if (StringUtils.isNotEmpty(taskQueueStr)){ String[] splits = taskQueueStr.split("_"); if (splits.length >= 4){ - tasksQueueIds[i++]=Integer.parseInt(splits[3]); + tasksQueueIds[i++] = Integer.parseInt(splits[3]); } } } @@ -350,7 +350,7 @@ public class DataAnalysisService extends BaseService{ if (StringUtils.isNotEmpty(taskKillStr)){ String[] splits = taskKillStr.split("-"); if (splits.length == 2){ - tasksKillIds[i++]=Integer.parseInt(splits[1]); + tasksKillIds[i++] = Integer.parseInt(splits[1]); } } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java index 88098ba608..c7afd76cc6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java @@ -16,36 +16,209 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.queue.ITaskQueue; +import org.apache.dolphinscheduler.common.queue.TaskQueueFactory; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.dao.ProcessDao; +import org.apache.dolphinscheduler.dao.entity.CommandCount; +import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; +import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.*; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest({TaskQueueFactory.class}) public class DataAnalysisServiceTest { - private static final Logger logger = LoggerFactory.getLogger(DataAnalysisServiceTest.class); - - @Autowired + + @InjectMocks private DataAnalysisService dataAnalysisService; + @Mock + ProjectMapper projectMapper; + + @Mock + ProjectService projectService; + + @Mock + ProcessInstanceMapper processInstanceMapper; + + @Mock + ProcessDefinitionMapper processDefinitionMapper; + + @Mock + CommandMapper commandMapper; + + @Mock + ErrorCommandMapper errorCommandMapper; + + @Mock + TaskInstanceMapper taskInstanceMapper; + + @Mock + ITaskQueue taskQueue; + + @Mock + ProcessDao processDao; + + private Project project; + + private Map resultMap; + + private User user; + + @Before + public void setUp() { + + user = new User(); + project = new Project(); + project.setId(1); + resultMap = new HashMap<>(); + Mockito.when(projectMapper.selectById(1)).thenReturn(project); + Mockito.when(projectService.hasProjectAndPerm(user,project,resultMap)).thenReturn(true); + + } + + + @After + public void after(){ + + user = null; + projectMapper = null; + resultMap = null; + } + + @Test - public void countDefinitionByUser(){ - User loginUser = new User(); - loginUser.setId(27); - loginUser.setUserType(UserType.GENERAL_USER); - Map map = dataAnalysisService.countDefinitionByUser(loginUser, 21); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); + public void testCountTaskStateByProject(){ + + String startDate = "2020-02-11 16:02:18"; + String endDate = "2020-02-11 16:03:18"; + + //checkProject false + Map result = dataAnalysisService.countTaskStateByProject(user, 2, startDate, endDate); + Assert.assertTrue(result.isEmpty()); + + // task instance state count error + result = dataAnalysisService.countTaskStateByProject(user, 1, startDate, endDate); + Assert.assertEquals(Status.TASK_INSTANCE_STATE_COUNT_ERROR,result.get(Constants.STATUS)); + + //SUCCESS + Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(DateUtils.getScheduleDate(startDate), + DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(getTaskInstanceStateCounts()); + + result = dataAnalysisService.countTaskStateByProject(user, 1, startDate, endDate); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + + + @Test + public void testCountProcessInstanceStateByProject(){ + + String startDate = "2020-02-11 16:02:18"; + String endDate = "2020-02-11 16:03:18"; + //checkProject false + Map result = dataAnalysisService.countProcessInstanceStateByProject(user,2,startDate,endDate); + Assert.assertTrue(result.isEmpty()); + + //COUNT_PROCESS_INSTANCE_STATE_ERROR + result = dataAnalysisService.countProcessInstanceStateByProject(user,1,startDate,endDate); + Assert.assertEquals(Status.COUNT_PROCESS_INSTANCE_STATE_ERROR,result.get(Constants.STATUS)); + + //SUCCESS + Mockito.when(processInstanceMapper.countInstanceStateByUser(DateUtils.getScheduleDate(startDate), + DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(getTaskInstanceStateCounts()); + result = dataAnalysisService.countProcessInstanceStateByProject(user,1,startDate,endDate); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testCountDefinitionByUser(){ + + Map result = dataAnalysisService.countDefinitionByUser(user,1); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + + @Test + public void testCountCommandState(){ + + String startDate = "2020-02-11 16:02:18"; + String endDate = "2020-02-11 16:03:18"; + //checkProject false + Map result = dataAnalysisService.countCommandState(user,2,startDate,endDate); + Assert.assertTrue(result.isEmpty()); + List commandCounts = new ArrayList<>(1); + CommandCount commandCount = new CommandCount(); + commandCount.setCommandType(CommandType.START_PROCESS); + commandCounts.add(commandCount); + Mockito.when(commandMapper.countCommandState(0, DateUtils.getScheduleDate(startDate), + DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(commandCounts); + + Mockito.when(errorCommandMapper.countCommandState( DateUtils.getScheduleDate(startDate), + DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(commandCounts); + + result = dataAnalysisService.countCommandState(user,1,startDate,endDate); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + + @Test + public void testCountQueueState(){ + + PowerMockito.mockStatic(TaskQueueFactory.class); + List taskQueueList = new ArrayList<>(1); + taskQueueList.add("1_0_1_1_-1"); + List taskKillList = new ArrayList<>(1); + taskKillList.add("1-0"); + PowerMockito.when(taskQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_QUEUE)).thenReturn(taskQueueList); + PowerMockito.when(taskQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_KILL)).thenReturn(taskKillList); + PowerMockito.when(TaskQueueFactory.getTaskQueueInstance()).thenReturn(taskQueue); + //checkProject false + Map result = dataAnalysisService.countQueueState(user,2); + Assert.assertTrue(result.isEmpty()); + + result = dataAnalysisService.countQueueState(user,1); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + //admin + user.setUserType(UserType.ADMIN_USER); + result = dataAnalysisService.countQueueState(user,1); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + + /** + * get list + * @return + */ + private List getTaskInstanceStateCounts(){ + + List taskInstanceStateCounts = new ArrayList<>(1); + ExecuteStatusCount executeStatusCount = new ExecuteStatusCount(); + executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXEUTION); + taskInstanceStateCounts.add(executeStatusCount); + + return taskInstanceStateCounts; } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 77da183f53..53184e7412 100644 --- a/pom.xml +++ b/pom.xml @@ -701,6 +701,7 @@ **/api/service/BaseServiceTest.java **/api/service/BaseDAGServiceTest.java **/api/service/LoggerServiceTest.java + **/api/service/DataAnalysisServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From 768b5d9dfdbf4b9397f958713b950d9b5583205a Mon Sep 17 00:00:00 2001 From: Yelli Date: Wed, 12 Feb 2020 15:05:21 +0800 Subject: [PATCH 36/43] add state null check (#1939) add process definition UT --- .../api/service/ProcessDefinitionService.java | 7 + .../service/ProcessDefinitionServiceTest.java | 160 +++++++++++++++++- 2 files changed, 159 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 8a762d7557..e9cfe341dd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -442,6 +442,13 @@ public class ProcessDefinitionService extends BaseDAGService { } ReleaseState state = ReleaseState.getEnum(releaseState); + + // check state + if (null == state) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "releaseState"); + return result; + } + ProcessDefinition processDefinition = processDefineMapper.selectById(id); switch (state) { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index a4b07e1835..51b440b9ca 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.service; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.metadata.IPage; import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.dto.ProcessMeta; import org.apache.dolphinscheduler.api.enums.Status; @@ -99,7 +100,7 @@ public class ProcessDefinitionServiceTest { "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; @Test - public void queryProccessDefinitionList() throws Exception { + public void testQueryProccessDefinitionList() { String projectName = "project_test1"; Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); @@ -111,15 +112,23 @@ public class ProcessDefinitionServiceTest { Map result = new HashMap<>(5); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); + //project not found Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); - Map map = processDefinitionService.queryProccessDefinitionList(loginUser,"project_test1"); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); - logger.info(JSON.toJSONString(map)); + + //project check auth success + putMsg(result, Status.SUCCESS, projectName); + Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + List resourceList = new ArrayList<>(); + resourceList.add(getProcessDefinition()); + Mockito.when(processDefineMapper.queryAllDefinitionList(project.getId())).thenReturn(resourceList); + Map checkSuccessRes = processDefinitionService.queryProccessDefinitionList(loginUser,"project_test1"); + Assert.assertEquals(Status.SUCCESS, checkSuccessRes.get(Constants.STATUS)); } @Test - public void queryProcessDefinitionListPagingTest() throws Exception { + public void testQueryProcessDefinitionListPaging() { String projectName = "project_test1"; Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); @@ -132,12 +141,46 @@ public class ProcessDefinitionServiceTest { Map result = new HashMap<>(5); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); + //project not found Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); - Map map = processDefinitionService.queryProcessDefinitionListPaging(loginUser, "project_test1", "",1, 5,0); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); + + } + + @Test + public void testQueryProccessDefinitionById() { + String projectName = "project_test1"; + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + + Project project = getProject(projectName); + User loginUser = new User(); + loginUser.setId(-1); + loginUser.setUserType(UserType.GENERAL_USER); + + Map result = new HashMap<>(5); + putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); + + //project check auth fail + Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Map map = processDefinitionService.queryProccessDefinitionById(loginUser, + "project_test1", 1); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); - logger.info(JSON.toJSONString(map)); + + //project check auth success, instance not exist + putMsg(result, Status.SUCCESS, projectName); + Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(processDefineMapper.selectById(1)).thenReturn(null); + Map instanceNotexitRes = processDefinitionService.queryProccessDefinitionById(loginUser, + "project_test1", 1); + Assert.assertEquals(Status.PROCESS_INSTANCE_NOT_EXIST, instanceNotexitRes.get(Constants.STATUS)); + + //instance exit + Mockito.when(processDefineMapper.selectById(46)).thenReturn(getProcessDefinition()); + Map successRes = processDefinitionService.queryProccessDefinitionById(loginUser, + "project_test1", 46); + Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); } @Test @@ -150,14 +193,115 @@ public class ProcessDefinitionServiceTest { loginUser.setId(-1); loginUser.setUserType(UserType.GENERAL_USER); + //project check auth fail Map result = new HashMap<>(5); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); - Map map = processDefinitionService.deleteProcessDefinitionById(loginUser, "project_test1", 6); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); + //project check auth success, instance not exist + putMsg(result, Status.SUCCESS, projectName); + Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(processDefineMapper.selectById(1)).thenReturn(null); + Map instanceNotexitRes = processDefinitionService.deleteProcessDefinitionById(loginUser, + "project_test1", 1); + Assert.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST, instanceNotexitRes.get(Constants.STATUS)); + + ProcessDefinition processDefinition = getProcessDefinition(); + //user no auth + loginUser.setUserType(UserType.GENERAL_USER); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); + Map userNoAuthRes = processDefinitionService.deleteProcessDefinitionById(loginUser, + "project_test1", 46); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, userNoAuthRes.get(Constants.STATUS)); + + //process definition online + loginUser.setUserType(UserType.ADMIN_USER); + processDefinition.setReleaseState(ReleaseState.ONLINE); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); + Map dfOnlineRes = processDefinitionService.deleteProcessDefinitionById(loginUser, + "project_test1", 46); + Assert.assertEquals(Status.PROCESS_DEFINE_STATE_ONLINE, dfOnlineRes.get(Constants.STATUS)); + + //scheduler list elements > 1 + processDefinition.setReleaseState(ReleaseState.OFFLINE); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); + List schedules = new ArrayList<>(); + schedules.add(getSchedule()); + schedules.add(getSchedule()); + Mockito.when(scheduleMapper.queryByProcessDefinitionId(46)).thenReturn(schedules); + Map schedulerGreaterThanOneRes = processDefinitionService.deleteProcessDefinitionById(loginUser, + "project_test1", 46); + Assert.assertEquals(Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR, schedulerGreaterThanOneRes.get(Constants.STATUS)); + + //scheduler online + schedules.clear(); + Schedule schedule = getSchedule(); + schedule.setReleaseState(ReleaseState.ONLINE); + schedules.add(schedule); + Mockito.when(scheduleMapper.queryByProcessDefinitionId(46)).thenReturn(schedules); + Map schedulerOnlineRes = processDefinitionService.deleteProcessDefinitionById(loginUser, + "project_test1", 46); + Assert.assertEquals(Status.SCHEDULE_CRON_STATE_ONLINE, schedulerOnlineRes.get(Constants.STATUS)); + + //delete fail + schedules.clear(); + schedule.setReleaseState(ReleaseState.OFFLINE); + schedules.add(schedule); + Mockito.when(scheduleMapper.queryByProcessDefinitionId(46)).thenReturn(schedules); + Mockito.when(processDefineMapper.deleteById(46)).thenReturn(0); + Map deleteFail = processDefinitionService.deleteProcessDefinitionById(loginUser, + "project_test1", 46); + Assert.assertEquals(Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR, deleteFail.get(Constants.STATUS)); + + //delte success + Mockito.when(processDefineMapper.deleteById(46)).thenReturn(1); + Map deleteSuccess = processDefinitionService.deleteProcessDefinitionById(loginUser, + "project_test1", 46); + Assert.assertEquals(Status.SUCCESS, deleteSuccess.get(Constants.STATUS)); + } + + @Test + public void testReleaseProcessDefinition() { + String projectName = "project_test1"; + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + + Project project = getProject(projectName); + User loginUser = new User(); + loginUser.setId(-1); + loginUser.setUserType(UserType.GENERAL_USER); + + //project check auth fail + Map result = new HashMap<>(5); + putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); + Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Map map = processDefinitionService.releaseProcessDefinition(loginUser, "project_test1", + 6, ReleaseState.OFFLINE.getCode()); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); - logger.info(JSON.toJSONString(map)); + + //project check auth success, processs definition online + putMsg(result, Status.SUCCESS, projectName); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(getProcessDefinition()); + Mockito.when(processDefineMapper.updateById(getProcessDefinition())).thenReturn(1); + Map onlineRes = processDefinitionService.releaseProcessDefinition(loginUser, "project_test1", + 46, ReleaseState.ONLINE.getCode()); + Assert.assertEquals(Status.SUCCESS, onlineRes.get(Constants.STATUS)); + + //process definition offline + List schedules = new ArrayList<>(); + Schedule schedule = getSchedule(); + schedules.add(schedule); + Mockito.when(scheduleMapper.selectAllByProcessDefineArray(new int[]{46})).thenReturn(schedules); + Mockito.when(scheduleMapper.updateById(schedule)).thenReturn(1); + Map offlineRes = processDefinitionService.releaseProcessDefinition(loginUser, "project_test1", + 46, ReleaseState.OFFLINE.getCode()); + Assert.assertEquals(Status.SUCCESS, offlineRes.get(Constants.STATUS)); + + //release error code + Map failRes = processDefinitionService.releaseProcessDefinition(loginUser, "project_test1", + 46, 2); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failRes.get(Constants.STATUS)); } /** From 44b76468a823ce380e57272cd2b0ce6861419b62 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Thu, 13 Feb 2020 12:42:08 +0800 Subject: [PATCH 37/43] Adapting partial code(file name start with C) to the sonar cloud rules (#1918) * Adapting partial code(file name start with C) to the sonar cloud rules * add some UT * add some UT(rollback JSONUtils.checkJsonVaild) * add some UT(fix OSUtils.exeCmd UT) --- .../alert/utils/Constants.java | 4 +- .../api/service/ResourcesService.java | 2 +- .../api/utils/CheckUtils.java | 4 +- .../api/utils/CheckUtilsTest.java | 111 ++++++++++++++++++ .../dolphinscheduler/common/Constants.java | 52 ++++---- .../common/job/db/ClickHouseDataSource.java | 2 +- .../task/subprocess/SubProcessParameters.java | 2 +- .../common/utils/CollectionUtils.java | 27 +---- .../common/utils/CommonUtils.java | 7 +- .../common/utils/OSUtils.java | 23 +--- .../common/utils/CollectionUtilsTest.java | 43 ++++++- .../common/utils/CommonUtilsTest.java | 36 ++++++ .../common/utils/OSUtilsTest.java | 83 +++++++++++++ .../dao/datasource/ConnectionFactory.java | 5 - .../dao/mapper/CommandMapper.java | 5 +- .../dao/utils/cron/CronUtils.java | 8 +- .../dao/utils/cron/CycleFactory.java | 4 +- .../dao/cron/CronUtilsTest.java | 21 ++-- .../dao/mapper/ConnectionFactoryTest.java | 2 +- .../server/master/MasterServer.java | 2 +- .../server/utils/FlinkArgsUtils.java | 2 +- .../server/worker/WorkerServer.java | 2 +- .../worker/task/AbstractCommandExecutor.java | 2 +- 23 files changed, 339 insertions(+), 110 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java index 55c0f9ffbf..94d95b3c26 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java @@ -20,7 +20,9 @@ package org.apache.dolphinscheduler.alert.utils; * constants */ public class Constants { - + private Constants() { + throw new IllegalStateException("Constants class"); + } /** * alert properties path */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 29a16447e1..6438e206f8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -119,7 +119,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR); return result; } - if (file.getSize() > Constants.maxFileSize) { + if (file.getSize() > Constants.MAX_FILE_SIZE) { logger.error("file size is too large: {}", file.getOriginalFilename()); putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT); return result; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index 7099378b1d..c5c702404d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -35,7 +35,9 @@ import java.util.regex.Pattern; */ public class CheckUtils { - + private CheckUtils() { + throw new IllegalStateException("CheckUtils class"); + } /** * check username * diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java index b0b4319fb4..24a0ed31d6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java @@ -17,8 +17,26 @@ package org.apache.dolphinscheduler.api.utils; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ProgramType; +import org.apache.dolphinscheduler.common.enums.TaskType; +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.task.datax.DataxParameters; +import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; +import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; +import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters; +import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters; +import org.apache.dolphinscheduler.common.task.python.PythonParameters; +import org.apache.dolphinscheduler.common.task.shell.ShellParameters; +import org.apache.dolphinscheduler.common.task.spark.SparkParameters; +import org.apache.dolphinscheduler.common.task.sql.SqlParameters; +import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -78,6 +96,14 @@ public class CheckUtilsTest { } + @Test + public void testCheckOtherParams() { + assertFalse(CheckUtils.checkOtherParams(null)); + assertFalse(CheckUtils.checkOtherParams("")); + assertTrue(CheckUtils.checkOtherParams("xxx")); + assertFalse(CheckUtils.checkOtherParams("{}")); + assertFalse(CheckUtils.checkOtherParams("{\"key1\":111}")); + } /** * check passwd */ @@ -106,5 +132,90 @@ public class CheckUtilsTest { assertTrue(CheckUtils.checkPhone("17362537263")); } + @Test + public void testCheckTaskNodeParameters() { + + assertFalse(CheckUtils.checkTaskNodeParameters(null,null)); + assertFalse(CheckUtils.checkTaskNodeParameters(null,"unKnown")); + assertFalse(CheckUtils.checkTaskNodeParameters("unKnown","unKnown")); + assertFalse(CheckUtils.checkTaskNodeParameters("unKnown",null)); + + // sub SubProcessParameters + SubProcessParameters subProcessParameters = new SubProcessParameters(); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(subProcessParameters), TaskType.SUB_PROCESS.toString())); + subProcessParameters.setProcessDefinitionId(1234); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(subProcessParameters), TaskType.SUB_PROCESS.toString())); + + // ShellParameters + ShellParameters shellParameters = new ShellParameters(); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(shellParameters), TaskType.SHELL.toString())); + shellParameters.setRawScript(""); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(shellParameters), TaskType.SHELL.toString())); + shellParameters.setRawScript("sss"); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(shellParameters), TaskType.SHELL.toString())); + + // ProcedureParameters + ProcedureParameters procedureParameters = new ProcedureParameters(); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(procedureParameters), TaskType.PROCEDURE.toString())); + procedureParameters.setDatasource(1); + procedureParameters.setType("xx"); + procedureParameters.setMethod("yy"); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(procedureParameters), TaskType.PROCEDURE.toString())); + + // SqlParameters + SqlParameters sqlParameters = new SqlParameters(); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(sqlParameters), TaskType.SQL.toString())); + sqlParameters.setDatasource(1); + sqlParameters.setType("xx"); + sqlParameters.setSql("yy"); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(sqlParameters), TaskType.SQL.toString())); + + // MapreduceParameters + MapreduceParameters mapreduceParameters = new MapreduceParameters(); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(mapreduceParameters), TaskType.MR.toString())); + mapreduceParameters.setMainJar(new ResourceInfo()); + mapreduceParameters.setProgramType(ProgramType.JAVA); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(mapreduceParameters), TaskType.MR.toString())); + + // SparkParameters + SparkParameters sparkParameters = new SparkParameters(); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(sparkParameters), TaskType.SPARK.toString())); + sparkParameters.setMainJar(new ResourceInfo()); + sparkParameters.setProgramType(ProgramType.SCALA); + sparkParameters.setSparkVersion("1.1.1"); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(sparkParameters), TaskType.SPARK.toString())); + + // PythonParameters + PythonParameters pythonParameters = new PythonParameters(); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(pythonParameters), TaskType.PYTHON.toString())); + pythonParameters.setRawScript("ss"); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(pythonParameters), TaskType.PYTHON.toString())); + + // DependentParameters + DependentParameters dependentParameters = new DependentParameters(); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(dependentParameters), TaskType.DEPENDENT.toString())); + + // FlinkParameters + FlinkParameters flinkParameters = new FlinkParameters(); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(flinkParameters), TaskType.FLINK.toString())); + flinkParameters.setMainJar(new ResourceInfo()); + flinkParameters.setProgramType(ProgramType.JAVA); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(flinkParameters), TaskType.FLINK.toString())); + + // HTTP + HttpParameters httpParameters = new HttpParameters(); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(httpParameters), TaskType.HTTP.toString())); + httpParameters.setUrl("httpUrl"); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(httpParameters), TaskType.HTTP.toString())); + + // DataxParameters + DataxParameters dataxParameters = new DataxParameters(); + assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(dataxParameters), TaskType.DATAX.toString())); + dataxParameters.setDataSource(111); + dataxParameters.setDataTarget(333); + dataxParameters.setSql("sql"); + dataxParameters.setTargetTable("tar"); + assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(dataxParameters), TaskType.DATAX.toString())); + } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 9cb1a5821e..73125f4926 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -25,7 +25,9 @@ import java.util.regex.Pattern; * Constants */ public final class Constants { - + private Constants() { + throw new IllegalStateException("Constants class"); + } /** * common properties path */ @@ -124,49 +126,41 @@ public final class Constants { /** * MasterServer directory registered in zookeeper */ - //public static final String ZOOKEEPER_DOLPHINSCHEDULER_MASTERS = "zookeeper.dolphinscheduler.masters"; public static final String ZOOKEEPER_DOLPHINSCHEDULER_MASTERS = "/masters"; /** * WorkerServer directory registered in zookeeper */ - //public static final String ZOOKEEPER_DOLPHINSCHEDULER_WORKERS = "zookeeper.dolphinscheduler.workers"; public static final String ZOOKEEPER_DOLPHINSCHEDULER_WORKERS = "/workers"; /** * all servers directory registered in zookeeper */ - //public static final String ZOOKEEPER_DOLPHINSCHEDULER_DEAD_SERVERS = "zookeeper.dolphinscheduler.dead.servers"; public static final String ZOOKEEPER_DOLPHINSCHEDULER_DEAD_SERVERS = "/dead-servers"; /** * MasterServer lock directory registered in zookeeper */ - //public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_MASTERS = "zookeeper.dolphinscheduler.lock.masters"; public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_MASTERS = "/lock/masters"; /** * WorkerServer lock directory registered in zookeeper */ - //public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_WORKERS = "zookeeper.dolphinscheduler.lock.workers"; public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_WORKERS = "/lock/workers"; /** * MasterServer failover directory registered in zookeeper */ - //public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS = "zookeeper.dolphinscheduler.lock.failover.masters"; public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS = "/lock/failover/masters"; /** * WorkerServer failover directory registered in zookeeper */ - //public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS = "zookeeper.dolphinscheduler.lock.failover.workers"; public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS = "/lock/failover/workers"; /** * MasterServer startup failover runing and fault tolerance process */ - //public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS = "zookeeper.dolphinscheduler.lock.failover.startup.masters"; public static final String ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS = "/lock/failover/startup-masters"; /** @@ -354,87 +348,87 @@ public final class Constants { /** * heartbeat threads number */ - public static final int defaulWorkerHeartbeatThreadNum = 1; + public static final int DEFAUL_WORKER_HEARTBEAT_THREAD_NUM = 1; /** * heartbeat interval */ - public static final int defaultWorkerHeartbeatInterval = 60; + public static final int DEFAULT_WORKER_HEARTBEAT_INTERVAL = 60; /** * worker fetch task number */ - public static final int defaultWorkerFetchTaskNum = 1; + public static final int DEFAULT_WORKER_FETCH_TASK_NUM = 1; /** * worker execute threads number */ - public static final int defaultWorkerExecThreadNum = 10; + public static final int DEFAULT_WORKER_EXEC_THREAD_NUM = 10; /** * master cpu load */ - public static final int defaultMasterCpuLoad = Runtime.getRuntime().availableProcessors() * 2; + public static final int DEFAULT_MASTER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2; /** * master reserved memory */ - public static final double defaultMasterReservedMemory = OSUtils.totalMemorySize() / 10; + public static final double DEFAULT_MASTER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10; /** * worker cpu load */ - public static final int defaultWorkerCpuLoad = Runtime.getRuntime().availableProcessors() * 2; + public static final int DEFAULT_WORKER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2; /** * worker reserved memory */ - public static final double defaultWorkerReservedMemory = OSUtils.totalMemorySize() / 10; + public static final double DEFAULT_WORKER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10; /** * master execute threads number */ - public static final int defaultMasterExecThreadNum = 100; + public static final int DEFAULT_MASTER_EXEC_THREAD_NUM = 100; /** * default master concurrent task execute num */ - public static final int defaultMasterTaskExecNum = 20; + public static final int DEFAULT_MASTER_TASK_EXEC_NUM = 20; /** * default log cache rows num,output when reach the number */ - public static final int defaultLogRowsNum = 4 * 16; + public static final int DEFAULT_LOG_ROWS_NUM = 4 * 16; /** * log flush interval，output when reach the interval */ - public static final int defaultLogFlushInterval = 1000; + public static final int DEFAULT_LOG_FLUSH_INTERVAL = 1000; /** * default master heartbeat thread number */ - public static final int defaulMasterHeartbeatThreadNum = 1; + public static final int DEFAULT_MASTER_HEARTBEAT_THREAD_NUM = 1; /** * default master heartbeat interval */ - public static final int defaultMasterHeartbeatInterval = 60; + public static final int DEFAULT_MASTER_HEARTBEAT_INTERVAL = 60; /** * default master commit retry times */ - public static final int defaultMasterCommitRetryTimes = 5; + public static final int DEFAULT_MASTER_COMMIT_RETRY_TIMES = 5; /** * default master commit retry interval */ - public static final int defaultMasterCommitRetryInterval = 3000; + public static final int DEFAULT_MASTER_COMMIT_RETRY_INTERVAL = 3000; /** * time unit secong to minutes @@ -474,9 +468,9 @@ public final class Constants { public static final String THREAD_NAME_MASTER_SERVER = "Master-Server"; public static final String THREAD_NAME_WORKER_SERVER = "Worker-Server"; - public static String TASK_RECORD_TABLE_HIVE_LOG = "eamp_hive_log_hd"; + public static final String TASK_RECORD_TABLE_HIVE_LOG = "eamp_hive_log_hd"; - public static String TASK_RECORD_TABLE_HISTORY_HIVE_LOG = "eamp_hive_hist_log_hd"; + public static final String TASK_RECORD_TABLE_HISTORY_HIVE_LOG = "eamp_hive_hist_log_hd"; /** @@ -874,7 +868,7 @@ public final class Constants { public static final String FLINK_JOB_MANAGE_MEM = "-yjm"; public static final String FLINK_TASK_MANAGE_MEM = "-ytm"; - public static final String FLINK_detach = "-d"; + public static final String FLINK_DETACH = "-d"; public static final String FLINK_MAIN_CLASS = "-c"; @@ -989,7 +983,7 @@ public final class Constants { * session timeout */ public static final int SESSION_TIME_OUT = 7200; - public static final int maxFileSize = 1024 * 1024 * 1024; + public static final int MAX_FILE_SIZE = 1024 * 1024 * 1024; public static final String UDF = "UDF"; public static final String CLASS = "class"; public static final String RECEIVERS = "receivers"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java index fe76497ff8..48550c31cc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java @@ -38,7 +38,7 @@ public class ClickHouseDataSource extends BaseDataSource { @Override public String getJdbcUrl() { String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { + if (jdbcUrl.lastIndexOf('/') != (jdbcUrl.length() - 1)) { jdbcUrl += "/"; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/subprocess/SubProcessParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/subprocess/SubProcessParameters.java index 21e3ce2248..c7784de8dd 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/subprocess/SubProcessParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/subprocess/SubProcessParameters.java @@ -38,7 +38,7 @@ public class SubProcessParameters extends AbstractParameters { @Override public boolean checkParameters() { - return this.processDefinitionId != 0; + return this.processDefinitionId != null && this.processDefinitionId != 0; } @Override diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java index 9c02111c36..22c58640cc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java @@ -37,6 +37,9 @@ import java.util.*; */ public class CollectionUtils { + private CollectionUtils() { + throw new IllegalStateException("CollectionUtils class"); + } /** * Returns a new {@link Collection} containing a minus a subset of * b. Only the elements of b that satisfy the predicate @@ -139,26 +142,6 @@ public class CollectionUtils { cardinalityB = CollectionUtils.getCardinalityMap(b); } - /** - * Returns the maximum frequency of an object. - * - * @param obj the object - * @return the maximum frequency of the object - */ - private int max(final Object obj) { - return Math.max(freqA(obj), freqB(obj)); - } - - /** - * Returns the minimum frequency of an object. - * - * @param obj the object - * @return the minimum frequency of the object - */ - private int min(final Object obj) { - return Math.min(freqA(obj), freqB(obj)); - } - /** * Returns the frequency of this object in collection A. * @@ -225,7 +208,7 @@ public class CollectionUtils { if (a.size() != b.size()) { return false; } - final CardinalityHelper helper = new CardinalityHelper(a, b); + final CardinalityHelper helper = new CardinalityHelper<>(a, b); if (helper.cardinalityA.size() != helper.cardinalityB.size()) { return false; } @@ -250,7 +233,7 @@ public class CollectionUtils { * @return the populated cardinality map */ public static Map getCardinalityMap(final Iterable coll) { - final Map count = new HashMap(); + final Map count = new HashMap<>(); for (final O obj : coll) { count.put(obj, count.getOrDefault(obj, 0) + 1); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index 842c74edc0..b4b89bfe26 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -20,8 +20,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; @@ -29,8 +27,9 @@ import java.io.File; * common utils */ public class CommonUtils { - - private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class); + private CommonUtils() { + throw new IllegalStateException("CommonUtils class"); + } /** * @return get the path of system environment variables diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 0c061fc0ba..c8d33e3ef2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -370,25 +370,14 @@ public class OSUtils { double systemCpuLoad; double systemReservedMemory; - if(isMaster){ - systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.defaultMasterCpuLoad); - systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.defaultMasterReservedMemory); + if(Boolean.TRUE.equals(isMaster)){ + systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD); + systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY); }else{ - systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.defaultWorkerCpuLoad); - systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.defaultWorkerReservedMemory); - } - - // judging usage - double loadAverage = OSUtils.loadAverage(); - // - double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); - - if(loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory){ - logger.warn("load or availablePhysicalMemorySize(G) is too high, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); - return false; - }else{ - return true; + systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD); + systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY); } + return checkResource(systemCpuLoad,systemReservedMemory); } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java index 7321879ab8..99685265e6 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java @@ -76,11 +76,11 @@ public class CollectionUtilsTest { a = CollectionUtils.stringToMap("a=b;c=d", null); Assert.assertTrue(a.isEmpty()); a = CollectionUtils.stringToMap("a=b;c=d;e=f", ";"); - Assert.assertEquals(a.size(), 3); + Assert.assertEquals(3, a.size()); a = CollectionUtils.stringToMap("a;b=f", ";"); Assert.assertTrue(a.isEmpty()); a = CollectionUtils.stringToMap("a=b;c=d;e=f;", ";", "test"); - Assert.assertEquals(a.size(), 3); + Assert.assertEquals(3, a.size()); Assert.assertNotNull(a.get("testa")); } @@ -91,14 +91,14 @@ public class CollectionUtilsTest { originList.add(1); originList.add(2); List> ret = CollectionUtils.getListByExclusion(originList, null); - Assert.assertEquals(ret.size(), 2); + Assert.assertEquals(2, ret.size()); ret = CollectionUtils.getListByExclusion(originList, new HashSet<>()); - Assert.assertEquals(ret.size(), 2); + Assert.assertEquals(2, ret.size()); Assert.assertFalse(ret.get(0).isEmpty()); Set exclusion = new HashSet<>(); exclusion.add(Constants.CLASS); ret = CollectionUtils.getListByExclusion(originList, exclusion); - Assert.assertEquals(ret.size(), 2); + Assert.assertEquals(2, ret.size()); Assert.assertTrue(ret.get(0).isEmpty()); } @@ -108,5 +108,38 @@ public class CollectionUtilsTest { Assert.assertFalse(CollectionUtils.isNotEmpty(list)); Assert.assertFalse(CollectionUtils.isNotEmpty(null)); } + @Test + public void isEmpty(){ + List list = new ArrayList<>(); + Assert.assertTrue(CollectionUtils.isEmpty(list)); + Assert.assertTrue(CollectionUtils.isEmpty(null)); + list.add(1); + Assert.assertFalse(CollectionUtils.isEmpty(list)); + } + @Test + public void isEqualCollection() { + List a = new ArrayList<>(); + a.add(1); + List b = new ArrayList<>(); + b.add(1); + Assert.assertTrue(CollectionUtils.isEqualCollection(a,b)); + b.add(2); + Assert.assertFalse(CollectionUtils.isEqualCollection(a,b)); + } + @Test + public void getCardinalityMap(){ + List a = new ArrayList<>(); + a.add(1); + a.add(2); + a.add(2); + a.add(3); + a.add(3); + a.add(3); + Map cardinalityMap = CollectionUtils.getCardinalityMap(a); + Assert.assertEquals(3, cardinalityMap.size()); + Assert.assertEquals(1, cardinalityMap.get(1).intValue()); + Assert.assertEquals(2, cardinalityMap.get(2).intValue()); + Assert.assertEquals(3, cardinalityMap.get(3).intValue()); + } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java index f38b9b4c3b..42c9958810 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,24 +29,58 @@ import java.net.UnknownHostException; */ public class CommonUtilsTest { private static final Logger logger = LoggerFactory.getLogger(CommonUtilsTest.class); + @Test + public void getSystemEnvPath() { + logger.info(CommonUtils.getSystemEnvPath()); + Assert.assertTrue(true); + } + @Test + public void getQueueImplValue(){ + logger.info(CommonUtils.getQueueImplValue()); + Assert.assertTrue(true); + } + @Test + public void isDevelopMode() { + logger.info("develop mode: {}",CommonUtils.isDevelopMode()); + Assert.assertTrue(true); + } + @Test + public void getKerberosStartupState(){ + logger.info("kerberos startup state: {}",CommonUtils.getKerberosStartupState()); + Assert.assertTrue(true); + } + @Test + public void loadKerberosConf(){ + try { + CommonUtils.loadKerberosConf(); + Assert.assertTrue(true); + } catch (Exception e) { + Assert.fail("load Kerberos Conf failed"); + } + } + @Test public void getHdfsDataBasePath() { logger.info(HadoopUtils.getHdfsDataBasePath()); + Assert.assertTrue(true); } @Test public void getDownloadFilename() { logger.info(FileUtils.getDownloadFilename("a.txt")); + Assert.assertTrue(true); } @Test public void getUploadFilename() { logger.info(FileUtils.getUploadFilename("1234", "a.txt")); + Assert.assertTrue(true); } @Test public void getHdfsDir() { logger.info(HadoopUtils.getHdfsResDir("1234")); + Assert.assertTrue(true); } @Test @@ -57,5 +92,6 @@ public class CommonUtilsTest { } catch (UnknownHostException e) { e.printStackTrace(); } + Assert.assertTrue(true); } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 391fb594f0..5b23847ba3 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -16,10 +16,16 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.yetus.audience.InterfaceAudience; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import java.io.IOException; import java.util.List; public class OSUtilsTest { @@ -31,4 +37,81 @@ public class OSUtilsTest { Assert.assertNotEquals("System user list should not be empty", userList.size(), 0); logger.info("OS user list : {}", userList.toString()); } + @Test + public void testOSMetric(){ + + double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); + Assert.assertTrue(availablePhysicalMemorySize > 0.0f); + double totalMemorySize = OSUtils.totalMemorySize(); + Assert.assertTrue(totalMemorySize > 0.0f); + double loadAverage = OSUtils.loadAverage(); + logger.info("loadAverage {}", loadAverage); + double memoryUsage = OSUtils.memoryUsage(); + Assert.assertTrue(memoryUsage > 0.0f); + double cpuUsage = OSUtils.cpuUsage(); + Assert.assertTrue(cpuUsage > 0.0f); + } + @Test + public void getGroup() { + if(OSUtils.isMacOS() || !OSUtils.isWindows()){ + try { + String group = OSUtils.getGroup(); + Assert.assertNotNull(group); + } catch (IOException e) { + Assert.fail("get group failed " + e.getMessage()); + } + } + } + @Test + public void exeCmd() { + if(OSUtils.isMacOS() || !OSUtils.isWindows()){ + try { + String result = OSUtils.exeCmd("echo helloWorld"); + Assert.assertEquals("helloWorld\n",result); + } catch (IOException e) { + Assert.fail("exeCmd " + e.getMessage()); + } + } + } + @Test + public void getProcessID(){ + int processId = OSUtils.getProcessID(); + Assert.assertNotEquals(0, processId); + } + @Test + public void getHost(){ + String host = OSUtils.getHost(); + Assert.assertNotNull(host); + Assert.assertNotEquals("", host); + } + @Test + public void checkResource(){ + boolean resource = OSUtils.checkResource(100,0); + Assert.assertTrue(resource); + resource = OSUtils.checkResource(0,Double.MAX_VALUE); + Assert.assertFalse(resource); + + Configuration configuration = new PropertiesConfiguration(); + + configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,100); + configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,0); + resource = OSUtils.checkResource(configuration,true); + Assert.assertTrue(resource); + + configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,0); + configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,Double.MAX_VALUE); + resource = OSUtils.checkResource(configuration,true); + Assert.assertFalse(resource); + + configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,100); + configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,0); + resource = OSUtils.checkResource(configuration,false); + Assert.assertTrue(resource); + + configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,0); + configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,Double.MAX_VALUE); + resource = OSUtils.checkResource(configuration,false); + Assert.assertFalse(resource); + + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java index 6fdc233455..a3bc6a0150 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java @@ -20,8 +20,6 @@ import com.alibaba.druid.pool.DruidDataSource; import com.baomidou.mybatisplus.core.MybatisConfiguration; import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.dolphinscheduler.common.Constants; import org.apache.ibatis.mapping.Environment; import org.apache.ibatis.session.SqlSession; @@ -31,8 +29,6 @@ import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; import org.mybatis.spring.SqlSessionTemplate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Service; import javax.sql.DataSource; @@ -117,7 +113,6 @@ public class ConnectionFactory extends SpringConnectionFactory{ sqlSessionFactoryBean.setTypeEnumsPackage("org.apache.dolphinscheduler.*.enums"); sqlSessionFactory = sqlSessionFactoryBean.getObject(); - return sqlSessionFactory; } } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java index 18b643513f..c358cab3f3 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java @@ -16,13 +16,10 @@ */ package org.apache.dolphinscheduler.dao.mapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.CommandCount; -import com.baomidou.mybatisplus.core.conditions.Wrapper; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.toolkit.Constants; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; import java.util.Date; import java.util.List; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java index 8a9087a33c..2ad029064e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java @@ -38,7 +38,9 @@ import static org.apache.dolphinscheduler.dao.utils.cron.CycleFactory.*; * cron utils */ public class CronUtils { - + private CronUtils() { + throw new IllegalStateException("CronUtils class"); + } private static final Logger logger = LoggerFactory.getLogger(CronUtils.class); @@ -169,7 +171,7 @@ public class CronUtils { cronExpression = parse2CronExpression(cron); }catch (ParseException e){ logger.error(e.getMessage(), e); - return Collections.EMPTY_LIST; + return Collections.emptyList(); } return getSelfFireDateList(startTime, endTime, cronExpression); } @@ -202,7 +204,7 @@ public class CronUtils { calendar.add(Calendar.DATE, 1); break; default: - logger.error("Dependent process definition's cycleEnum is {},not support!!", cycleEnum.name()); + logger.error("Dependent process definition's cycleEnum is {},not support!!", cycleEnum); break; } maxExpirationTime = calendar.getTime(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleFactory.java index 10906b42a3..b2f52566fc 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleFactory.java @@ -25,7 +25,9 @@ import com.cronutils.model.field.expression.QuestionMark; * Crontab Cycle Tool Factory */ public class CycleFactory { - + private CycleFactory() { + throw new IllegalStateException("CycleFactory class"); + } /** * min * @param cron cron diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java index 1135cf20f5..5ecc6620dd 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java @@ -61,7 +61,7 @@ public class CronUtilsTest { String cronAsString = cron.asString(); // 0 */5 * * * ? * Every five minutes(once every 5 minutes) - Assert.assertEquals(cronAsString, "0 */5 * * * ? *"); + Assert.assertEquals("0 */5 * * * ? *", cronAsString); } @@ -74,12 +74,12 @@ public class CronUtilsTest { String strCrontab = "0 1 2 3 * ? *"; Cron depCron = CronUtils.parse2Cron(strCrontab); - Assert.assertEquals(depCron.retrieve(CronFieldName.SECOND).getExpression().asString(), "0"); - Assert.assertEquals(depCron.retrieve(CronFieldName.MINUTE).getExpression().asString(), "1"); - Assert.assertEquals(depCron.retrieve(CronFieldName.HOUR).getExpression().asString(), "2"); - Assert.assertEquals(depCron.retrieve(CronFieldName.DAY_OF_MONTH).getExpression().asString(), "3"); - Assert.assertEquals(depCron.retrieve(CronFieldName.MONTH).getExpression().asString(), "*"); - Assert.assertEquals(depCron.retrieve(CronFieldName.YEAR).getExpression().asString(), "*"); + Assert.assertEquals("0", depCron.retrieve(CronFieldName.SECOND).getExpression().asString()); + Assert.assertEquals("1", depCron.retrieve(CronFieldName.MINUTE).getExpression().asString()); + Assert.assertEquals("2", depCron.retrieve(CronFieldName.HOUR).getExpression().asString()); + Assert.assertEquals("3", depCron.retrieve(CronFieldName.DAY_OF_MONTH).getExpression().asString()); + Assert.assertEquals("*", depCron.retrieve(CronFieldName.MONTH).getExpression().asString()); + Assert.assertEquals("*", depCron.retrieve(CronFieldName.YEAR).getExpression().asString()); } /** @@ -89,13 +89,13 @@ public class CronUtilsTest { @Test public void testScheduleType() throws ParseException { CycleEnum cycleEnum = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 */1 * * * ? *")); - Assert.assertEquals(cycleEnum.name(), "MINUTE"); + Assert.assertEquals("MINUTE", cycleEnum.name()); CycleEnum cycleEnum2 = CronUtils.getMaxCycle("0 * * * * ? *"); - Assert.assertEquals(cycleEnum2.name(), "MINUTE"); + Assert.assertEquals("MINUTE", cycleEnum2.name()); CycleEnum cycleEnum3 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 * * * * ? *")); - Assert.assertEquals(cycleEnum3.name(), "MINUTE"); + Assert.assertEquals("MINUTE", cycleEnum3.name()); } /** @@ -164,6 +164,7 @@ public class CronUtilsTest { logger.info("can't get scheduleType"); } } + Assert.assertTrue(true); } @Test diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ConnectionFactoryTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ConnectionFactoryTest.java index 57b002e18d..5ba2936aaf 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ConnectionFactoryTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ConnectionFactoryTest.java @@ -32,6 +32,6 @@ public class ConnectionFactoryTest { @Test public void testConnection()throws Exception{ Connection connection = ConnectionFactory.getDataSource().getPooledConnection().getConnection(); - Assert.assertEquals(connection != null , true); + Assert.assertTrue(connection != null); } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 0647b9450b..ab4ba5c8ab 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -111,7 +111,7 @@ public class MasterServer implements IStoppable { masterSchedulerService = ThreadUtils.newDaemonSingleThreadExecutor("Master-Scheduler-Thread"); - heartbeatMasterService = ThreadUtils.newDaemonThreadScheduledExecutor("Master-Main-Thread",Constants.defaulMasterHeartbeatThreadNum); + heartbeatMasterService = ThreadUtils.newDaemonThreadScheduledExecutor("Master-Main-Thread",Constants.DEFAULT_MASTER_HEARTBEAT_THREAD_NUM); // heartbeat thread implement Runnable heartBeatThread = heartBeatThread(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java index ab34ddfc2b..4c33ef8db2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java @@ -87,7 +87,7 @@ public class FlinkArgsUtils { args.add(taskManagerMemory); } - args.add(Constants.FLINK_detach); //-d + args.add(Constants.FLINK_DETACH); //-d } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index d270880408..99d418f048 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -149,7 +149,7 @@ public class WorkerServer implements IStoppable { this.fetchTaskExecutorService = ThreadUtils.newDaemonSingleThreadExecutor("Worker-Fetch-Thread-Executor"); - heartbeatWorkerService = ThreadUtils.newDaemonThreadScheduledExecutor("Worker-Heartbeat-Thread-Executor", Constants.defaulWorkerHeartbeatThreadNum); + heartbeatWorkerService = ThreadUtils.newDaemonThreadScheduledExecutor("Worker-Heartbeat-Thread-Executor", Constants.DEFAUL_WORKER_HEARTBEAT_THREAD_NUM); // heartbeat thread implement Runnable heartBeatThread = heartBeatThread(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 04098215dd..f1c01aff36 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -535,7 +535,7 @@ public abstract class AbstractCommandExecutor { /** * when log buffer siz or flush time reach condition , then flush */ - if (logBuffer.size() >= Constants.defaultLogRowsNum || now - lastFlushTime > Constants.defaultLogFlushInterval) { + if (logBuffer.size() >= Constants.DEFAULT_LOG_ROWS_NUM || now - lastFlushTime > Constants.DEFAULT_LOG_FLUSH_INTERVAL) { lastFlushTime = now; /** log handle */ logHandler.accept(logBuffer); From 6c90575a39721b8342aff72bb725d646654c1154 Mon Sep 17 00:00:00 2001 From: Yeleights Date: Thu, 13 Feb 2020 15:39:30 +0800 Subject: [PATCH 38/43] add process null check Remove duplicate code in processDefinitionService add processDefinitionService UT --- .../api/service/BaseDAGService.java | 40 +--- .../api/service/ProcessDefinitionService.java | 87 +++------ .../service/ProcessDefinitionServiceTest.java | 177 ++++++++++++++++-- .../dolphinscheduler/dao/utils/DagHelper.java | 35 +++- .../dao/utils/DagHelperTest.java | 19 +- 5 files changed, 239 insertions(+), 119 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java index af66591bed..de2c8d9cea 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java @@ -20,12 +20,11 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.utils.DagHelper; -import java.util.ArrayList; import java.util.List; /** @@ -48,41 +47,8 @@ public class BaseDAGService extends BaseService{ List taskNodeList = processData.getTasks(); - List taskNodeRelations = new ArrayList<>(); + ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); - //Traversing node information and building relationships - for (TaskNode taskNode : taskNodeList) { - String preTasks = taskNode.getPreTasks(); - List preTasksList = JSONUtils.toList(preTasks, String.class); - - //if previous tasks not empty - if (preTasksList != null) { - for (String depNode : preTasksList) { - taskNodeRelations.add(new TaskNodeRelation(depNode, taskNode.getName())); - } - } - } - - ProcessDag processDag = new ProcessDag(); - processDag.setEdges(taskNodeRelations); - processDag.setNodes(taskNodeList); - - - // generate detail Dag, to be executed - DAG dag = new DAG<>(); - - if (CollectionUtils.isNotEmpty(processDag.getNodes())) { - for (TaskNode node : processDag.getNodes()) { - dag.addNode(node.getName(), node); - } - } - - if (CollectionUtils.isNotEmpty(processDag.getEdges())) { - for (TaskNodeRelation edge : processDag.getEdges()) { - dag.addEdge(edge.getStartNode(), edge.getEndNode()); - } - } - - return dag; + return DagHelper.buildDagGraph(processDag); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index e9cfe341dd..38755d6727 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -46,6 +46,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -948,11 +949,16 @@ public class ProcessDefinitionService extends BaseDAGService { return result; } - String processDefinitionJson = processDefinition.getProcessDefinitionJson(); - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + //process data check + if (null == processData) { + logger.error("process data is null"); + putMsg(result,Status.DATA_IS_NOT_VALID, processDefinitionJson); + return result; + } + List taskNodeList = (processData.getTasks() == null) ? new ArrayList<>() : processData.getTasks(); result.put(Constants.DATA_LIST, taskNodeList); @@ -974,14 +980,13 @@ public class ProcessDefinitionService extends BaseDAGService { Map> taskNodeMap = new HashMap<>(); String[] idList = defineIdList.split(","); - List definitionIdList = Arrays.asList(idList); List idIntList = new ArrayList<>(); - for(String definitionId : definitionIdList) { + for(String definitionId : idList) { idIntList.add(Integer.parseInt(definitionId)); } Integer[] idArray = idIntList.toArray(new Integer[idIntList.size()]); List processDefinitionList = processDefineMapper.queryDefinitionListByIdList(idArray); - if (processDefinitionList == null || processDefinitionList.size() ==0) { + if (CollectionUtils.isEmpty(processDefinitionList)) { logger.info("process definition not exists"); putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineIdList); return result; @@ -1031,9 +1036,10 @@ public class ProcessDefinitionService extends BaseDAGService { Map result = new HashMap<>(); ProcessDefinition processDefinition = processDefineMapper.selectById(processId); - if (processDefinition == null) { + if (null == processDefinition) { logger.info("process define not exists"); - throw new RuntimeException("process define not exists"); + putMsg(result,Status.PROCESS_DEFINE_NOT_EXIST, processDefinition); + return result; } DAG dag = genDagGraph(processDefinition); /** @@ -1121,10 +1127,10 @@ public class ProcessDefinitionService extends BaseDAGService { pTreeViewDto.getChildren().add(treeViewDto); } postNodeList = dag.getSubsequentNodes(nodeName); - if (postNodeList != null && postNodeList.size() > 0) { + if (CollectionUtils.isNotEmpty(postNodeList)) { for (String nextNodeName : postNodeList) { List treeViewDtoList = waitingRunningNodeMap.get(nextNodeName); - if (treeViewDtoList != null && treeViewDtoList.size() > 0) { + if (CollectionUtils.isNotEmpty(treeViewDtoList)) { treeViewDtoList.add(treeViewDto); waitingRunningNodeMap.put(nextNodeName, treeViewDtoList); } else { @@ -1136,7 +1142,6 @@ public class ProcessDefinitionService extends BaseDAGService { } runningNodeMap.remove(nodeName); } - if (waitingRunningNodeMap == null || waitingRunningNodeMap.size() == 0) { break; } else { @@ -1161,68 +1166,22 @@ public class ProcessDefinitionService extends BaseDAGService { private DAG genDagGraph(ProcessDefinition processDefinition) throws Exception { String processDefinitionJson = processDefinition.getProcessDefinitionJson(); - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - List taskNodeList = processData.getTasks(); + //check process data + if (null != processData) { + List taskNodeList = processData.getTasks(); + processDefinition.setGlobalParamList(processData.getGlobalParams()); + ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); - processDefinition.setGlobalParamList(processData.getGlobalParams()); - - - List taskNodeRelations = new ArrayList<>(); - - // Traverse node information and build relationships - for (TaskNode taskNode : taskNodeList) { - String preTasks = taskNode.getPreTasks(); - List preTasksList = JSONUtils.toList(preTasks, String.class); - - // If the dependency is not empty - if (preTasksList != null) { - for (String depNode : preTasksList) { - taskNodeRelations.add(new TaskNodeRelation(depNode, taskNode.getName())); - } - } + // Generate concrete Dag to be executed + return DagHelper.buildDagGraph(processDag); } - ProcessDag processDag = new ProcessDag(); - processDag.setEdges(taskNodeRelations); - processDag.setNodes(taskNodeList); - - - // Generate concrete Dag to be executed - return genDagGraph(processDag); - - + return new DAG<>(); } - /** - * Generate the DAG of process - * - * @return DAG - */ - private DAG genDagGraph(ProcessDag processDag) { - DAG dag = new DAG<>(); - - /** - * Add the ndoes - */ - if (CollectionUtils.isNotEmpty(processDag.getNodes())) { - for (TaskNode node : processDag.getNodes()) { - dag.addNode(node.getName(), node); - } - } - - /** - * Add the edges - */ - if (CollectionUtils.isNotEmpty(processDag.getEdges())) { - for (TaskNodeRelation edge : processDag.getEdges()) { - dag.addEdge(edge.getStartNode(), edge.getEndNode()); - } - } - return dag; - } /** diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 51b440b9ca..290f6ae015 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -16,10 +16,8 @@ */ package org.apache.dolphinscheduler.api.service; -import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.core.metadata.IPage; import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.dto.ProcessMeta; import org.apache.dolphinscheduler.api.enums.Status; @@ -82,6 +80,12 @@ public class ProcessDefinitionServiceTest { @Mock private ProcessDao processDao; + @Mock + private ProcessInstanceMapper processInstanceMapper; + + @Mock + private TaskInstanceMapper taskInstanceMapper; + private String sqlDependentJson = "{\"globalParams\":[]," + "\"tasks\":[{\"type\":\"SQL\",\"id\":\"tasks-27297\",\"name\":\"sql\"," + "\"params\":{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from test\"," + @@ -99,6 +103,12 @@ public class ProcessDefinitionServiceTest { "\"timeout\":{\"strategy\":\"\",\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + private String shellJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + + "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + + "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + + "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + @Test public void testQueryProccessDefinitionList() { String projectName = "project_test1"; @@ -149,7 +159,7 @@ public class ProcessDefinitionServiceTest { } @Test - public void testQueryProccessDefinitionById() { + public void testQueryProcessDefinitionById() { String projectName = "project_test1"; Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); @@ -255,7 +265,7 @@ public class ProcessDefinitionServiceTest { "project_test1", 46); Assert.assertEquals(Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR, deleteFail.get(Constants.STATUS)); - //delte success + //delete success Mockito.when(processDefineMapper.deleteById(46)).thenReturn(1); Map deleteSuccess = processDefinitionService.deleteProcessDefinitionById(loginUser, "project_test1", 46); @@ -304,6 +314,155 @@ public class ProcessDefinitionServiceTest { Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failRes.get(Constants.STATUS)); } + @Test + public void testVerifyProcessDefinitionName() { + String projectName = "project_test1"; + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + + Project project = getProject(projectName); + User loginUser = new User(); + loginUser.setId(-1); + loginUser.setUserType(UserType.GENERAL_USER); + + //project check auth fail + Map result = new HashMap<>(5); + putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); + Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Map map = processDefinitionService.verifyProccessDefinitionName(loginUser, + "project_test1", "test_pdf"); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); + + //project check auth success, process not exist + putMsg(result, Status.SUCCESS, projectName); + Mockito.when(processDefineMapper.queryByDefineName(project.getId(),"test_pdf")).thenReturn(null); + Map processNotExistRes = processDefinitionService.verifyProccessDefinitionName(loginUser, + "project_test1", "test_pdf"); + Assert.assertEquals(Status.SUCCESS, processNotExistRes.get(Constants.STATUS)); + + //process exist + Mockito.when(processDefineMapper.queryByDefineName(project.getId(),"test_pdf")).thenReturn(getProcessDefinition()); + Map processExistRes = processDefinitionService.verifyProccessDefinitionName(loginUser, + "project_test1", "test_pdf"); + Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST, processExistRes.get(Constants.STATUS)); + } + + @Test + public void testCheckProcessNodeList() { + + Map dataNotValidRes = processDefinitionService.checkProcessNodeList(null, ""); + Assert.assertEquals(Status.DATA_IS_NOT_VALID, dataNotValidRes.get(Constants.STATUS)); + + //task not empty + String processDefinitionJson = shellJson; + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + assert processData != null; + Map taskEmptyRes = processDefinitionService.checkProcessNodeList(processData, processDefinitionJson); + Assert.assertEquals(Status.SUCCESS, taskEmptyRes.get(Constants.STATUS)); + + //task empty + processData.setTasks(null); + Map taskNotEmptyRes = processDefinitionService.checkProcessNodeList(processData, processDefinitionJson); + Assert.assertEquals(Status.DATA_IS_NULL, taskNotEmptyRes.get(Constants.STATUS)); + + //json abnormal + String abnormalJson = processDefinitionJson.replaceAll("SHELL",""); + processData = JSONUtils.parseObject(abnormalJson, ProcessData.class); + Map abnormalTaskRes = processDefinitionService.checkProcessNodeList(processData, abnormalJson); + Assert.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID, abnormalTaskRes.get(Constants.STATUS)); + } + + @Test + public void testGetTaskNodeListByDefinitionId() throws Exception { + //process definition not exist + Mockito.when(processDefineMapper.selectById(46)).thenReturn(null); + Map processDefinitionNullRes = processDefinitionService.getTaskNodeListByDefinitionId(46); + Assert.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST, processDefinitionNullRes.get(Constants.STATUS)); + + //process data null + ProcessDefinition processDefinition = getProcessDefinition(); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); + Map successRes = processDefinitionService.getTaskNodeListByDefinitionId(46); + Assert.assertEquals(Status.DATA_IS_NOT_VALID, successRes.get(Constants.STATUS)); + + //success + processDefinition.setProcessDefinitionJson(shellJson); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); + Map dataNotValidRes = processDefinitionService.getTaskNodeListByDefinitionId(46); + Assert.assertEquals(Status.SUCCESS, dataNotValidRes.get(Constants.STATUS)); + } + + @Test + public void testGetTaskNodeListByDefinitionIdList() throws Exception { + //process definition not exist + String defineIdList = "46"; + Integer[] idArray = {46}; + Mockito.when(processDefineMapper.queryDefinitionListByIdList(idArray)).thenReturn(null); + Map processNotExistRes = processDefinitionService.getTaskNodeListByDefinitionIdList(defineIdList); + Assert.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST, processNotExistRes.get(Constants.STATUS)); + + //process definition exist + ProcessDefinition processDefinition = getProcessDefinition(); + processDefinition.setProcessDefinitionJson(shellJson); + List processDefinitionList = new ArrayList<>(); + processDefinitionList.add(processDefinition); + Mockito.when(processDefineMapper.queryDefinitionListByIdList(idArray)).thenReturn(processDefinitionList); + Map successRes = processDefinitionService.getTaskNodeListByDefinitionIdList(defineIdList); + Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + } + + @Test + public void testQueryProccessDefinitionAllByProjectId() { + int projectId = 1; + ProcessDefinition processDefinition = getProcessDefinition(); + processDefinition.setProcessDefinitionJson(shellJson); + List processDefinitionList = new ArrayList<>(); + processDefinitionList.add(processDefinition); + Mockito.when(processDefineMapper.queryAllDefinitionList(projectId)).thenReturn(processDefinitionList); + Map successRes = processDefinitionService.queryProccessDefinitionAllByProjectId(projectId); + Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + } + + @Test + public void testViewTree() throws Exception { + //process definition not exist + ProcessDefinition processDefinition = getProcessDefinition(); + processDefinition.setProcessDefinitionJson(shellJson); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(null); + Map processDefinitionNullRes = processDefinitionService.viewTree(46, 10); + Assert.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST, processDefinitionNullRes.get(Constants.STATUS)); + + List processInstanceList = new ArrayList<>(); + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setId(1); + processInstance.setName("test_instance"); + processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setHost("192.168.xx.xx"); + processInstance.setStartTime(new Date()); + processInstance.setEndTime(new Date()); + processInstanceList.add(processInstance); + + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setStartTime(new Date()); + taskInstance.setEndTime(new Date()); + taskInstance.setTaskType("SHELL"); + taskInstance.setId(1); + taskInstance.setName("test_task_instance"); + taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + taskInstance.setHost("192.168.xx.xx"); + + //task instance not exist + Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); + Mockito.when(processInstanceMapper.queryByProcessDefineId(46, 10)).thenReturn(processInstanceList); + Mockito.when(taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), "shell-1")).thenReturn(null); + Map taskNullRes = processDefinitionService.viewTree(46, 10); + Assert.assertEquals(Status.SUCCESS, taskNullRes.get(Constants.STATUS)); + + //task instance exist + Mockito.when(taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), "shell-1")).thenReturn(taskInstance); + Map taskNotNuLLRes = processDefinitionService.viewTree(46, 10); + Assert.assertEquals(Status.SUCCESS, taskNotNuLLRes.get(Constants.STATUS)); + } + /** * add datasource param and dependent when export process * @throws JSONException @@ -334,13 +493,9 @@ public class ProcessDefinitionServiceTest { @Test public void testAddExportTaskNodeSpecialParam() throws JSONException { - String shellJson = "{\"globalParams\":[],\"tasks\":[{\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + String shellData = shellJson; - String resultStr = processDefinitionService.addExportTaskNodeSpecialParam(shellJson); + String resultStr = processDefinitionService.addExportTaskNodeSpecialParam(shellData); JSONAssert.assertEquals(shellJson, resultStr, false); } @@ -610,7 +765,7 @@ public class ProcessDefinitionServiceTest { private ProcessDefinition getProcessDefinition(){ ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setId(46); - processDefinition.setName("testProject"); + processDefinition.setName("test_pdf"); processDefinition.setProjectId(2); processDefinition.setTenantId(1); processDefinition.setDescription(""); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java index ac38ddd2e8..7a4dc655f7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java @@ -319,18 +319,14 @@ public class DagHelper { DAG dag = new DAG<>(); - /** - * add vertex - */ + //add vertex if (CollectionUtils.isNotEmpty(processDag.getNodes())){ for (TaskNode node : processDag.getNodes()){ dag.addNode(node.getName(),node); } } - /** - * add edge - */ + //add edge if (CollectionUtils.isNotEmpty(processDag.getEdges())){ for (TaskNodeRelation edge : processDag.getEdges()){ dag.addEdge(edge.getStartNode(),edge.getEndNode()); @@ -338,4 +334,31 @@ public class DagHelper { } return dag; } + + /** + * get process dag + * @param taskNodeList task node list + * @return Process dag + */ + public static ProcessDag getProcessDag(List taskNodeList) { + List taskNodeRelations = new ArrayList<>(); + + // Traverse node information and build relationships + for (TaskNode taskNode : taskNodeList) { + String preTasks = taskNode.getPreTasks(); + List preTasksList = JSONUtils.toList(preTasks, String.class); + + // If the dependency is not empty + if (preTasksList != null) { + for (String depNode : preTasksList) { + taskNodeRelations.add(new TaskNodeRelation(depNode, taskNode.getName())); + } + } + } + + ProcessDag processDag = new ProcessDag(); + processDag.setEdges(taskNodeRelations); + processDag.setNodes(taskNodeList); + return processDag; + } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java index a1e3f819e3..95c7d2f086 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java @@ -24,6 +24,8 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.junit.Assert; import org.junit.Test; @@ -37,7 +39,6 @@ import java.util.Map; * dag helper test */ public class DagHelperTest { - /** * test task node can submit * @throws JsonProcessingException if error throws JsonProcessingException @@ -131,4 +132,20 @@ public class DagHelperTest { return DagHelper.buildDagGraph(processDag); } + @Test + public void testBuildDagGraph() { + String shellJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + + "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + + "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + + "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + + ProcessData processData = JSONUtils.parseObject(shellJson, ProcessData.class); + assert processData != null; + List taskNodeList = processData.getTasks(); + ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); + DAG dag = DagHelper.buildDagGraph(processDag); + Assert.assertNotNull(dag); + } + } From 77023aee1b999a18afbb073484d0ed0bea8e9806 Mon Sep 17 00:00:00 2001 From: Yeleights Date: Thu, 13 Feb 2020 16:18:26 +0800 Subject: [PATCH 39/43] add DagHelperTest to pom --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 53184e7412..11facc1e5a 100644 --- a/pom.xml +++ b/pom.xml @@ -715,6 +715,7 @@ **/dao/mapper/AlertMapperTest.java **/dao/mapper/CommandMapperTest.java **/dao/cron/CronUtilsTest.java + **/dao/utils/DagHelperTest.java **/alert/template/AlertTemplateFactoryTest.java **/alert/template/impl/DefaultHTMLTemplateTest.java **/server/worker/task/datax/DataxTaskTest.java From cdd53a7673e00c01ce0879e7e36e4747f9d29269 Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Fri, 14 Feb 2020 10:39:02 +0800 Subject: [PATCH 40/43] Fix socket bug: read before sock connect. Add UT for FourLetterWordMain. (#1954) * Fix socket bug: read before sock connect. Add UT for FourLetterWordMain. * Fix code smell Remove @Ignore and @Test from testCmd(). * correct comment --- .../api/utils/FourLetterWordMain.java | 31 ++- .../api/utils/FourLetterWordMainTest.java | 218 ++++++++++++++++++ pom.xml | 1 + 3 files changed, 238 insertions(+), 12 deletions(-) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java index b04e773aea..1f523edac5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.api.utils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,13 +28,17 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; import java.net.SocketTimeoutException; - +import java.util.Objects; public class FourLetterWordMain { private static final int DEFAULT_SOCKET_TIMEOUT = 5000; protected static final Logger LOG = LoggerFactory.getLogger(FourLetterWordMain.class); + private FourLetterWordMain() { + throw new IllegalStateException("FourLetterWordMain class"); + } + /** * Send the 4letterword * @param host the destination host @@ -48,6 +51,7 @@ public class FourLetterWordMain { throws IOException { return send4LetterWord(host, port, cmd, DEFAULT_SOCKET_TIMEOUT); } + /** * Send the 4letterword * @param host the destination host @@ -59,30 +63,33 @@ public class FourLetterWordMain { */ public static String send4LetterWord(String host, int port, String cmd, int timeout) throws IOException { + Objects.requireNonNull(cmd, "cmd must not be null"); LOG.info("connecting to {} {}", host, port); InetSocketAddress hostaddress= host != null ? new InetSocketAddress(host, port) : new InetSocketAddress(InetAddress.getByName(null), port); - try (Socket sock = new Socket(); - OutputStream outstream = sock.getOutputStream(); - BufferedReader reader = - new BufferedReader( - new InputStreamReader(sock.getInputStream()))) { + try (Socket sock = new Socket()) { sock.setSoTimeout(timeout); sock.connect(hostaddress, timeout); + OutputStream outstream = sock.getOutputStream(); outstream.write(cmd.getBytes()); outstream.flush(); // this replicates NC - close the output stream before reading sock.shutdownOutput(); - StringBuilder sb = new StringBuilder(); - String line; - while((line = reader.readLine()) != null) { - sb.append(line + "\n"); + try (BufferedReader reader = + new BufferedReader( + new InputStreamReader(sock.getInputStream()))) { + StringBuilder sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line + "\n"); + } + return sb.toString(); } - return sb.toString(); } catch (SocketTimeoutException e) { throw new IOException("Exception while executing four letter word: " + cmd, e); } } + } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java new file mode 100644 index 0000000000..e8adc6ca9c --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.utils; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketTimeoutException; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({FourLetterWordMain.class, Socket.class}) +public class FourLetterWordMainTest { + + private static final Logger logger = + LoggerFactory.getLogger(FourLetterWordMainTest.class); + private static final String NEW_LINE = "\n"; + + @InjectMocks + private FourLetterWordMain fourLetterWord; + @Mock + private Socket socket; + @Mock + private InetSocketAddress socketAddress; + + private final String localHost = "127.0.0.1"; + private final int zkPort = 2181; + private ByteArrayOutputStream byteArrayOutputStream; + private InputStream inputStream; + + private String cmd; + private String testResult; + private String expectedStr; + + @Before + public void setUp() { + // mock socket class + PowerMockito.mockStatic(Socket.class); + try { + PowerMockito.whenNew(Socket.class).withNoArguments() + .thenReturn(socket); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * None mock test method, just to check zookeeper status. + * Comment @Before notation to run this test. + * Zookeeper status will be as: + * Zookeeper version: 3.4.11 ... + * Received: 6739707 + * Sent: 6739773 + * Connections: 20 + * Outstanding: 0 + * Zxid: 0x9ba + * Mode: standalone + * Node count: 263 + */ + public void testCmd() { + // "192.168.64.11" + // final String zkHost = localHost; + final String zkHost = "192.168.64.11"; + cmd = "srvr"; + try { + // Change localhost to right zk host ip. + final String result = FourLetterWordMain + .send4LetterWord(zkHost, zkPort, cmd); + logger.info(cmd + ": " + result + "<<<"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void testEmptyCmd() { + cmd = ""; + expectedStr = ""; + testSend4LetterWord(cmd, expectedStr); + } + + @Test + public void testNullCmd() { + cmd = null; + + try { + testResult = FourLetterWordMain + .send4LetterWord(localHost, zkPort, cmd); + } catch (Exception e) { + testResult = e.getMessage(); + } + + logger.info("testNullCmd result: " + testResult); + assertEquals("cmd must not be null", testResult); + } + + @Test + public void testNullSocketOutput() { + cmd = "test null socket output"; + expectedStr = null; + testSend4LetterWord(cmd, expectedStr); + } + + @Test + public void testOneLineOutput() { + cmd = "line 1"; + + // line end without \n + expectedStr = "line 1" + NEW_LINE; + testSend4LetterWord(cmd, expectedStr); + + // line end with \n + expectedStr = "line 1\n" + NEW_LINE; + testSend4LetterWord(cmd, expectedStr); + } + + @Test + public void testMultiline() { + cmd = "line 1 " + NEW_LINE + + "line 2 " + NEW_LINE + + "line 3 " + NEW_LINE; + + expectedStr = cmd + NEW_LINE; + testSend4LetterWord(cmd, expectedStr); + + expectedStr = NEW_LINE + NEW_LINE + NEW_LINE; + testSend4LetterWord(cmd, expectedStr); + } + + @Test + public void testSocketTimeOut() { + cmd = "test socket time out"; + + try { + doThrow(new SocketTimeoutException()) + .when(socket) + .connect(any(InetSocketAddress.class), Mockito.anyInt()); + testResult = FourLetterWordMain + .send4LetterWord(localHost, zkPort, cmd); + } catch (Exception e) { + testResult = e.getMessage(); + } + + logger.info("testSocketTimeOut result: " + testResult); + assertEquals( + "Exception while executing four letter word: " + cmd, + testResult + ); + } + + /** + * Test FourLetterWordMain.send4LetterWord() with input cmd and output + * string. + * @param cmd + * @param expectedStr + */ + public void testSend4LetterWord(String cmd, String expectedStr) { + try { + final byte[] strBytes = cmd.getBytes(); + byteArrayOutputStream = new ByteArrayOutputStream(strBytes.length); + byteArrayOutputStream.write(strBytes, 0, strBytes.length); + + inputStream = new ByteArrayInputStream(expectedStr.getBytes()); + + when(socket.getOutputStream()) + .thenReturn(byteArrayOutputStream); + when(socket.getInputStream()).thenReturn(inputStream); + + final String result = FourLetterWordMain + .send4LetterWord(localHost, zkPort, cmd); + logger.info( + "testSend4LetterWord: " + + "cmd: " + cmd + + ", expectedStr: " + expectedStr + + ", result: " + result + "." + ); + Assert.assertEquals(expectedStr, result); + } catch (Exception e) { + e.printStackTrace(); + } + } + +} diff --git a/pom.xml b/pom.xml index 53184e7412..de7ee81403 100644 --- a/pom.xml +++ b/pom.xml @@ -682,6 +682,7 @@ **/common/queue/*.java **/api/utils/CheckUtilsTest.java **/api/utils/FileUtilsTest.java + **/api/utils/FourLetterWordTest.java **/api/utils/exportprocess/DataSourceParamTest.java **/api/utils/exportprocess/DependentParamTest.java **/api/enums/*.java From c1ee1333924eff051fc2408480cbc10898e4021a Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Fri, 14 Feb 2020 22:00:20 +0800 Subject: [PATCH 41/43] Refactor Architecture Basic modification #1658 (#1946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 1，remove dolphinscheduler-rpc module 2，add dolphinscheduler-remote module 3，add dolphinscheduler-service module 4，refactor LoggerServer module (#1925) * 1，remove dolphinscheduler-rpc module 2，add dolphinscheduler-remote module 3，add dolphinscheduler-service module 4，refactor LoggerServer module * ProcessUtils modify * Refactor architecture (#1926) * move version to parent pom * move version properties to parent pom for easy management * remove freemarker dependency * delete CombinedApplicationServer * #1871 correct spelling * #1873 some updates for TaskQueueZkImpl * #1875 remove unused properties in pom * #1878 1. remove tomcat dependency 2. remove combined_logback.xml in api module 3. format pom.xml for not aligning * #1885 fix api server startup failure 1. add jsp-2.1 dependency 2. remove jasper-runtime dependency * add stringutils ut (#1921) * add stringutils ut * Newfeature for #1675. (#1908) Continue to finish the rest works, add the cache feature for dependence,mr,python,sub_process,procedure and shell. * Add modify user name for process definition (#1919) * class overrides equals() and should therefore also override hashCode() * #1862 add modify user in process difinition list * #1862 add pg-1.2.2 ddl.sql * modify ScriptRunnerTest * add updateProessDifinition UT * modify updateProcessDifinition UT * modify updateProcessDifinition UT * modify mysql 1.2.2 ddl.sql&dml.sql * add scope test to mysql in pom * modify pg-1.2.2 ddl.sql * refactor module * updates Co-authored-by: khadgarmage Co-authored-by: zhukai Co-authored-by: Yelli * dolphinscheduler-common remove spring (#1931) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * SpringApplicationContext class title add license (#1932) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * add license (#1934) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * Refactor architecture (#1936) * move datasource classes to dao module * fix send4LetterWord bug * LoggerServiceTest remove ProcessDao (#1944) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * LoggerServiceTest remove ProcessDao * exclude jasper-compiler in case of runtime conflict (#1938) * move datasource classes to dao module * fix send4LetterWord bug * exclude jasper-compiler in case of runtime conflict * DataAnaylysisServiceTest and ProcessDefinitionService modify * remote module add comment * OSUtilsTest modify * add finally block to close channel (#1951) * move datasource classes to dao module * fix send4LetterWord bug * exclude jasper-compiler in case of runtime conflict * add finally block to close channel Co-authored-by: Tboy Co-authored-by: khadgarmage Co-authored-by: zhukai Co-authored-by: Yelli --- dolphinscheduler-api/pom.xml | 14 +- .../controller/ProcessInstanceController.java | 4 +- .../dolphinscheduler/api/log/LogClient.java | 137 --------- .../api/service/DataAnalysisService.java | 10 +- .../api/service/DataSourceService.java | 3 +- .../api/service/ExecutorService.java | 34 +-- .../api/service/LoggerService.java | 39 ++- .../api/service/ProcessDefinitionService.java | 8 +- .../api/service/ProcessInstanceService.java | 44 +-- .../api/service/SchedulerService.java | 22 +- .../api/service/TaskInstanceService.java | 4 +- .../api/utils/FourLetterWordMain.java | 2 +- .../api/utils/ZookeeperMonitor.java | 4 +- .../api/service/DataAnalysisServiceTest.java | 8 +- .../api/service/ExecutorService2Test.java | 28 +- .../api/service/LoggerServiceTest.java | 10 +- .../service/ProcessDefinitionServiceTest.java | 6 +- dolphinscheduler-common/pom.xml | 25 +- .../common/utils/Preconditions.java | 23 +- .../common/utils/OSUtilsTest.java | 1 - dolphinscheduler-dao/pom.xml | 15 - .../dao/datasource}/BaseDataSource.java | 2 +- .../dao/datasource}/ClickHouseDataSource.java | 2 +- .../dao/datasource}/DB2ServerDataSource.java | 2 +- .../dao/datasource}/DataSourceFactory.java | 4 +- .../dao/datasource}/HiveDataSource.java | 6 +- .../dao/datasource}/MySQLDataSource.java | 2 +- .../dao/datasource}/OracleDataSource.java | 2 +- .../dao/datasource}/PostgreDataSource.java | 2 +- .../dao/datasource}/SQLServerDataSource.java | 2 +- .../dao/datasource}/SparkDataSource.java | 2 +- dolphinscheduler-remote/pom.xml | 44 +++ .../remote/NettyRemotingClient.java | 272 ++++++++++++++++++ .../remote/NettyRemotingServer.java | 220 ++++++++++++++ .../remote/codec/NettyDecoder.java | 109 +++++++ .../remote/codec/NettyEncoder.java | 52 ++++ .../remote/command/Command.java | 105 +++++++ .../remote/command/CommandHeader.java | 64 +++++ .../remote/command/CommandType.java | 1 + .../command/ExecuteTaskRequestCommand.java | 1 + .../command/ExecuteTaskResponseCommand.java | 1 + .../dolphinscheduler/remote/command/Ping.java | 74 +++++ .../dolphinscheduler/remote/command/Pong.java | 75 +++++ .../log/GetLogBytesRequestCommand.java | 69 +++++ .../log/GetLogBytesResponseCommand.java | 65 +++++ .../log/RollViewLogRequestCommand.java | 97 +++++++ .../log/RollViewLogResponseCommand.java | 64 +++++ .../command/log/ViewLogRequestCommand.java | 66 +++++ .../command/log/ViewLogResponseCommand.java | 64 +++++ .../remote/config/NettyClientConfig.java | 91 ++++++ .../remote/config/NettyServerConfig.java | 116 ++++++++ .../remote/exceptions/RemotingException.java | 94 ++++++ .../remote/handler/NettyClientHandler.java | 174 +++++++++++ .../remote/handler/NettyServerHandler.java | 173 +++++++++++ .../processor/NettyRequestProcessor.java | 32 +-- .../remote/utils/Address.java | 96 +++++++ .../remote/utils/ChannelUtils.java | 57 ++++ .../remote/utils/Constants.java | 42 ++- .../remote/utils/FastJsonSerializer.java | 60 ++++ .../dolphinscheduler/remote/utils/Pair.java | 53 ++++ .../remote/NettyRemotingClientTest.java | 77 +++++ dolphinscheduler-rpc/pom.xml | 113 -------- .../src/main/proto/scheduler.proto | 101 ------- dolphinscheduler-server/pom.xml | 2 +- .../server/log/LoggerRequestProcessor.java | 179 ++++++++++++ .../server/log/LoggerServer.java | 91 ++++++ .../server/master/MasterServer.java | 27 +- .../runner/MasterBaseTaskExecThread.java | 18 +- .../master/runner/MasterExecThread.java | 55 ++-- .../master/runner/MasterSchedulerThread.java | 22 +- .../master/runner/MasterTaskExecThread.java | 10 +- .../runner/SubProcessTaskExecThread.java | 16 +- .../server/monitor/ZKMonitorImpl.java | 2 +- .../server/rpc/LogClient.java | 149 ---------- .../server/rpc/LoggerServer.java | 238 --------------- .../server/utils/ProcessUtils.java | 15 +- .../server/utils/RemoveZKNode.java | 2 +- .../server/worker/WorkerServer.java | 36 +-- .../server/worker/runner/FetchTaskThread.java | 29 +- .../worker/runner/TaskScheduleThread.java | 26 +- .../worker/task/AbstractCommandExecutor.java | 20 +- .../server/worker/task/AbstractYarnTask.java | 12 +- .../server/worker/task/datax/DataxTask.java | 22 +- .../task/dependent/DependentExecute.java | 16 +- .../worker/task/dependent/DependentTask.java | 12 +- .../server/worker/task/flink/FlinkTask.java | 2 +- .../server/worker/task/http/HttpTask.java | 12 +- .../task/processdure/ProcedureTask.java | 16 +- .../server/worker/task/python/PythonTask.java | 12 +- .../server/worker/task/shell/ShellTask.java | 10 +- .../server/worker/task/sql/SqlTask.java | 29 +- .../server/zk/ZKMasterClient.java | 18 +- .../server/zk/ZKWorkerClient.java | 2 +- .../server/master/MasterExecThreadTest.java | 18 +- .../shell/ShellCommandExecutorTest.java | 10 +- .../server/worker/sql/SqlExecutorTest.java | 10 +- .../worker/task/datax/DataxTaskTest.java | 22 +- dolphinscheduler-service/pom.xml | 56 ++++ .../bean}/SpringApplicationContext.java | 3 +- .../service/log/LogClientService.java | 166 +++++++++++ .../service/log/LogPromise.java | 81 ++++++ .../service}/permission/PermissionCheck.java | 36 +-- .../service/process/ProcessService.java | 14 +- .../quartz/DruidConnectionProvider.java | 3 +- .../service}/quartz/ProcessScheduleJob.java | 27 +- .../service}/quartz/QuartzExecutors.java | 4 +- .../service/quartz}/cron/AbstractCycle.java | 4 +- .../service/quartz}/cron/CronUtils.java | 10 +- .../service/quartz}/cron/CycleFactory.java | 4 +- .../service/quartz}/cron/CycleLinks.java | 4 +- .../service}/queue/ITaskQueue.java | 2 +- .../service}/queue/TaskQueueFactory.java | 6 +- .../service}/queue/TaskQueueZkImpl.java | 4 +- .../service}/zk/AbstractZKClient.java | 5 +- .../service}/zk/DefaultEnsembleProvider.java | 2 +- .../service}/zk/ZookeeperCachedOperator.java | 7 +- .../service}/zk/ZookeeperConfig.java | 2 +- .../service}/zk/ZookeeperOperator.java | 4 +- .../src/main/resources/quartz.properties | 2 +- .../src/test/java}/cron/CronUtilsTest.java | 8 +- .../test/java}/queue/BaseTaskQueueTest.java | 5 +- .../test/java}/queue/TaskQueueZKImplTest.java | 2 +- .../src/test/java/queue}/ZKServer.java | 2 +- .../test/java}/utils/PreconditionsTest.java | 5 +- pom.xml | 13 +- 125 files changed, 3534 insertions(+), 1296 deletions(-) delete mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/log/LogClient.java rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/BaseDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/ClickHouseDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/DB2ServerDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/DataSourceFactory.java (98%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/HiveDataSource.java (93%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/MySQLDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/OracleDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/PostgreDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/SQLServerDataSource.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db => dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource}/SparkDataSource.java (97%) create mode 100644 dolphinscheduler-remote/pom.xml create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandHeader.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java rename dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZkServer.java => dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java (65%) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Address.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java rename dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZk.java => dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java (66%) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java create mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java delete mode 100644 dolphinscheduler-rpc/pom.xml delete mode 100644 dolphinscheduler-rpc/src/main/proto/scheduler.proto create mode 100644 dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java create mode 100644 dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java delete mode 100644 dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java delete mode 100644 dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LoggerServer.java create mode 100644 dolphinscheduler-service/pom.xml rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean}/SpringApplicationContext.java (96%) create mode 100644 dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java create mode 100644 dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java rename {dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/permission/PermissionCheck.java (80%) rename dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java (99%) rename {dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/quartz/DruidConnectionProvider.java (99%) rename {dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/quartz/ProcessScheduleJob.java (83%) rename {dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/quartz/QuartzExecutors.java (99%) rename {dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz}/cron/AbstractCycle.java (99%) rename {dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz}/cron/CronUtils.java (98%) rename {dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz}/cron/CycleFactory.java (99%) rename {dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz}/cron/CycleLinks.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/queue/ITaskQueue.java (97%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/queue/TaskQueueFactory.java (93%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/queue/TaskQueueZkImpl.java (99%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/zk/AbstractZKClient.java (99%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/zk/DefaultEnsembleProvider.java (96%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/zk/ZookeeperCachedOperator.java (90%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/zk/ZookeeperConfig.java (98%) rename {dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service}/zk/ZookeeperOperator.java (98%) rename {dolphinscheduler-common => dolphinscheduler-service}/src/main/resources/quartz.properties (96%) rename {dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao => dolphinscheduler-service/src/test/java}/cron/CronUtilsTest.java (98%) rename {dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/test/java}/queue/BaseTaskQueueTest.java (90%) rename {dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/test/java}/queue/TaskQueueZKImplTest.java (99%) rename {dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk => dolphinscheduler-service/src/test/java/queue}/ZKServer.java (99%) rename {dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common => dolphinscheduler-service/src/test/java}/utils/PreconditionsTest.java (97%) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index c10f443384..6440805af7 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -31,12 +31,6 @@ org.apache.dolphinscheduler dolphinscheduler-alert - - - org.apache.dolphinscheduler - dolphinscheduler-dao - - @@ -129,13 +123,13 @@ - com.github.xiaoymin - swagger-bootstrap-ui + org.apache.dolphinscheduler + dolphinscheduler-service - org.apache.dolphinscheduler - dolphinscheduler-rpc + com.github.xiaoymin + swagger-bootstrap-ui diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index 150c647f99..80db6c86af 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -22,12 +22,12 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.common.queue.ITaskQueue; -import org.apache.dolphinscheduler.common.queue.TaskQueueFactory; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.User; import io.swagger.annotations.*; +import org.apache.dolphinscheduler.service.queue.ITaskQueue; +import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/log/LogClient.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/log/LogClient.java deleted file mode 100644 index 3452060ec9..0000000000 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/log/LogClient.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.log; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import org.apache.dolphinscheduler.rpc.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; - -/** - * log client - */ -public class LogClient { - - private static final Logger logger = LoggerFactory.getLogger(LogClient.class); - - private final ManagedChannel channel; - private final LogViewServiceGrpc.LogViewServiceBlockingStub blockingStub; - - /** - * construct client connecting to HelloWorld server at {@code host:port} - * - * @param host host - * @param port port - */ - public LogClient(String host, int port) { - this(ManagedChannelBuilder.forAddress(host, port) - // Channels are secure by default (via SSL/TLS). For the example we disable TLS to avoid - // needing certificates. - .usePlaintext(true)); - } - - /** - * construct client for accessing RouteGuide server using the existing channel - * - */ - LogClient(ManagedChannelBuilder channelBuilder) { - /** - * set max read size - */ - channelBuilder.maxInboundMessageSize(Integer.MAX_VALUE); - channel = channelBuilder.build(); - blockingStub = LogViewServiceGrpc.newBlockingStub(channel); - } - - /** - * shutdown - * - * @throws InterruptedException InterruptedException - */ - public void shutdown() throws InterruptedException { - channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); - } - - /** - * roll view log - * - * @param path path - * @param skipLineNum skip line number - * @param limit limit - * @return log content - */ - public String rollViewLog(String path,int skipLineNum,int limit) { - logger.info("roll view log : path {},skipLineNum {} ,limit {}", path, skipLineNum, limit); - LogParameter pathParameter = LogParameter - .newBuilder() - .setPath(path) - .setSkipLineNum(skipLineNum) - .setLimit(limit) - .build(); - RetStrInfo retStrInfo; - try { - retStrInfo = blockingStub.rollViewLog(pathParameter); - return retStrInfo.getMsg(); - } catch (StatusRuntimeException e) { - logger.error("roll view log error", e); - return null; - } - } - - /** - * view log - * - * @param path path - * @return log content - */ - public String viewLog(String path) { - logger.info("view log path {}",path); - PathParameter pathParameter = PathParameter.newBuilder().setPath(path).build(); - RetStrInfo retStrInfo; - try { - retStrInfo = blockingStub.viewLog(pathParameter); - return retStrInfo.getMsg(); - } catch (StatusRuntimeException e) { - logger.error("view log error", e); - return null; - } - } - - /** - * get log size - * - * @param path log path - * @return log content bytes - */ - public byte[] getLogBytes(String path) { - logger.info("log path {}",path); - PathParameter pathParameter = PathParameter.newBuilder().setPath(path).build(); - RetByteInfo retByteInfo; - try { - retByteInfo = blockingStub.getLogBytes(pathParameter); - return retByteInfo.getData().toByteArray(); - } catch (StatusRuntimeException e) { - logger.error("log size error", e); - return null; - } - } - -} \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java index b95782711f..bafe833fab 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java @@ -24,13 +24,13 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.queue.ITaskQueue; -import org.apache.dolphinscheduler.common.queue.TaskQueueFactory; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.queue.ITaskQueue; +import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -69,7 +69,7 @@ public class DataAnalysisService extends BaseService{ TaskInstanceMapper taskInstanceMapper; @Autowired - ProcessDao processDao; + ProcessService processService; /** * statistical task instance status data @@ -296,7 +296,7 @@ public class DataAnalysisService extends BaseService{ if(projectId !=0){ projectIds.add(projectId); }else if(loginUser.getUserType() == UserType.GENERAL_USER){ - projectIds = processDao.getProjectIdListHavePerm(loginUser.getId()); + projectIds = processService.getProjectIdListHavePerm(loginUser.getId()); if(projectIds.size() ==0 ){ projectIds.add(0); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 5d33b46bd2..f6d8903dd8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -21,10 +21,9 @@ import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.job.db.*; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.datasource.*; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.User; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 257f15d580..6edd48d499 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -25,12 +25,12 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; -import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -67,7 +67,7 @@ public class ExecutorService extends BaseService{ @Autowired - private ProcessDao processDao; + private ProcessService processService; /** * execute process instance @@ -186,13 +186,13 @@ public class ExecutorService extends BaseService{ return checkResult; } - ProcessInstance processInstance = processDao.findProcessInstanceDetailById(processInstanceId); + ProcessInstance processInstance = processService.findProcessInstanceDetailById(processInstanceId); if (processInstance == null) { putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } - ProcessDefinition processDefinition = processDao.findProcessDefineById(processInstance.getProcessDefinitionId()); + ProcessDefinition processDefinition = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); if(executeType != ExecuteType.STOP && executeType != ExecuteType.PAUSE){ result = checkProcessDefinitionValid(processDefinition, processInstance.getProcessDefinitionId()); if (result.get(Constants.STATUS) != Status.SUCCESS) { @@ -227,7 +227,7 @@ public class ExecutorService extends BaseService{ } else { processInstance.setCommandType(CommandType.STOP); processInstance.addHistoryCmd(CommandType.STOP); - processDao.updateProcessInstance(processInstance); + processService.updateProcessInstance(processInstance); result = updateProcessInstanceState(processInstanceId, ExecutionStatus.READY_STOP); } break; @@ -237,7 +237,7 @@ public class ExecutorService extends BaseService{ } else { processInstance.setCommandType(CommandType.PAUSE); processInstance.addHistoryCmd(CommandType.PAUSE); - processDao.updateProcessInstance(processInstance); + processService.updateProcessInstance(processInstance); result = updateProcessInstanceState(processInstanceId, ExecutionStatus.READY_PAUSE); } break; @@ -257,7 +257,7 @@ public class ExecutorService extends BaseService{ */ private boolean checkTenantSuitable(ProcessDefinition processDefinition) { // checkTenantExists(); - Tenant tenant = processDao.getTenantForProcess(processDefinition.getTenantId(), + Tenant tenant = processService.getTenantForProcess(processDefinition.getTenantId(), processDefinition.getUserId()); if(tenant == null){ return false; @@ -319,7 +319,7 @@ public class ExecutorService extends BaseService{ private Map updateProcessInstanceState(Integer processInstanceId, ExecutionStatus executionStatus) { Map result = new HashMap<>(5); - int update = processDao.updateProcessInstanceState(processInstanceId, executionStatus); + int update = processService.updateProcessInstanceState(processInstanceId, executionStatus); if (update > 0) { putMsg(result, Status.SUCCESS); } else { @@ -347,12 +347,12 @@ public class ExecutorService extends BaseService{ CMDPARAM_RECOVER_PROCESS_ID_STRING, instanceId)); command.setExecutorId(loginUser.getId()); - if(!processDao.verifyIsNeedCreateCommand(command)){ + if(!processService.verifyIsNeedCreateCommand(command)){ putMsg(result, Status.PROCESS_INSTANCE_EXECUTING_COMMAND,processDefinitionId); return result; } - int create = processDao.createCommand(command); + int create = processService.createCommand(command); if (create > 0) { putMsg(result, Status.SUCCESS); @@ -376,7 +376,7 @@ public class ExecutorService extends BaseService{ putMsg(result,Status.REQUEST_PARAMS_NOT_VALID_ERROR,"process definition id"); } List ids = new ArrayList<>(); - processDao.recurseFindSubProcessId(processDefineId, ids); + processService.recurseFindSubProcessId(processDefineId, ids); Integer[] idArray = ids.toArray(new Integer[ids.size()]); if (ids.size() > 0){ List processDefinitionList; @@ -506,9 +506,9 @@ public class ExecutorService extends BaseService{ cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(start)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(end)); command.setCommandParam(JSONUtils.toJson(cmdParam)); - return processDao.createCommand(command); + return processService.createCommand(command); }else if (runMode == RunMode.RUN_MODE_PARALLEL){ - List schedules = processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefineId); + List schedules = processService.queryReleaseSchedulerListByProcessDefinitionId(processDefineId); List listDate = new LinkedList<>(); if(!CollectionUtils.isEmpty(schedules)){ for (Schedule item : schedules) { @@ -521,7 +521,7 @@ public class ExecutorService extends BaseService{ cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(date)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(date)); command.setCommandParam(JSONUtils.toJson(cmdParam)); - processDao.createCommand(command); + processService.createCommand(command); } return listDate.size(); }else{ @@ -532,7 +532,7 @@ public class ExecutorService extends BaseService{ cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(start)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(start)); command.setCommandParam(JSONUtils.toJson(cmdParam)); - processDao.createCommand(command); + processService.createCommand(command); start = DateUtils.getSomeDay(start, 1); } return runCunt; @@ -544,7 +544,7 @@ public class ExecutorService extends BaseService{ } }else{ command.setCommandParam(JSONUtils.toJson(cmdParam)); - return processDao.createCommand(command); + return processService.createCommand(command); } return 0; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java index 2587290fd3..bff54b6c21 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.log.LogClient; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.service.log.LogClientService; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -37,7 +37,7 @@ public class LoggerService { private static final Logger logger = LoggerFactory.getLogger(LoggerService.class); @Autowired - private ProcessDao processDao; + private ProcessService processService; /** * view log @@ -49,7 +49,7 @@ public class LoggerService { */ public Result queryLog(int taskInstId, int skipLineNum, int limit) { - TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId); + TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); if (taskInstance == null){ return new Result(Status.TASK_INSTANCE_NOT_FOUND.getCode(), Status.TASK_INSTANCE_NOT_FOUND.getMsg()); @@ -64,11 +64,17 @@ public class LoggerService { Result result = new Result(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); logger.info("log host : {} , logPath : {} , logServer port : {}",host,taskInstance.getLogPath(),Constants.RPC_PORT); - - LogClient logClient = new LogClient(host, Constants.RPC_PORT); - String log = logClient.rollViewLog(taskInstance.getLogPath(),skipLineNum,limit); - result.setData(log); - logger.info(log); + LogClientService logClient = null; + try { + logClient = new LogClientService(host, Constants.RPC_PORT); + String log = logClient.rollViewLog(taskInstance.getLogPath(),skipLineNum,limit); + result.setData(log); + logger.info(log); + } finally { + if(logClient != null){ + logClient.close(); + } + } return result; } @@ -80,17 +86,20 @@ public class LoggerService { * @return log byte array */ public byte[] getLogBytes(int taskInstId) { - TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId); + TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); if (taskInstance == null){ throw new RuntimeException("task instance is null"); } String host = taskInstance.getHost(); - if(StringUtils.isEmpty(host)){ - throw new RuntimeException("task instance host is null"); + LogClientService logClient = null; + try { + logClient = new LogClientService(host, Constants.RPC_PORT); + return logClient.getLogBytes(taskInstance.getLogPath()); + } finally { + if(logClient != null){ + logClient.close(); + } } - - LogClient logClient = new LogClient(host, Constants.RPC_PORT); - return logClient.getLogBytes(taskInstance.getLogPath()); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index e9cfe341dd..123dc8186a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -43,9 +43,9 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -94,7 +94,7 @@ public class ProcessDefinitionService extends BaseDAGService { private ScheduleMapper scheduleMapper; @Autowired - private ProcessDao processDao; + private ProcessService processService; @Autowired private WorkerGroupMapper workerGroupMapper; @@ -283,7 +283,7 @@ public class ProcessDefinitionService extends BaseDAGService { if ((checkProcessJson.get(Constants.STATUS) != Status.SUCCESS)) { return checkProcessJson; } - ProcessDefinition processDefinition = processDao.findProcessDefineById(id); + ProcessDefinition processDefinition = processService.findProcessDefineById(id); if (processDefinition == null) { // check process definition exists putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, id); @@ -296,7 +296,7 @@ public class ProcessDefinitionService extends BaseDAGService { putMsg(result, Status.SUCCESS); } - ProcessDefinition processDefine = processDao.findProcessDefineById(id); + ProcessDefinition processDefine = processService.findProcessDefineById(id); Date now = new Date(); processDefine.setId(id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 87e1a0ede1..2b1f04e6ce 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -30,15 +30,15 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.Property; -import org.apache.dolphinscheduler.common.queue.ITaskQueue; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import com.alibaba.fastjson.JSON; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -72,7 +72,7 @@ public class ProcessInstanceService extends BaseDAGService { ProjectService projectService; @Autowired - ProcessDao processDao; + ProcessService processService; @Autowired ProcessInstanceMapper processInstanceMapper; @@ -112,7 +112,7 @@ public class ProcessInstanceService extends BaseDAGService { if (resultEnum != Status.SUCCESS) { return checkResult; } - ProcessInstance processInstance = processDao.findProcessInstanceDetailById(processId); + ProcessInstance processInstance = processService.findProcessInstanceDetailById(processId); String workerGroupName = ""; if(processInstance.getWorkerGroupId() == -1){ workerGroupName = DEFAULT; @@ -125,7 +125,7 @@ public class ProcessInstanceService extends BaseDAGService { } } processInstance.setWorkerGroupName(workerGroupName); - ProcessDefinition processDefinition = processDao.findProcessDefineById(processInstance.getProcessDefinitionId()); + ProcessDefinition processDefinition = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); processInstance.setReceivers(processDefinition.getReceivers()); processInstance.setReceiversCc(processDefinition.getReceiversCc()); result.put(Constants.DATA_LIST, processInstance); @@ -228,8 +228,8 @@ public class ProcessInstanceService extends BaseDAGService { if (resultEnum != Status.SUCCESS) { return checkResult; } - ProcessInstance processInstance = processDao.findProcessInstanceDetailById(processId); - List taskInstanceList = processDao.findValidTaskListByProcessId(processId); + ProcessInstance processInstance = processService.findProcessInstanceDetailById(processId); + List taskInstanceList = processService.findValidTaskListByProcessId(processId); AddDependResultForTaskList(taskInstanceList); Map resultMap = new HashMap<>(); resultMap.put(PROCESS_INSTANCE_STATE, processInstance.getState().toString()); @@ -304,7 +304,7 @@ public class ProcessInstanceService extends BaseDAGService { return checkResult; } - TaskInstance taskInstance = processDao.findTaskInstanceById(taskId); + TaskInstance taskInstance = processService.findTaskInstanceById(taskId); if (taskInstance == null) { putMsg(result, Status.TASK_INSTANCE_NOT_EXISTS, taskId); return result; @@ -314,7 +314,7 @@ public class ProcessInstanceService extends BaseDAGService { return result; } - ProcessInstance subWorkflowInstance = processDao.findSubProcessInstance( + ProcessInstance subWorkflowInstance = processService.findSubProcessInstance( taskInstance.getProcessInstanceId(), taskInstance.getId()); if (subWorkflowInstance == null) { putMsg(result, Status.SUB_PROCESS_INSTANCE_NOT_EXIST, taskId); @@ -356,7 +356,7 @@ public class ProcessInstanceService extends BaseDAGService { } //check process instance exists - ProcessInstance processInstance = processDao.findProcessInstanceDetailById(processInstanceId); + ProcessInstance processInstance = processService.findProcessInstanceDetailById(processInstanceId); if (processInstance == null) { putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; @@ -380,7 +380,7 @@ public class ProcessInstanceService extends BaseDAGService { String globalParams = null; String originDefParams = null; int timeout = processInstance.getTimeout(); - ProcessDefinition processDefinition = processDao.findProcessDefineById(processInstance.getProcessDefinitionId()); + ProcessDefinition processDefinition = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); if (StringUtils.isNotEmpty(processInstanceJson)) { ProcessData processData = JSONUtils.parseObject(processInstanceJson, ProcessData.class); //check workflow json is valid @@ -396,7 +396,7 @@ public class ProcessInstanceService extends BaseDAGService { processInstance.getCmdTypeIfComplement(), schedule); timeout = processData.getTimeout(); processInstance.setTimeout(timeout); - Tenant tenant = processDao.getTenantForProcess(processData.getTenantId(), + Tenant tenant = processService.getTenantForProcess(processData.getTenantId(), processDefinition.getUserId()); if(tenant != null){ processInstance.setTenantCode(tenant.getTenantCode()); @@ -406,7 +406,7 @@ public class ProcessInstanceService extends BaseDAGService { } // int update = processDao.updateProcessInstance(processInstanceId, processInstanceJson, // globalParams, schedule, flag, locations, connects); - int update = processDao.updateProcessInstance(processInstance); + int update = processService.updateProcessInstance(processInstance); int updateDefine = 1; if (syncDefine && StringUtils.isNotEmpty(processInstanceJson)) { processDefinition.setProcessDefinitionJson(processInstanceJson); @@ -445,7 +445,7 @@ public class ProcessInstanceService extends BaseDAGService { return checkResult; } - ProcessInstance subInstance = processDao.findProcessInstanceDetailById(subId); + ProcessInstance subInstance = processService.findProcessInstanceDetailById(subId); if (subInstance == null) { putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, subId); return result; @@ -455,7 +455,7 @@ public class ProcessInstanceService extends BaseDAGService { return result; } - ProcessInstance parentWorkflowInstance = processDao.findParentProcessInstance(subId); + ProcessInstance parentWorkflowInstance = processService.findParentProcessInstance(subId); if (parentWorkflowInstance == null) { putMsg(result, Status.SUB_PROCESS_INSTANCE_NOT_EXIST); return result; @@ -476,7 +476,7 @@ public class ProcessInstanceService extends BaseDAGService { * @return delete result code */ @Transactional(rollbackFor = Exception.class) - public Map deleteProcessInstanceById(User loginUser, String projectName, Integer processInstanceId,ITaskQueue tasksQueue) { + public Map deleteProcessInstanceById(User loginUser, String projectName, Integer processInstanceId, ITaskQueue tasksQueue) { Map result = new HashMap<>(5); Project project = projectMapper.queryByName(projectName); @@ -486,8 +486,8 @@ public class ProcessInstanceService extends BaseDAGService { if (resultEnum != Status.SUCCESS) { return checkResult; } - ProcessInstance processInstance = processDao.findProcessInstanceDetailById(processInstanceId); - List taskInstanceList = processDao.findValidTaskListByProcessId(processInstanceId); + ProcessInstance processInstance = processService.findProcessInstanceDetailById(processInstanceId); + List taskInstanceList = processService.findValidTaskListByProcessId(processInstanceId); if (null == processInstance) { putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId); @@ -512,7 +512,7 @@ public class ProcessInstanceService extends BaseDAGService { .append(taskInstance.getId()) .append(UNDERLINE); - int taskWorkerGroupId = processDao.getTaskWorkerGroupId(taskInstance); + int taskWorkerGroupId = processService.getTaskWorkerGroupId(taskInstance); WorkerGroup workerGroup = workerGroupMapper.selectById(taskWorkerGroupId); if(workerGroup == null){ @@ -541,9 +541,9 @@ public class ProcessInstanceService extends BaseDAGService { } // delete database cascade - int delete = processDao.deleteWorkProcessInstanceById(processInstanceId); - processDao.deleteAllSubWorkProcessByParentId(processInstanceId); - processDao.deleteWorkProcessMapByParentId(processInstanceId); + int delete = processService.deleteWorkProcessInstanceById(processInstanceId); + processService.deleteAllSubWorkProcessByParentId(processInstanceId); + processService.deleteWorkProcessMapByParentId(processInstanceId); if (delete > 0) { putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java index bdce9470ca..72122100a1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java @@ -26,7 +26,6 @@ import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Schedule; @@ -34,11 +33,12 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; -import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; -import org.apache.dolphinscheduler.dao.quartz.ProcessScheduleJob; -import org.apache.dolphinscheduler.dao.quartz.QuartzExecutors; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.quartz.ProcessScheduleJob; +import org.apache.dolphinscheduler.service.quartz.QuartzExecutors; +import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,7 +68,7 @@ public class SchedulerService extends BaseService { private MonitorService monitorService; @Autowired - private ProcessDao processDao; + private ProcessService processService; @Autowired private ScheduleMapper scheduleMapper; @@ -119,7 +119,7 @@ public class SchedulerService extends BaseService { } // check work flow define release state - ProcessDefinition processDefinition = processDao.findProcessDefineById(processDefineId); + ProcessDefinition processDefinition = processService.findProcessDefineById(processDefineId); result = executorService.checkProcessDefinitionValid(processDefinition, processDefineId); if (result.get(Constants.STATUS) != Status.SUCCESS) { return result; @@ -221,7 +221,7 @@ public class SchedulerService extends BaseService { return result; } - ProcessDefinition processDefinition = processDao.findProcessDefineById(schedule.getProcessDefinitionId()); + ProcessDefinition processDefinition = processService.findProcessDefineById(schedule.getProcessDefinitionId()); if (processDefinition == null) { putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, schedule.getProcessDefinitionId()); return result; @@ -321,7 +321,7 @@ public class SchedulerService extends BaseService { putMsg(result, Status.SCHEDULE_CRON_REALEASE_NEED_NOT_CHANGE, scheduleStatus); return result; } - ProcessDefinition processDefinition = processDao.findProcessDefineById(scheduleObj.getProcessDefinitionId()); + ProcessDefinition processDefinition = processService.findProcessDefineById(scheduleObj.getProcessDefinitionId()); if (processDefinition == null) { putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, scheduleObj.getProcessDefinitionId()); return result; @@ -338,7 +338,7 @@ public class SchedulerService extends BaseService { } // check sub process definition release state List subProcessDefineIds = new ArrayList<>(); - processDao.recurseFindSubProcessId(scheduleObj.getProcessDefinitionId(), subProcessDefineIds); + processService.recurseFindSubProcessId(scheduleObj.getProcessDefinitionId(), subProcessDefineIds); Integer[] idArray = subProcessDefineIds.toArray(new Integer[subProcessDefineIds.size()]); if (subProcessDefineIds.size() > 0){ List subProcessDefinitionList = @@ -423,7 +423,7 @@ public class SchedulerService extends BaseService { return result; } - ProcessDefinition processDefinition = processDao.findProcessDefineById(processDefineId); + ProcessDefinition processDefinition = processService.findProcessDefineById(processDefineId); if (processDefinition == null) { putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processDefineId); return result; @@ -472,7 +472,7 @@ public class SchedulerService extends BaseService { logger.info("set schedule, project id: {}, scheduleId: {}", projectId, scheduleId); - Schedule schedule = processDao.querySchedule(scheduleId); + Schedule schedule = processService.querySchedule(scheduleId); if (schedule == null) { logger.warn("process schedule info not exists"); return; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java index 74afa2a44e..9690f5c69f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -33,6 +32,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -56,7 +56,7 @@ public class TaskInstanceService extends BaseService { ProjectService projectService; @Autowired - ProcessDao processDao; + ProcessService processService; @Autowired TaskInstanceMapper taskInstanceMapper; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java index 1f523edac5..340a389d1c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.utils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,5 +92,4 @@ public class FourLetterWordMain { throw new IOException("Exception while executing four letter word: " + cmd, e); } } - } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java index 66f57f6a11..f91d3bc68c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java @@ -18,9 +18,9 @@ package org.apache.dolphinscheduler.api.utils; import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord; +import org.apache.dolphinscheduler.service.zk.AbstractZKClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -34,7 +34,7 @@ import java.util.List; * monitor zookeeper info */ @Component -public class ZookeeperMonitor extends AbstractZKClient{ +public class ZookeeperMonitor extends AbstractZKClient { private static final Logger LOG = LoggerFactory.getLogger(ZookeeperMonitor.class); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java index c7afd76cc6..6f308e7b17 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java @@ -21,15 +21,15 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.queue.ITaskQueue; -import org.apache.dolphinscheduler.common.queue.TaskQueueFactory; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.CommandCount; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.queue.ITaskQueue; +import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -78,7 +78,7 @@ public class DataAnalysisServiceTest { ITaskQueue taskQueue; @Mock - ProcessDao processDao; + ProcessService processService; private Project project; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java index 66c7a3ebab..07d7477930 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java @@ -22,10 +22,10 @@ import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.RunMode; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -52,7 +52,7 @@ public class ExecutorService2Test { private ExecutorService executorService; @Mock - private ProcessDao processDao; + private ProcessService processService; @Mock private ProcessDefinitionMapper processDefinitionMapper; @@ -100,8 +100,8 @@ public class ExecutorService2Test { Mockito.when(projectMapper.queryByName(projectName)).thenReturn(project); Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(checkProjectAndAuth()); Mockito.when(processDefinitionMapper.selectById(processDefinitionId)).thenReturn(processDefinition); - Mockito.when(processDao.getTenantForProcess(tenantId, userId)).thenReturn(new Tenant()); - Mockito.when(processDao.createCommand(any(Command.class))).thenReturn(1); + Mockito.when(processService.getTenantForProcess(tenantId, userId)).thenReturn(new Tenant()); + Mockito.when(processService.createCommand(any(Command.class))).thenReturn(1); } /** @@ -111,7 +111,7 @@ public class ExecutorService2Test { @Test public void testNoComplement() throws ParseException { try { - Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); + Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); Map result = executorService.execProcessInstance(loginUser, projectName, processDefinitionId, cronTime, CommandType.START_PROCESS, null, null, @@ -119,7 +119,7 @@ public class ExecutorService2Test { "", "", RunMode.RUN_MODE_SERIAL, Priority.LOW, 0, 110); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - verify(processDao, times(1)).createCommand(any(Command.class)); + verify(processService, times(1)).createCommand(any(Command.class)); }catch (Exception e){ Assert.assertTrue(false); } @@ -132,7 +132,7 @@ public class ExecutorService2Test { @Test public void testDateError() throws ParseException { try { - Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); + Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); Map result = executorService.execProcessInstance(loginUser, projectName, processDefinitionId, "2020-01-31 23:00:00,2020-01-01 00:00:00", CommandType.COMPLEMENT_DATA, null, null, @@ -140,7 +140,7 @@ public class ExecutorService2Test { "", "", RunMode.RUN_MODE_SERIAL, Priority.LOW, 0, 110); Assert.assertEquals(Status.START_PROCESS_INSTANCE_ERROR, result.get(Constants.STATUS)); - verify(processDao, times(0)).createCommand(any(Command.class)); + verify(processService, times(0)).createCommand(any(Command.class)); }catch (Exception e){ Assert.assertTrue(false); } @@ -153,7 +153,7 @@ public class ExecutorService2Test { @Test public void testSerial() throws ParseException { try { - Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); + Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); Map result = executorService.execProcessInstance(loginUser, projectName, processDefinitionId, cronTime, CommandType.COMPLEMENT_DATA, null, null, @@ -161,7 +161,7 @@ public class ExecutorService2Test { "", "", RunMode.RUN_MODE_SERIAL, Priority.LOW, 0, 110); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - verify(processDao, times(1)).createCommand(any(Command.class)); + verify(processService, times(1)).createCommand(any(Command.class)); }catch (Exception e){ Assert.assertTrue(false); } @@ -174,7 +174,7 @@ public class ExecutorService2Test { @Test public void testParallelWithOutSchedule() throws ParseException { try{ - Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); + Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); Map result = executorService.execProcessInstance(loginUser, projectName, processDefinitionId, cronTime, CommandType.COMPLEMENT_DATA, null, null, @@ -182,7 +182,7 @@ public class ExecutorService2Test { "", "", RunMode.RUN_MODE_PARALLEL, Priority.LOW, 0, 110); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - verify(processDao, times(31)).createCommand(any(Command.class)); + verify(processService, times(31)).createCommand(any(Command.class)); }catch (Exception e){ Assert.assertTrue(false); } @@ -195,7 +195,7 @@ public class ExecutorService2Test { @Test public void testParallelWithSchedule() throws ParseException { try{ - Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(oneSchedulerList()); + Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(oneSchedulerList()); Map result = executorService.execProcessInstance(loginUser, projectName, processDefinitionId, cronTime, CommandType.COMPLEMENT_DATA, null, null, @@ -203,7 +203,7 @@ public class ExecutorService2Test { "", "", RunMode.RUN_MODE_PARALLEL, Priority.LOW, 0, 110); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - verify(processDao, times(15)).createCommand(any(Command.class)); + verify(processService, times(15)).createCommand(any(Command.class)); }catch (Exception e){ Assert.assertTrue(false); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index c6ab6f8e74..20571577e3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -40,14 +40,14 @@ public class LoggerServiceTest { @InjectMocks private LoggerService loggerService; @Mock - private ProcessDao processDao; + private ProcessService processService; @Test public void testQueryDataSourceList(){ TaskInstance taskInstance = new TaskInstance(); - Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); + Mockito.when(processService.findTaskInstanceById(1)).thenReturn(taskInstance); Result result = loggerService.queryLog(2,1,1); //TASK_INSTANCE_NOT_FOUND Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(),result.getCode().intValue()); @@ -59,7 +59,7 @@ public class LoggerServiceTest { //SUCCESS taskInstance.setHost("127.0.0.1"); taskInstance.setLogPath("/temp/log"); - Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); + Mockito.when(processService.findTaskInstanceById(1)).thenReturn(taskInstance); result = loggerService.queryLog(1,1,1); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); } @@ -68,7 +68,7 @@ public class LoggerServiceTest { public void testGetLogBytes(){ TaskInstance taskInstance = new TaskInstance(); - Mockito.when(processDao.findTaskInstanceById(1)).thenReturn(taskInstance); + Mockito.when(processService.findTaskInstanceById(1)).thenReturn(taskInstance); //task instance is null try{ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 51b440b9ca..aa9f5fe20f 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -28,9 +28,9 @@ import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.http.entity.ContentType; import org.json.JSONException; import org.junit.Assert; @@ -80,7 +80,7 @@ public class ProcessDefinitionServiceTest { private WorkerGroupMapper workerGroupMapper; @Mock - private ProcessDao processDao; + private ProcessService processService; private String sqlDependentJson = "{\"globalParams\":[]," + "\"tasks\":[{\"type\":\"SQL\",\"id\":\"tasks-27297\",\"name\":\"sql\"," + @@ -584,7 +584,7 @@ public class ProcessDefinitionServiceTest { Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); - Mockito.when(processDao.findProcessDefineById(1)).thenReturn(getProcessDefinition()); + Mockito.when(processService.findProcessDefineById(1)).thenReturn(getProcessDefinition()); Map updateResult = processDefinitionService.updateProcessDefinition(loginUser, projectName, 1, "test", sqlDependentJson, "", "", ""); diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index bd2448eee7..2ef61d7f36 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -85,21 +85,7 @@ com.fasterxml.jackson.core jackson-databind - - org.apache.curator - curator-client - ${curator.version} - - - log4j-1.2-api - org.apache.logging.log4j - - - io.netty - netty - - - + org.apache.commons commons-collections4 @@ -548,6 +534,10 @@ log4j-web org.apache.logging.log4j + + jasper-compiler + tomcat + @@ -601,11 +591,6 @@ compile - - org.springframework - spring-context - - org.codehaus.janino janino diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java index 92337f5de6..e59cbd1b96 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.springframework.lang.Nullable; /** * A collection of static utility methods to validate input. @@ -57,7 +56,7 @@ public final class Preconditions { * * @throws NullPointerException Thrown, if the passed reference was null. */ - public static T checkNotNull(T reference, @Nullable String errorMessage) { + public static T checkNotNull(T reference, String errorMessage) { if (reference == null) { throw new NullPointerException(String.valueOf(errorMessage)); } @@ -84,8 +83,8 @@ public final class Preconditions { * @throws NullPointerException Thrown, if the passed reference was null. */ public static T checkNotNull(T reference, - @Nullable String errorMessageTemplate, - @Nullable Object... errorMessageArgs) { + String errorMessageTemplate, + Object... errorMessageArgs) { if (reference == null) { throw new NullPointerException(format(errorMessageTemplate, errorMessageArgs)); @@ -121,7 +120,7 @@ public final class Preconditions { * * @throws IllegalArgumentException Thrown, if the condition is violated. */ - public static void checkArgument(boolean condition, @Nullable Object errorMessage) { + public static void checkArgument(boolean condition, Object errorMessage) { if (!condition) { throw new IllegalArgumentException(String.valueOf(errorMessage)); } @@ -141,8 +140,8 @@ public final class Preconditions { * @throws IllegalArgumentException Thrown, if the condition is violated. */ public static void checkArgument(boolean condition, - @Nullable String errorMessageTemplate, - @Nullable Object... errorMessageArgs) { + String errorMessageTemplate, + Object... errorMessageArgs) { if (!condition) { throw new IllegalArgumentException(format(errorMessageTemplate, errorMessageArgs)); @@ -177,7 +176,7 @@ public final class Preconditions { * * @throws IllegalStateException Thrown, if the condition is violated. */ - public static void checkState(boolean condition, @Nullable Object errorMessage) { + public static void checkState(boolean condition, Object errorMessage) { if (!condition) { throw new IllegalStateException(String.valueOf(errorMessage)); } @@ -197,8 +196,8 @@ public final class Preconditions { * @throws IllegalStateException Thrown, if the condition is violated. */ public static void checkState(boolean condition, - @Nullable String errorMessageTemplate, - @Nullable Object... errorMessageArgs) { + String errorMessageTemplate, + Object... errorMessageArgs) { if (!condition) { throw new IllegalStateException(format(errorMessageTemplate, errorMessageArgs)); @@ -231,7 +230,7 @@ public final class Preconditions { * @throws IllegalArgumentException Thrown, if size is negative. * @throws IndexOutOfBoundsException Thrown, if the index negative or greater than or equal to size */ - public static void checkElementIndex(int index, int size, @Nullable String errorMessage) { + public static void checkElementIndex(int index, int size, String errorMessage) { checkArgument(size >= 0, "Size was negative."); if (index < 0 || index >= size) { throw new IndexOutOfBoundsException(String.valueOf(errorMessage) + " Index: " + index + ", Size: " + size); @@ -248,7 +247,7 @@ public final class Preconditions { * *

This method is taken quasi verbatim from the Guava Preconditions class. */ - private static String format(@Nullable String template, @Nullable Object... args) { + private static String format( String template, Object... args) { final int numArgs = args == null ? 0 : args.length; template = String.valueOf(template); // null -> "null" diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 5b23847ba3..3d51aa830f 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.dolphinscheduler.common.Constants; -import org.apache.yetus.audience.InterfaceAudience; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index b3b22c128d..20d19410e2 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -116,21 +116,6 @@ cron-utils - - org.quartz-scheduler - quartz - - - c3p0 - c3p0 - - - - - - org.quartz-scheduler - quartz-jobs - commons-configuration commons-configuration diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/BaseDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index 41a9b3a566..a46e5aabcc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; /** * data source base class diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java index 48550c31cc..e159f81d2e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java index 44ee200c5d..3c2366b5b0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DataSourceFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java similarity index 98% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DataSourceFactory.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java index 48ec319eaa..9571f9c9f6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DataSourceFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java similarity index 93% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java index cddedd1f73..0a8f527536 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java @@ -14,14 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.*; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; /** * data source of hive diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java index fa149e67e2..6e2fbe3dd8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java index c3dc3a96df..cefaf879b5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java index 4989e7681e..176cba2587 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java index 8554992efc..07770c06a7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java index 5d10c63e5d..81a5ac6f04 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.job.db; +package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml new file mode 100644 index 0000000000..b67b033ffa --- /dev/null +++ b/dolphinscheduler-remote/pom.xml @@ -0,0 +1,44 @@ + + + + + dolphinscheduler + org.apache.dolphinscheduler + 1.2.1-SNAPSHOT + + 4.0.0 + + dolphinscheduler-remote + + dolphinscheduler-remote + + http://www.example.com + + + UTF-8 + 1.7 + 1.7 + + + + + io.netty + netty-all + + + org.slf4j + slf4j-api + + + com.alibaba + fastjson + + + junit + junit + test + + + + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java new file mode 100644 index 0000000000..df0c13ad38 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote; + +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.*; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import org.apache.dolphinscheduler.remote.codec.NettyDecoder; +import org.apache.dolphinscheduler.remote.codec.NettyEncoder; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.config.NettyClientConfig; +import org.apache.dolphinscheduler.remote.exceptions.RemotingException; +import org.apache.dolphinscheduler.remote.handler.NettyClientHandler; +import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; +import org.apache.dolphinscheduler.remote.utils.Address; +import org.apache.dolphinscheduler.remote.utils.Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * remoting netty client + */ +public class NettyRemotingClient { + + private final Logger logger = LoggerFactory.getLogger(NettyRemotingClient.class); + + /** + * bootstrap + */ + private final Bootstrap bootstrap = new Bootstrap(); + + /** + * encoder + */ + private final NettyEncoder encoder = new NettyEncoder(); + + /** + * channels + */ + private final ConcurrentHashMap channels = new ConcurrentHashMap(); + + /** + * default executor + */ + private final ExecutorService defaultExecutor = Executors.newFixedThreadPool(Constants.CPUS); + + /** + * started flag + */ + private final AtomicBoolean isStarted = new AtomicBoolean(false); + + /** + * worker group + */ + private final NioEventLoopGroup workerGroup; + + /** + * client handler + */ + private final NettyClientHandler clientHandler = new NettyClientHandler(this); + + /** + * netty client config + */ + private final NettyClientConfig clientConfig; + + /** + * netty client init + * + * @param clientConfig client config + */ + public NettyRemotingClient(final NettyClientConfig clientConfig){ + this.clientConfig = clientConfig; + this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyClient_%d", this.threadIndex.incrementAndGet())); + } + }); + this.start(); + } + + /** + * netty server start + */ + private void start(){ + + this.bootstrap + .group(this.workerGroup) + .channel(NioSocketChannel.class) + .option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive()) + .option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay()) + .option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize()) + .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize()) + .handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast( + new NettyDecoder(), + clientHandler, + encoder); + } + }); + isStarted.compareAndSet(false, true); + } + + /** + * register processor + * + * @param commandType command type + * @param processor processor + */ + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) { + registerProcessor(commandType, processor, null); + } + + /** + * register processor + * + * @param commandType command type + * @param processor processor + * @param executor thread executor + */ + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + this.clientHandler.registerProcessor(commandType, processor, executor); + } + + /** + * send connect + * @param address address + * @param command command + * @throws RemotingException + */ + public void send(final Address address, final Command command) throws RemotingException { + final Channel channel = getChannel(address); + if (channel == null) { + throw new RemotingException("network error"); + } + try { + channel.writeAndFlush(command).addListener(new ChannelFutureListener(){ + + @Override + public void operationComplete(ChannelFuture future) throws Exception { + if(future.isSuccess()){ + logger.info("sent command {} to {}", command, address); + } else{ + logger.error("send command {} to {} failed, error {}", command, address, future.cause()); + } + } + }); + } catch (Exception ex) { + String msg = String.format("send command %s to address %s encounter error", command, address); + throw new RemotingException(msg, ex); + } + } + + /** + * get channel + * @param address address + * @return channel + */ + public Channel getChannel(Address address) { + Channel channel = channels.get(address); + if(channel != null && channel.isActive()){ + return channel; + } + return createChannel(address, true); + } + + /** + * create channel + * @param address address + * @param isSync is sync + * @return channel + */ + public Channel createChannel(Address address, boolean isSync) { + ChannelFuture future; + try { + synchronized (bootstrap){ + future = bootstrap.connect(new InetSocketAddress(address.getHost(), address.getPort())); + } + if(isSync){ + future.sync(); + } + if (future.isSuccess()) { + Channel channel = future.channel(); + channels.put(address, channel); + return channel; + } + } catch (Exception ex) { + logger.info("connect to {} error {}", address, ex); + } + return null; + } + + /** + * get default thread executor + * @return thread executor + */ + public ExecutorService getDefaultExecutor() { + return defaultExecutor; + } + + /** + * close client + */ + public void close() { + if(isStarted.compareAndSet(true, false)){ + try { + closeChannels(); + if(workerGroup != null){ + this.workerGroup.shutdownGracefully(); + } + if(defaultExecutor != null){ + defaultExecutor.shutdown(); + } + } catch (Exception ex) { + logger.error("netty client close exception", ex); + } + logger.info("netty client closed"); + } + } + + /** + * close channel + */ + private void closeChannels(){ + for (Channel channel : this.channels.values()) { + channel.close(); + } + this.channels.clear(); + } + + /** + * remove channel + * @param address address + */ + public void removeChannel(Address address){ + Channel channel = this.channels.remove(address); + if(channel != null){ + channel.close(); + } + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java new file mode 100644 index 0000000000..c69bf09540 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import org.apache.dolphinscheduler.remote.codec.NettyDecoder; +import org.apache.dolphinscheduler.remote.codec.NettyEncoder; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +import org.apache.dolphinscheduler.remote.handler.NettyServerHandler; +import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; +import org.apache.dolphinscheduler.remote.utils.Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * remoting netty server + */ +public class NettyRemotingServer { + + private final Logger logger = LoggerFactory.getLogger(NettyRemotingServer.class); + + /** + * server bootstart + */ + private final ServerBootstrap serverBootstrap = new ServerBootstrap(); + + /** + * encoder + */ + private final NettyEncoder encoder = new NettyEncoder(); + + /** + * default executor + */ + private final ExecutorService defaultExecutor = Executors.newFixedThreadPool(Constants.CPUS); + + /** + * boss group + */ + private final NioEventLoopGroup bossGroup; + + /** + * worker group + */ + private final NioEventLoopGroup workGroup; + + /** + * server config + */ + private final NettyServerConfig serverConfig; + + /** + * server handler + */ + private final NettyServerHandler serverHandler = new NettyServerHandler(this); + + /** + * started flag + */ + private final AtomicBoolean isStarted = new AtomicBoolean(false); + + /** + * server init + * + * @param serverConfig server config + */ + public NettyRemotingServer(final NettyServerConfig serverConfig){ + this.serverConfig = serverConfig; + + this.bossGroup = new NioEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerBossThread_%d", this.threadIndex.incrementAndGet())); + } + }); + + this.workGroup = new NioEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); + } + }); + } + + /** + * server start + */ + public void start(){ + + if(this.isStarted.get()){ + return; + } + + this.serverBootstrap + .group(this.bossGroup, this.workGroup) + .channel(NioServerSocketChannel.class) + .option(ChannelOption.SO_REUSEADDR, true) + .option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog()) + .childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive()) + .childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay()) + .childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize()) + .childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize()) + .childHandler(new ChannelInitializer() { + + @Override + protected void initChannel(NioSocketChannel ch) throws Exception { + initNettyChannel(ch); + } + }); + + ChannelFuture future; + try { + future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); + } catch (Exception e) { + logger.error("NettyRemotingServer bind fail {}, exit", e); + throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); + } + if (future.isSuccess()) { + logger.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort()); + } else if (future.cause() != null) { + throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()), future.cause()); + } else { + throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); + } + // + isStarted.compareAndSet(false, true); + } + + /** + * init netty channel + * @param ch socket channel + * @throws Exception + */ + private void initNettyChannel(NioSocketChannel ch) throws Exception{ + ChannelPipeline pipeline = ch.pipeline(); + pipeline.addLast("encoder", encoder); + pipeline.addLast("decoder", new NettyDecoder()); + pipeline.addLast("handler", serverHandler); + } + + /** + * register processor + * @param commandType command type + * @param processor processor + */ + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) { + this.registerProcessor(commandType, processor, null); + } + + /** + * register processor + * + * @param commandType command type + * @param processor processor + * @param executor thread executor + */ + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + this.serverHandler.registerProcessor(commandType, processor, executor); + } + + /** + * get default thread executor + * @return thread executor + */ + public ExecutorService getDefaultExecutor() { + return defaultExecutor; + } + + public void close() { + if(isStarted.compareAndSet(true, false)){ + try { + if(bossGroup != null){ + this.bossGroup.shutdownGracefully(); + } + if(workGroup != null){ + this.workGroup.shutdownGracefully(); + } + if(defaultExecutor != null){ + defaultExecutor.shutdown(); + } + } catch (Exception ex) { + logger.error("netty server close exception", ex); + } + logger.info("netty server closed"); + } + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java new file mode 100644 index 0000000000..caa4fbdd17 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.codec; + + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ReplayingDecoder; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandHeader; +import org.apache.dolphinscheduler.remote.command.CommandType; + +import java.util.List; + +/** + * netty decoder + */ +public class NettyDecoder extends ReplayingDecoder { + + public NettyDecoder(){ + super(State.MAGIC); + } + + private final CommandHeader commandHeader = new CommandHeader(); + + /** + * decode + * + * @param ctx channel handler context + * @param in byte buffer + * @param out out content + * @throws Exception + */ + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { + switch (state()){ + case MAGIC: + checkMagic(in.readByte()); + checkpoint(State.COMMAND); + case COMMAND: + commandHeader.setType(in.readByte()); + checkpoint(State.OPAQUE); + case OPAQUE: + commandHeader.setOpaque(in.readLong()); + checkpoint(State.BODY_LENGTH); + case BODY_LENGTH: + commandHeader.setBodyLength(in.readInt()); + checkpoint(State.BODY); + case BODY: + byte[] body = new byte[commandHeader.getBodyLength()]; + in.readBytes(body); + // + Command packet = new Command(); + packet.setType(commandType(commandHeader.getType())); + packet.setOpaque(commandHeader.getOpaque()); + packet.setBody(body); + out.add(packet); + // + checkpoint(State.MAGIC); + } + } + + /** + * get command type + * @param type type + * @return + */ + private CommandType commandType(byte type){ + for(CommandType ct : CommandType.values()){ + if(ct.ordinal() == type){ + return ct; + } + } + return null; + } + + /** + * check magic + * @param magic magic + */ + private void checkMagic(byte magic) { + if (magic != Command.MAGIC) { + throw new IllegalArgumentException("illegal packet [magic]" + magic); + } + } + + enum State{ + MAGIC, + COMMAND, + OPAQUE, + BODY_LENGTH, + BODY; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java new file mode 100644 index 0000000000..4e9836a26f --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.codec; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandler.Sharable; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.MessageToByteEncoder; +import org.apache.dolphinscheduler.remote.command.Command; + +/** + * netty encoder + */ +@Sharable +public class NettyEncoder extends MessageToByteEncoder { + + /** + * encode + * + * @param ctx channel handler context + * @param msg command + * @param out byte buffer + * @throws Exception + */ + @Override + protected void encode(ChannelHandlerContext ctx, Command msg, ByteBuf out) throws Exception { + if(msg == null){ + throw new Exception("encode msg is null"); + } + out.writeByte(Command.MAGIC); + out.writeByte(msg.getType().ordinal()); + out.writeLong(msg.getOpaque()); + out.writeInt(msg.getBody().length); + out.writeBytes(msg.getBody()); + } + +} + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java new file mode 100644 index 0000000000..ee95044764 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import java.io.Serializable; + +/** + * receive task log request command and content fill + * for netty data serializable transfer + */ +public class Command implements Serializable { + + private static final long serialVersionUID = 1L; + + public static final byte MAGIC = (byte) 0xbabe; + + public Command(){ + } + + public Command(long opaque){ + this.opaque = opaque; + } + + /** + * command type + */ + private CommandType type; + + /** + * request unique identification + */ + private long opaque; + + /** + * data body + */ + private byte[] body; + + public CommandType getType() { + return type; + } + + public void setType(CommandType type) { + this.type = type; + } + + public long getOpaque() { + return opaque; + } + + public void setOpaque(long opaque) { + this.opaque = opaque; + } + + public byte[] getBody() { + return body; + } + + public void setBody(byte[] body) { + this.body = body; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (opaque ^ (opaque >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Command other = (Command) obj; + return opaque == other.opaque; + } + + @Override + public String toString() { + return "Command [type=" + type + ", opaque=" + opaque + ", bodyLen=" + (body == null ? 0 : body.length) + "]"; + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandHeader.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandHeader.java new file mode 100644 index 0000000000..78948a5c0c --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandHeader.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import java.io.Serializable; + +/** + * command header + */ +public class CommandHeader implements Serializable { + + /** + * type + */ + private byte type; + + /** + * request unique identification + */ + private long opaque; + + /** + * body length + */ + private int bodyLength; + + public int getBodyLength() { + return bodyLength; + } + + public void setBodyLength(int bodyLength) { + this.bodyLength = bodyLength; + } + + public byte getType() { + return type; + } + + public void setType(byte type) { + this.type = type; + } + + public long getOpaque() { + return opaque; + } + + public void setOpaque(long opaque) { + this.opaque = opaque; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java new file mode 100644 index 0000000000..b1b24d3303 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java @@ -0,0 +1 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; public enum CommandType { /** * roll view log request */ ROLL_VIEW_LOG_REQUEST, /** * roll view log response */ ROLL_VIEW_LOG_RESPONSE, /** * view whole log request */ VIEW_WHOLE_LOG_REQUEST, /** * view whole log response */ VIEW_WHOLE_LOG_RESPONSE, /** * get log bytes request */ GET_LOG_BYTES_REQUEST, /** * get log bytes response */ GET_LOG_BYTES_RESPONSE, WORKER_REQUEST, MASTER_RESPONSE, /** * execute task request */ EXECUTE_TASK_REQUEST, /** * execute task response */ EXECUTE_TASK_RESPONSE, /** * ping */ PING, /** * pong */ PONG; } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java new file mode 100644 index 0000000000..a582221cd3 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java @@ -0,0 +1 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.List; import java.util.concurrent.atomic.AtomicLong; /** * execute task request command */ public class ExecuteTaskRequestCommand implements Serializable { private static final AtomicLong REQUEST = new AtomicLong(1); /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * application name */ private String applicationName; /** * group name */ private String groupName; /** * task name */ private String taskName; /** * connect port */ private int connectorPort; /** * description info */ private String description; /** * class name */ private String className; /** * method name */ private String methodName; /** * params */ private String params; /** * shard items */ private List shardItems; public List getShardItems() { return shardItems; } public void setShardItems(List shardItems) { this.shardItems = shardItems; } public String getParams() { return params; } public void setParams(String params) { this.params = params; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getApplicationName() { return applicationName; } public void setApplicationName(String applicationName) { this.applicationName = applicationName; } public String getGroupName() { return groupName; } public void setGroupName(String groupName) { this.groupName = groupName; } public String getTaskName() { return taskName; } public void setTaskName(String taskName) { this.taskName = taskName; } public int getConnectorPort() { return connectorPort; } public void setConnectorPort(int connectorPort) { this.connectorPort = connectorPort; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getClassName() { return className; } public void setClassName(String className) { this.className = className; } public String getMethodName() { return methodName; } public void setMethodName(String methodName) { this.methodName = methodName; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(REQUEST.getAndIncrement()); command.setType(CommandType.EXECUTE_TASK_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java new file mode 100644 index 0000000000..0268653b5d --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java @@ -0,0 +1 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.concurrent.atomic.AtomicLong; /** * execute taks response command */ public class ExecuteTaskResponseCommand implements Serializable { private static final AtomicLong REQUEST = new AtomicLong(1); /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * result info */ private Object result; /** * receive time */ private long receivedTime; /** * execute count */ private int executeCount; /** * execute time */ private long executeTime; public String getAttemptId() { return attemptId; } public void setAttemptId(String attemptId) { this.attemptId = attemptId; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public Object getResult() { return result; } public void setResult(Object result) { this.result = result; } public long getReceivedTime() { return receivedTime; } public void setReceivedTime(long receivedTime) { this.receivedTime = receivedTime; } public int getExecuteCount() { return executeCount; } public void setExecuteCount(int executeCount) { this.executeCount = executeCount; } public long getExecuteTime() { return executeTime; } public void setExecuteTime(long executeTime) { this.executeTime = executeTime; } /** * package response command * * @return command */ public Command convert2Command(){ Command command = new Command(REQUEST.getAndIncrement()); command.setType(CommandType.EXECUTE_TASK_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java new file mode 100644 index 0000000000..4f32d5f699 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.command; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; + +import java.io.Serializable; +import java.util.concurrent.atomic.AtomicLong; + +/** + * ping machine + */ +public class Ping implements Serializable { + + private static final AtomicLong ID = new AtomicLong(1); + + /** + * ping body + */ + protected static ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; + + /** + * request command body + */ + private static byte[] EMPTY_BODY_ARRAY = new byte[0]; + + private static final ByteBuf PING_BUF; + + static { + ByteBuf ping = Unpooled.buffer(); + ping.writeByte(Command.MAGIC); + ping.writeByte(CommandType.PING.ordinal()); + ping.writeLong(0); + ping.writeInt(0); + ping.writeBytes(EMPTY_BODY); + PING_BUF = Unpooled.unreleasableBuffer(ping).asReadOnly(); + } + + /** + * ping connect + * @return result + */ + public static ByteBuf pingContent(){ + return PING_BUF.duplicate(); + } + + /** + * package ping command + * + * @return command + */ + public static Command create(){ + Command command = new Command(ID.getAndIncrement()); + command.setType(CommandType.PING); + command.setBody(EMPTY_BODY_ARRAY); + return command; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java new file mode 100644 index 0000000000..e52cef6d92 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.command; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; + +import java.io.Serializable; + +/** + * Pong return after ping + */ +public class Pong implements Serializable { + + /** + * pong body + */ + protected static ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; + + /** + * pong command body + */ + private static byte[] EMPTY_BODY_ARRAY = new byte[0]; + + /** + * ping byte buffer + */ + private static final ByteBuf PONG_BUF; + + static { + ByteBuf ping = Unpooled.buffer(); + ping.writeByte(Command.MAGIC); + ping.writeByte(CommandType.PONG.ordinal()); + ping.writeLong(0); + ping.writeInt(0); + ping.writeBytes(EMPTY_BODY); + PONG_BUF = Unpooled.unreleasableBuffer(ping).asReadOnly(); + } + + /** + * ping content + * @return result + */ + public static ByteBuf pingContent(){ + return PONG_BUF.duplicate(); + } + + /** + * package pong command + * + * @param opaque request unique identification + * @return command + */ + public static Command create(long opaque){ + Command command = new Command(opaque); + command.setType(CommandType.PONG); + command.setBody(EMPTY_BODY_ARRAY); + return command; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java new file mode 100644 index 0000000000..9b064b7136 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.command.log; + +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; + +import java.io.Serializable; +import java.util.concurrent.atomic.AtomicLong; + +/** + * get log bytes request command + */ +public class GetLogBytesRequestCommand implements Serializable { + + /** + * request id + */ + private static final AtomicLong REQUEST = new AtomicLong(1); + + /** + * log path + */ + private String path; + + public GetLogBytesRequestCommand() { + } + + public GetLogBytesRequestCommand(String path) { + this.path = path; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(REQUEST.getAndIncrement()); + command.setType(CommandType.GET_LOG_BYTES_REQUEST); + byte[] body = FastJsonSerializer.serialize(this); + command.setBody(body); + return command; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java new file mode 100644 index 0000000000..deaf9b8d85 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.command.log; + +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; + +import java.io.Serializable; + +/** + * get log bytes response command + */ +public class GetLogBytesResponseCommand implements Serializable { + + /** + * log byte data + */ + private byte[] data; + + public GetLogBytesResponseCommand() { + } + + public GetLogBytesResponseCommand(byte[] data) { + this.data = data; + } + + public byte[] getData() { + return data; + } + + public void setData(byte[] data) { + this.data = data; + } + + /** + * package response command + * + * @param opaque request unique identification + * @return command + */ + public Command convert2Command(long opaque){ + Command command = new Command(opaque); + command.setType(CommandType.GET_LOG_BYTES_RESPONSE); + byte[] body = FastJsonSerializer.serialize(this); + command.setBody(body); + return command; + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java new file mode 100644 index 0000000000..f072c479f4 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.command.log; + +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; + +import java.io.Serializable; +import java.util.concurrent.atomic.AtomicLong; + +/** + * roll view log request command + */ +public class RollViewLogRequestCommand implements Serializable { + + /** + * request id + */ + private static final AtomicLong REQUEST = new AtomicLong(1); + + /** + * log path + */ + private String path; + + /** + * skip line number + */ + private int skipLineNum; + + /** + * query log line number limit + */ + private int limit; + + public RollViewLogRequestCommand() { + } + + public RollViewLogRequestCommand(String path, int skipLineNum, int limit) { + this.path = path; + this.skipLineNum = skipLineNum; + this.limit = limit; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public int getSkipLineNum() { + return skipLineNum; + } + + public void setSkipLineNum(int skipLineNum) { + this.skipLineNum = skipLineNum; + } + + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(REQUEST.getAndIncrement()); + command.setType(CommandType.ROLL_VIEW_LOG_REQUEST); + byte[] body = FastJsonSerializer.serialize(this); + command.setBody(body); + return command; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java new file mode 100644 index 0000000000..591d787200 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.command.log; + +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; + +import java.io.Serializable; + +/** + * roll view log response command + */ +public class RollViewLogResponseCommand implements Serializable { + + /** + * response data + */ + private String msg; + + public RollViewLogResponseCommand() { + } + + public RollViewLogResponseCommand(String msg) { + this.msg = msg; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + /** + * package response command + * + * @param opaque request unique identification + * @return command + */ + public Command convert2Command(long opaque){ + Command command = new Command(opaque); + command.setType(CommandType.ROLL_VIEW_LOG_RESPONSE); + byte[] body = FastJsonSerializer.serialize(this); + command.setBody(body); + return command; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java new file mode 100644 index 0000000000..5dcefc6233 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.command.log; + +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; + +import java.io.Serializable; +import java.util.concurrent.atomic.AtomicLong; + +/** + * view log request command + */ +public class ViewLogRequestCommand implements Serializable { + + /** + * request id + */ + private static final AtomicLong REQUEST = new AtomicLong(1); + + private String path; + + public ViewLogRequestCommand() { + } + + public ViewLogRequestCommand(String path) { + this.path = path; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(REQUEST.getAndIncrement()); + command.setType(CommandType.VIEW_WHOLE_LOG_REQUEST); + byte[] body = FastJsonSerializer.serialize(this); + command.setBody(body); + return command; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java new file mode 100644 index 0000000000..dffadade26 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.command.log; + +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; + +import java.io.Serializable; + +/** + * view log response command + */ +public class ViewLogResponseCommand implements Serializable { + + /** + * response data + */ + private String msg; + + public ViewLogResponseCommand() { + } + + public ViewLogResponseCommand(String msg) { + this.msg = msg; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + /** + * package response command + * + * @param opaque request unique identification + * @return command + */ + public Command convert2Command(long opaque){ + Command command = new Command(opaque); + command.setType(CommandType.VIEW_WHOLE_LOG_RESPONSE); + byte[] body = FastJsonSerializer.serialize(this); + command.setBody(body); + return command; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java new file mode 100644 index 0000000000..831e05f7e7 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.config; + +import org.apache.dolphinscheduler.remote.utils.Constants; + +/** + * netty client config + */ +public class NettyClientConfig { + + /** + * worker threads，default get machine cpus + */ + private int workerThreads = Constants.CPUS; + + /** + * whether tpc delay + */ + private boolean tcpNoDelay = true; + + /** + * whether keep alive + */ + private boolean soKeepalive = true; + + /** + * send buffer size + */ + private int sendBufferSize = 65535; + + /** + * receive buffer size + */ + private int receiveBufferSize = 65535; + + public int getWorkerThreads() { + return workerThreads; + } + + public void setWorkerThreads(int workerThreads) { + this.workerThreads = workerThreads; + } + + public boolean isTcpNoDelay() { + return tcpNoDelay; + } + + public void setTcpNoDelay(boolean tcpNoDelay) { + this.tcpNoDelay = tcpNoDelay; + } + + public boolean isSoKeepalive() { + return soKeepalive; + } + + public void setSoKeepalive(boolean soKeepalive) { + this.soKeepalive = soKeepalive; + } + + public int getSendBufferSize() { + return sendBufferSize; + } + + public void setSendBufferSize(int sendBufferSize) { + this.sendBufferSize = sendBufferSize; + } + + public int getReceiveBufferSize() { + return receiveBufferSize; + } + + public void setReceiveBufferSize(int receiveBufferSize) { + this.receiveBufferSize = receiveBufferSize; + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java new file mode 100644 index 0000000000..4ec8a0f7a7 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.config; + +import org.apache.dolphinscheduler.remote.utils.Constants; + +/** + * netty server config + */ +public class NettyServerConfig { + + /** + * init the server connectable queue + */ + private int soBacklog = 1024; + + /** + * whether tpc delay + */ + private boolean tcpNoDelay = true; + + /** + * whether keep alive + */ + private boolean soKeepalive = true; + + /** + * send buffer size + */ + private int sendBufferSize = 65535; + + /** + * receive buffer size + */ + private int receiveBufferSize = 65535; + + /** + * worker threads，default get machine cpus + */ + private int workerThread = Constants.CPUS; + + /** + * listen port + */ + private int listenPort = 12346; + + public int getListenPort() { + return listenPort; + } + + public void setListenPort(int listenPort) { + this.listenPort = listenPort; + } + + public int getSoBacklog() { + return soBacklog; + } + + public void setSoBacklog(int soBacklog) { + this.soBacklog = soBacklog; + } + + public boolean isTcpNoDelay() { + return tcpNoDelay; + } + + public void setTcpNoDelay(boolean tcpNoDelay) { + this.tcpNoDelay = tcpNoDelay; + } + + public boolean isSoKeepalive() { + return soKeepalive; + } + + public void setSoKeepalive(boolean soKeepalive) { + this.soKeepalive = soKeepalive; + } + + public int getSendBufferSize() { + return sendBufferSize; + } + + public void setSendBufferSize(int sendBufferSize) { + this.sendBufferSize = sendBufferSize; + } + + public int getReceiveBufferSize() { + return receiveBufferSize; + } + + public void setReceiveBufferSize(int receiveBufferSize) { + this.receiveBufferSize = receiveBufferSize; + } + + public int getWorkerThread() { + return workerThread; + } + + public void setWorkerThread(int workerThread) { + this.workerThread = workerThread; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java new file mode 100644 index 0000000000..29d48db8f8 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.exceptions; + +/** + * remote exception + */ +public class RemotingException extends Exception { + + public RemotingException() { + super(); + } + + /** Constructs a new runtime exception with the specified detail message. + * The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + * + * @param message the detail message. The detail message is saved for + * later retrieval by the {@link #getMessage()} method. + */ + public RemotingException(String message) { + super(message); + } + + /** + * Constructs a new runtime exception with the specified detail message and + * cause.

Note that the detail message associated with + * {@code cause} is not automatically incorporated in + * this runtime exception's detail message. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A null value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public RemotingException(String message, Throwable cause) { + super(message, cause); + } + + /** Constructs a new runtime exception with the specified cause and a + * detail message of (cause==null ? null : cause.toString()) + * (which typically contains the class and detail message of + * cause). This constructor is useful for runtime exceptions + * that are little more than wrappers for other throwables. + * + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A null value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public RemotingException(Throwable cause) { + super(cause); + } + + /** + * Constructs a new runtime exception with the specified detail + * message, cause, suppression enabled or disabled, and writable + * stack trace enabled or disabled. + * + * @param message the detail message. + * @param cause the cause. (A {@code null} value is permitted, + * and indicates that the cause is nonexistent or unknown.) + * @param enableSuppression whether or not suppression is enabled + * or disabled + * @param writableStackTrace whether or not the stack trace should + * be writable + * + * @since 1.7 + */ + protected RemotingException(String message, Throwable cause, + boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java new file mode 100644 index 0000000000..6aceb5a41b --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.handler; + +import io.netty.channel.*; +import org.apache.dolphinscheduler.remote.NettyRemotingClient; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; +import org.apache.dolphinscheduler.remote.utils.ChannelUtils; +import org.apache.dolphinscheduler.remote.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; + +/** + * netty client request handler + */ +@ChannelHandler.Sharable +public class NettyClientHandler extends ChannelInboundHandlerAdapter { + + private final Logger logger = LoggerFactory.getLogger(NettyClientHandler.class); + + /** + * netty remote client + */ + private final NettyRemotingClient nettyRemotingClient; + + /** + * client processors queue + */ + private final ConcurrentHashMap> processors = new ConcurrentHashMap(); + + public NettyClientHandler(NettyRemotingClient nettyRemotingClient){ + this.nettyRemotingClient = nettyRemotingClient; + } + + /** + * When the current channel is not active, + * the current channel has reached the end of its life cycle + * + * @param ctx channel handler context + * @throws Exception + */ + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + nettyRemotingClient.removeChannel(ChannelUtils.toAddress(ctx.channel())); + ctx.channel().close(); + } + + /** + * The current channel reads data from the remote + * + * @param ctx channel handler context + * @param msg message + * @throws Exception + */ + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + processReceived(ctx.channel(), (Command)msg); + } + + /** + * register processor + * + * @param commandType command type + * @param processor processor + */ + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) { + this.registerProcessor(commandType, processor, nettyRemotingClient.getDefaultExecutor()); + } + + /** + * register processor + * + * @param commandType command type + * @param processor processor + * @param executor thread executor + */ + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + ExecutorService executorRef = executor; + if(executorRef == null){ + executorRef = nettyRemotingClient.getDefaultExecutor(); + } + this.processors.putIfAbsent(commandType, new Pair(processor, executorRef)); + } + + /** + * process received logic + * + * @param channel channel + * @param msg message + */ + private void processReceived(final Channel channel, final Command msg) { + final CommandType commandType = msg.getType(); + final Pair pair = processors.get(commandType); + if (pair != null) { + Runnable r = new Runnable() { + @Override + public void run() { + try { + pair.getLeft().process(channel, msg); + } catch (Throwable ex) { + logger.error("process msg {} error : {}", msg, ex); + } + } + }; + try { + pair.getRight().submit(r); + } catch (RejectedExecutionException e) { + logger.warn("thread pool is full, discard msg {} from {}", msg, ChannelUtils.getRemoteAddress(channel)); + } + } else { + logger.warn("commandType {} not support", commandType); + } + } + + /** + * caught exception + * + * @param ctx channel handler context + * @param cause cause + * @throws Exception + */ + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + logger.error("exceptionCaught : {}", cause); + nettyRemotingClient.removeChannel(ChannelUtils.toAddress(ctx.channel())); + ctx.channel().close(); + } + + /** + * channel write changed + * @param ctx channel handler context + * @throws Exception + */ + @Override + public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception { + Channel ch = ctx.channel(); + ChannelConfig config = ch.config(); + + if (!ch.isWritable()) { + if (logger.isWarnEnabled()) { + logger.warn("{} is not writable, over high water level : {}", + new Object[]{ch, config.getWriteBufferHighWaterMark()}); + } + + config.setAutoRead(false); + } else { + if (logger.isWarnEnabled()) { + logger.warn("{} is writable, to low water : {}", + new Object[]{ch, config.getWriteBufferLowWaterMark()}); + } + config.setAutoRead(true); + } + } +} \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java new file mode 100644 index 0000000000..eabd6560de --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.handler; + +import io.netty.channel.*; +import org.apache.dolphinscheduler.remote.NettyRemotingServer; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; +import org.apache.dolphinscheduler.remote.utils.ChannelUtils; +import org.apache.dolphinscheduler.remote.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; + +/** + * netty server request handler + */ +@ChannelHandler.Sharable +public class NettyServerHandler extends ChannelInboundHandlerAdapter { + + private final Logger logger = LoggerFactory.getLogger(NettyServerHandler.class); + + /** + * netty remote server + */ + private final NettyRemotingServer nettyRemotingServer; + + /** + * server processors queue + */ + private final ConcurrentHashMap> processors = new ConcurrentHashMap(); + + public NettyServerHandler(NettyRemotingServer nettyRemotingServer){ + this.nettyRemotingServer = nettyRemotingServer; + } + + /** + * When the current channel is not active, + * the current channel has reached the end of its life cycle + * @param ctx channel handler context + * @throws Exception + */ + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + ctx.channel().close(); + } + + /** + * The current channel reads data from the remote end + * + * @param ctx channel handler context + * @param msg message + * @throws Exception + */ + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + processReceived(ctx.channel(), (Command)msg); + } + + /** + * register processor + * + * @param commandType command type + * @param processor processor + */ + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) { + this.registerProcessor(commandType, processor, null); + } + + /** + * register processor + * + * @param commandType command type + * @param processor processor + * @param executor thread executor + */ + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + ExecutorService executorRef = executor; + if(executorRef == null){ + executorRef = nettyRemotingServer.getDefaultExecutor(); + } + this.processors.putIfAbsent(commandType, new Pair(processor, executorRef)); + } + + /** + * process received logic + * @param channel channel + * @param msg message + */ + private void processReceived(final Channel channel, final Command msg) { + final CommandType commandType = msg.getType(); + final Pair pair = processors.get(commandType); + if (pair != null) { + Runnable r = new Runnable() { + + @Override + public void run() { + try { + pair.getLeft().process(channel, msg); + } catch (Throwable ex) { + logger.error("process msg {} error : {}", msg, ex); + } + } + }; + try { + pair.getRight().submit(r); + } catch (RejectedExecutionException e) { + logger.warn("thread pool is full, discard msg {} from {}", msg, ChannelUtils.getRemoteAddress(channel)); + } + } else { + logger.warn("commandType {} not support", commandType); + } + } + + /** + * caught exception + * + * @param ctx channel handler context + * @param cause cause + * @throws Exception + */ + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + logger.error("exceptionCaught : {}", cause); + ctx.channel().close(); + } + + /** + * channel write changed + * + * @param ctx channel handler context + * @throws Exception + */ + @Override + public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception { + Channel ch = ctx.channel(); + ChannelConfig config = ch.config(); + + if (!ch.isWritable()) { + if (logger.isWarnEnabled()) { + logger.warn("{} is not writable, over high water level : {}", + new Object[]{ch, config.getWriteBufferHighWaterMark()}); + } + + config.setAutoRead(false); + } else { + if (logger.isWarnEnabled()) { + logger.warn("{} is writable, to low water : {}", + new Object[]{ch, config.getWriteBufferLowWaterMark()}); + } + config.setAutoRead(true); + } + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZkServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java similarity index 65% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZkServer.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java index d1a0526309..6966b53d17 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZkServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java @@ -14,30 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.zk; +package org.apache.dolphinscheduler.remote.processor; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import io.netty.channel.Channel; +import org.apache.dolphinscheduler.remote.command.Command; /** - * demo for using zkServer + * netty request processor */ -public class TestZkServer { +public interface NettyRequestProcessor { - @Before - public void before(){ - ZKServer.start(); - } - - @Test - public void test(){ - Assert.assertTrue(ZKServer.isStarted()); - } - - @After - public void after(){ - ZKServer.stop(); - } + /** + * process logic + * @param channel channel + * @param command command + */ + void process(final Channel channel, final Command command); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Address.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Address.java new file mode 100644 index 0000000000..f61dcd615c --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Address.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.utils; + +import java.io.Serializable; + +/** + * server address + */ +public class Address implements Serializable { + + /** + * host + */ + private String host; + + /** + * port + */ + private int port; + + public Address(){ + //NOP + } + + public Address(String host, int port){ + this.host = host; + this.port = port; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((host == null) ? 0 : host.hashCode()); + result = prime * result + port; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Address other = (Address) obj; + if (host == null) { + if (other.host != null) { + return false; + } + } else if (!host.equals(other.host)) { + return false; + } + return port == other.port; + } + + @Override + public String toString() { + return "Address [host=" + host + ", port=" + port + "]"; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java new file mode 100644 index 0000000000..d7af5fe165 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.utils; + +import io.netty.channel.Channel; + +import java.net.InetSocketAddress; + +/** + * channel utils + */ +public class ChannelUtils { + + /** + * get local address + * + * @param channel channel + * @return local address + */ + public static String getLocalAddress(Channel channel){ + return ((InetSocketAddress)channel.localAddress()).getAddress().getHostAddress(); + } + + /** + * get remote address + * @param channel channel + * @return remote address + */ + public static String getRemoteAddress(Channel channel){ + return ((InetSocketAddress)channel.remoteAddress()).getAddress().getHostAddress(); + } + + /** + * channel to address + * @param channel channel + * @return address + */ + public static Address toAddress(Channel channel){ + InetSocketAddress socketAddress = ((InetSocketAddress)channel.remoteAddress()); + return new Address(socketAddress.getAddress().getHostAddress(), socketAddress.getPort()); + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZk.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java similarity index 66% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZk.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 5c3db2d5d1..5733b17790 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZk.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -14,30 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.zk; +package org.apache.dolphinscheduler.remote.utils; + +import java.nio.charset.Charset; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; /** - * demo for using zkServer + * constant */ -public class TestZk { - - @Before - public void before(){ - ZKServer.start(); - } - - @Test - public void test(){ - Assert.assertTrue(ZKServer.isStarted()); - } - - @After - public void after(){ - ZKServer.stop(); - } +public class Constants { + + public static final String COMMA = ","; + + public static final String SLASH = "/"; + + /** + * charset + */ + public static final Charset UTF8 = Charset.forName("UTF-8"); + + /** + * cpus + */ + public static final int CPUS = Runtime.getRuntime().availableProcessors(); + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java new file mode 100644 index 0000000000..e96796a05c --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.utils; + +import com.alibaba.fastjson.JSON; + +/** + * json serialize or deserialize + */ +public class FastJsonSerializer { + + /** + * serialize to byte + * + * @param obj object + * @param object type + * @return byte array + */ + public static byte[] serialize(T obj) { + String json = JSON.toJSONString(obj); + return json.getBytes(Constants.UTF8); + } + + /** + * serialize to string + * @param obj object + * @param object type + * @return string + */ + public static String serializeToString(T obj) { + return JSON.toJSONString(obj); + } + + /** + * deserialize + * + * @param src byte array + * @param clazz class + * @param deserialize type + * @return deserialize type + */ + public static T deserialize(byte[] src, Class clazz) { + return JSON.parseObject(new String(src, Constants.UTF8), clazz); + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java new file mode 100644 index 0000000000..2042191486 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.utils; + + +/** + * key value pair + * + * @param L generic type + * @param R generic type + */ +public class Pair { + + private L left; + + private R right; + + public Pair(L left, R right) { + this.left = left; + this.right = right; + } + + public L getLeft() { + return left; + } + + public void setLeft(L left) { + this.left = left; + } + + public R getRight() { + return right; + } + + public void setRight(R right) { + this.right = right; + } +} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java new file mode 100644 index 0000000000..6f0a802af6 --- /dev/null +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java @@ -0,0 +1,77 @@ +package org.apache.dolphinscheduler.remote;/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import io.netty.channel.Channel; +import org.apache.dolphinscheduler.remote.NettyRemotingClient; +import org.apache.dolphinscheduler.remote.NettyRemotingServer; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.command.Ping; +import org.apache.dolphinscheduler.remote.command.Pong; +import org.apache.dolphinscheduler.remote.config.NettyClientConfig; +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; +import org.apache.dolphinscheduler.remote.utils.Address; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; + +/** + * netty remote client test + */ +public class NettyRemotingClientTest { + + + /** + * test ping + */ + @Test + public void testSend(){ + NettyServerConfig serverConfig = new NettyServerConfig(); + + NettyRemotingServer server = new NettyRemotingServer(serverConfig); + server.registerProcessor(CommandType.PING, new NettyRequestProcessor() { + @Override + public void process(Channel channel, Command command) { + channel.writeAndFlush(Pong.create(command.getOpaque())); + } + }); + server.start(); + // + CountDownLatch latch = new CountDownLatch(1); + AtomicLong opaque = new AtomicLong(1); + final NettyClientConfig clientConfig = new NettyClientConfig(); + NettyRemotingClient client = new NettyRemotingClient(clientConfig); + client.registerProcessor(CommandType.PONG, new NettyRequestProcessor() { + @Override + public void process(Channel channel, Command command) { + opaque.set(command.getOpaque()); + latch.countDown(); + } + }); + Command commandPing = Ping.create(); + try { + client.send(new Address("127.0.0.1", serverConfig.getListenPort()), commandPing); + latch.await(); + } catch (Exception e) { + e.printStackTrace(); + } + Assert.assertEquals(opaque.get(), commandPing.getOpaque()); + } +} diff --git a/dolphinscheduler-rpc/pom.xml b/dolphinscheduler-rpc/pom.xml deleted file mode 100644 index 680a4a24c0..0000000000 --- a/dolphinscheduler-rpc/pom.xml +++ /dev/null @@ -1,113 +0,0 @@ - - - - - - org.apache.dolphinscheduler - dolphinscheduler - 1.2.1-SNAPSHOT - - 4.0.0 - - dolphinscheduler-rpc - - dolphinscheduler-rpc - https://github.com/apache/incubator-dolphinscheduler - - - UTF-8 - 1.8 - 1.8 - - 3.5.1 - 1.9.0 - - - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - io.grpc - grpc-netty - ${grpc.version} - - - io.grpc - grpc-protobuf - ${grpc.version} - - - io.grpc - grpc-stub - ${grpc.version} - - - - com.google.guava - guava - - - - - - - kr.motd.maven - os-maven-plugin - 1.5.0.Final - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - 0.5.0 - - com.google.protobuf:protoc:3.5.1-1:exe:${os.detected.classifier} - grpc-java - io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} - - - - compile - - compile - - - - compile-custom - - compile-custom - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - ${java.version} - ${java.version} - ${project.build.sourceEncoding} - - - - - diff --git a/dolphinscheduler-rpc/src/main/proto/scheduler.proto b/dolphinscheduler-rpc/src/main/proto/scheduler.proto deleted file mode 100644 index b8b595cb2a..0000000000 --- a/dolphinscheduler-rpc/src/main/proto/scheduler.proto +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -syntax = "proto3"; - -package schduler; - -option java_multiple_files = true; -option java_package = "org.apache.dolphinscheduler.rpc"; -option java_outer_classname = "SchdulerProto"; - - -/** - * return str info - */ -message RetStrInfo { - /** - * str msg info - */ - string msg = 1 ; -} - -/** - * return byte info - */ -message RetByteInfo { - /** - * byte data info - */ - bytes data = 1; -} - -/** - * log parameter - */ -message LogParameter { - - /** - * path - */ - string path = 1 ; - - /** - * skip line num - */ - int32 skipLineNum = 2 ; - - /** - * display limt num - */ - int32 limit = 3 ; -} - - -/** - * path parameter - */ -message PathParameter { - - /** - * path - */ - string path = 1 ; -} - -/** - * log view service - */ -service LogViewService { - - /** - * roll view log - */ - rpc rollViewLog(LogParameter) returns (RetStrInfo) {}; - - /** - * view all log - */ - rpc viewLog(PathParameter) returns (RetStrInfo) {}; - - /** - * get log bytes - */ - rpc getLogBytes(PathParameter) returns (RetByteInfo) {}; -} - diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 751fd919a8..080b87ebaa 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -71,7 +71,7 @@ org.apache.dolphinscheduler - dolphinscheduler-rpc + dolphinscheduler-service org.apache.curator diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java new file mode 100644 index 0000000000..4e4404ea1c --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.log; + +import io.netty.channel.Channel; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.command.log.*; +import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; +import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * logger request process logic + */ +public class LoggerRequestProcessor implements NettyRequestProcessor { + + private final Logger logger = LoggerFactory.getLogger(LoggerRequestProcessor.class); + + private final ThreadPoolExecutor executor; + + public LoggerRequestProcessor(){ + this.executor = new ThreadPoolExecutor(4, 4, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100)); + } + + @Override + public void process(Channel channel, Command command) { + logger.info("received command : {}", command); + + /** + * reuqest task log command type + */ + final CommandType commandType = command.getType(); + switch (commandType){ + case GET_LOG_BYTES_REQUEST: + GetLogBytesRequestCommand getLogRequest = FastJsonSerializer.deserialize( + command.getBody(), GetLogBytesRequestCommand.class); + byte[] bytes = getFileContentBytes(getLogRequest.getPath()); + GetLogBytesResponseCommand getLogResponse = new GetLogBytesResponseCommand(bytes); + channel.writeAndFlush(getLogResponse.convert2Command(command.getOpaque())); + break; + case VIEW_WHOLE_LOG_REQUEST: + ViewLogRequestCommand viewLogRequest = FastJsonSerializer.deserialize( + command.getBody(), ViewLogRequestCommand.class); + String msg = readWholeFileContent(viewLogRequest.getPath()); + ViewLogResponseCommand viewLogResponse = new ViewLogResponseCommand(msg); + channel.writeAndFlush(viewLogResponse.convert2Command(command.getOpaque())); + break; + case ROLL_VIEW_LOG_REQUEST: + RollViewLogRequestCommand rollViewLogRequest = FastJsonSerializer.deserialize( + command.getBody(), RollViewLogRequestCommand.class); + List lines = readPartFileContent(rollViewLogRequest.getPath(), + rollViewLogRequest.getSkipLineNum(), rollViewLogRequest.getLimit()); + StringBuilder builder = new StringBuilder(); + for (String line : lines){ + builder.append(line + "\r\n"); + } + RollViewLogResponseCommand rollViewLogRequestResponse = new RollViewLogResponseCommand(builder.toString()); + channel.writeAndFlush(rollViewLogRequestResponse.convert2Command(command.getOpaque())); + break; + default: + throw new IllegalArgumentException("unknown commandType"); + } + } + + public ExecutorService getExecutor(){ + return this.executor; + } + + /** + * get files content bytes，for down load file + * + * @param filePath file path + * @return byte array of file + * @throws Exception exception + */ + private byte[] getFileContentBytes(String filePath){ + InputStream in = null; + ByteArrayOutputStream bos = null; + try { + in = new FileInputStream(filePath); + bos = new ByteArrayOutputStream(); + byte[] buf = new byte[1024]; + int len; + while ((len = in.read(buf)) != -1) { + bos.write(buf, 0, len); + } + return bos.toByteArray(); + }catch (IOException e){ + logger.error("get file bytes error",e); + }finally { + if (bos != null){ + try { + bos.close(); + } catch (IOException ignore) {} + } + if (in != null){ + try { + in.close(); + } catch (IOException ignore) {} + } + } + return new byte[0]; + } + + /** + * read part file content，can skip any line and read some lines + * + * @param filePath file path + * @param skipLine skip line + * @param limit read lines limit + * @return part file content + */ + private List readPartFileContent(String filePath, + int skipLine, + int limit){ + try (Stream stream = Files.lines(Paths.get(filePath))) { + return stream.skip(skipLine).limit(limit).collect(Collectors.toList()); + } catch (IOException e) { + logger.error("read file error",e); + } + return Collections.EMPTY_LIST; + } + + /** + * read whole file content + * + * @param filePath file path + * @return whole file content + */ + private String readWholeFileContent(String filePath){ + BufferedReader br = null; + String line; + StringBuilder sb = new StringBuilder(); + try { + br = new BufferedReader(new InputStreamReader(new FileInputStream(filePath))); + while ((line = br.readLine()) != null){ + sb.append(line + "\r\n"); + } + return sb.toString(); + }catch (IOException e){ + logger.error("read file error",e); + }finally { + try { + if (br != null){ + br.close(); + } + } catch (IOException ignore) {} + } + return ""; + } +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java new file mode 100644 index 0000000000..3520fb09ec --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.log; + + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.remote.NettyRemotingServer; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * logger server + */ +public class LoggerServer { + + private static final Logger logger = LoggerFactory.getLogger(LoggerServer.class); + + /** + * netty server + */ + private final NettyRemotingServer server; + + /** + * netty server config + */ + private final NettyServerConfig serverConfig; + + /** + * loggger request processor + */ + private final LoggerRequestProcessor requestProcessor; + + public LoggerServer(){ + this.serverConfig = new NettyServerConfig(); + this.serverConfig.setListenPort(Constants.RPC_PORT); + this.server = new NettyRemotingServer(serverConfig); + this.requestProcessor = new LoggerRequestProcessor(); + this.server.registerProcessor(CommandType.GET_LOG_BYTES_REQUEST, requestProcessor, requestProcessor.getExecutor()); + this.server.registerProcessor(CommandType.ROLL_VIEW_LOG_REQUEST, requestProcessor, requestProcessor.getExecutor()); + this.server.registerProcessor(CommandType.VIEW_WHOLE_LOG_REQUEST, requestProcessor, requestProcessor.getExecutor()); + } + + /** + * main launches the server from the command line. + * @param args arguments + */ + public static void main(String[] args) { + final LoggerServer server = new LoggerServer(); + server.start(); + } + + /** + * server start + */ + public void start() { + this.server.start(); + logger.info("logger server started, listening on port : {}" , Constants.RPC_PORT); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + LoggerServer.this.stop(); + } + }); + } + + /** + * stop + */ + public void stop() { + this.server.close(); + logger.info("logger server shut down"); + } + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index ab4ba5c8ab..6b5063cba4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -22,14 +22,14 @@ import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread; -import org.apache.dolphinscheduler.dao.quartz.ProcessScheduleJob; -import org.apache.dolphinscheduler.dao.quartz.QuartzExecutors; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.quartz.ProcessScheduleJob; +import org.apache.dolphinscheduler.service.quartz.QuartzExecutors; import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,10 +66,10 @@ public class MasterServer implements IStoppable { private ScheduledExecutorService heartbeatMasterService; /** - * dolphinscheduler database interface + * process service */ @Autowired - protected ProcessDao processDao; + protected ProcessService processService; /** * master exec thread pool @@ -77,17 +77,18 @@ public class MasterServer implements IStoppable { private ExecutorService masterSchedulerService; /** - * spring application context - * only use it for initialization + * master config */ @Autowired - private SpringApplicationContext springApplicationContext; + private MasterConfig masterConfig; + /** - * master config + * spring application context + * only use it for initialization */ @Autowired - private MasterConfig masterConfig; + private SpringApplicationContext springApplicationContext; /** @@ -126,7 +127,7 @@ public class MasterServer implements IStoppable { // master scheduler thread MasterSchedulerThread masterSchedulerThread = new MasterSchedulerThread( zkMasterClient, - processDao, + processService, masterConfig.getMasterExecThreads()); // submit master scheduler thread @@ -136,7 +137,7 @@ public class MasterServer implements IStoppable { // what system should do if exception try { logger.info("start Quartz server..."); - ProcessScheduleJob.init(processDao); + ProcessScheduleJob.init(processService); QuartzExecutors.getInstance().start(); } catch (Exception e) { try { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index c1552c4621..f8fcb1456d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -16,15 +16,15 @@ */ package org.apache.dolphinscheduler.server.master.runner; -import org.apache.dolphinscheduler.common.queue.ITaskQueue; -import org.apache.dolphinscheduler.common.queue.TaskQueueFactory; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.AlertDao; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.utils.BeanContext; import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.queue.ITaskQueue; +import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,9 +41,9 @@ public class MasterBaseTaskExecThread implements Callable { private static final Logger logger = LoggerFactory.getLogger(MasterBaseTaskExecThread.class); /** - * process dao + * process service */ - protected ProcessDao processDao; + protected ProcessService processService; /** * alert database access @@ -81,7 +81,7 @@ public class MasterBaseTaskExecThread implements Callable { * @param processInstance process instance */ public MasterBaseTaskExecThread(TaskInstance taskInstance, ProcessInstance processInstance){ - this.processDao = BeanContext.getBean(ProcessDao.class); + this.processService = BeanContext.getBean(ProcessService.class); this.alertDao = BeanContext.getBean(AlertDao.class); this.processInstance = processInstance; this.taskQueue = TaskQueueFactory.getTaskQueueInstance(); @@ -121,14 +121,14 @@ public class MasterBaseTaskExecThread implements Callable { try { if(!submitDB){ // submit task to db - task = processDao.submitTask(taskInstance, processInstance); + task = processService.submitTask(taskInstance, processInstance); if(task != null && task.getId() != 0){ submitDB = true; } } if(submitDB && !submitQueue){ // submit task to queue - submitQueue = processDao.submitTaskToQueue(task); + submitQueue = processService.submitTaskToQueue(task); } if(submitDB && submitQueue){ return task; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 2b1ff4d23f..f5e31210a0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -28,14 +28,15 @@ import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.utils.DagHelper; -import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.utils.AlertManager; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -124,9 +125,9 @@ public class MasterExecThread implements Runnable { private DAG dag; /** - * process dao + * process service */ - private ProcessDao processDao; + private ProcessService processService; /** * master config @@ -136,10 +137,10 @@ public class MasterExecThread implements Runnable { /** * constructor of MasterExecThread * @param processInstance process instance - * @param processDao process dao + * @param processService process dao */ - public MasterExecThread(ProcessInstance processInstance,ProcessDao processDao){ - this.processDao = processDao; + public MasterExecThread(ProcessInstance processInstance, ProcessService processService){ + this.processService = processService; this.processInstance = processInstance; this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class); @@ -177,7 +178,7 @@ public class MasterExecThread implements Runnable { logger.error("process execute failed, process id:{}", processInstance.getId()); processInstance.setState(ExecutionStatus.FAILURE); processInstance.setEndTime(new Date()); - processDao.updateProcessInstance(processInstance); + processService.updateProcessInstance(processInstance); }finally { taskExecService.shutdown(); // post handle @@ -205,11 +206,11 @@ public class MasterExecThread implements Runnable { Date startDate = DateUtils.getScheduleDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE)); Date endDate = DateUtils.getScheduleDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE)); - processDao.saveProcessInstance(processInstance); + processService.saveProcessInstance(processInstance); // get schedules int processDefinitionId = processInstance.getProcessDefinitionId(); - List schedules = processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId); + List schedules = processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId); List listDate = Lists.newLinkedList(); if(!CollectionUtils.isEmpty(schedules)){ for (Schedule schedule : schedules) { @@ -223,7 +224,7 @@ public class MasterExecThread implements Runnable { iterator = listDate.iterator(); scheduleDate = iterator.next(); processInstance.setScheduleTime(scheduleDate); - processDao.updateProcessInstance(processInstance); + processService.updateProcessInstance(processInstance); }else{ scheduleDate = processInstance.getScheduleTime(); if(scheduleDate == null){ @@ -239,7 +240,7 @@ public class MasterExecThread implements Runnable { logger.error("process {} dag is null, please check out parameters", processInstance.getId()); processInstance.setState(ExecutionStatus.SUCCESS); - processDao.updateProcessInstance(processInstance); + processService.updateProcessInstance(processInstance); return; } @@ -281,10 +282,10 @@ public class MasterExecThread implements Runnable { processInstance.setCommandParam(JSONUtils.toJson(cmdParam)); } - List taskInstanceList = processDao.findValidTaskListByProcessId(processInstance.getId()); + List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); for(TaskInstance taskInstance : taskInstanceList){ taskInstance.setFlag(Flag.NO); - processDao.updateTaskInstance(taskInstance); + processService.updateTaskInstance(taskInstance); } processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); processInstance.setGlobalParams(ParameterUtils.curingGlobalParams( @@ -292,7 +293,7 @@ public class MasterExecThread implements Runnable { processInstance.getProcessDefinition().getGlobalParamList(), CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime())); - processDao.saveProcessInstance(processInstance); + processService.saveProcessInstance(processInstance); } // flow end @@ -320,11 +321,11 @@ public class MasterExecThread implements Runnable { */ private void endProcess() { processInstance.setEndTime(new Date()); - processDao.updateProcessInstance(processInstance); + processService.updateProcessInstance(processInstance); if(processInstance.getState().typeIsWaittingThread()){ - processDao.createRecoveryWaitingThreadCommand(null, processInstance); + processService.createRecoveryWaitingThreadCommand(null, processInstance); } - List taskInstances = processDao.findValidTaskListByProcessId(processInstance.getId()); + List taskInstances = processService.findValidTaskListByProcessId(processInstance.getId()); alertManager.sendAlertProcessInstance(processInstance, taskInstances); } @@ -361,7 +362,7 @@ public class MasterExecThread implements Runnable { dependFailedTask.clear(); completeTaskList.clear(); errorTaskList.clear(); - List taskInstanceList = processDao.findValidTaskListByProcessId(processInstance.getId()); + List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); for(TaskInstance task : taskInstanceList){ if(task.isTaskComplete()){ completeTaskList.put(task.getName(), task); @@ -417,7 +418,7 @@ public class MasterExecThread implements Runnable { * @return TaskInstance */ private TaskInstance findTaskIfExists(String taskName){ - List taskInstanceList = processDao.findValidTaskListByProcessId(this.processInstance.getId()); + List taskInstanceList = processService.findValidTaskListByProcessId(this.processInstance.getId()); for(TaskInstance taskInstance : taskInstanceList){ if(taskInstance.getName().equals(taskName)){ return taskInstance; @@ -706,7 +707,7 @@ public class MasterExecThread implements Runnable { * @return process instance execution status */ private ExecutionStatus getProcessInstanceState(){ - ProcessInstance instance = processDao.findProcessInstanceById(processInstance.getId()); + ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId()); ExecutionStatus state = instance.getState(); if(activeTaskNode.size() > 0){ @@ -784,10 +785,10 @@ public class MasterExecThread implements Runnable { processInstance.getState().toString(), state.toString(), processInstance.getCommandType().toString()); processInstance.setState(state); - ProcessInstance instance = processDao.findProcessInstanceById(processInstance.getId()); + ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId()); instance.setState(state); instance.setProcessDefinition(processInstance.getProcessDefinition()); - processDao.updateProcessInstance(instance); + processService.updateProcessInstance(instance); processInstance = instance; } } @@ -845,7 +846,7 @@ public class MasterExecThread implements Runnable { // send warning email if process time out. if( !sendTimeWarning && checkProcessTimeOut(processInstance) ){ alertManager.sendProcessTimeoutAlert(processInstance, - processDao.findProcessDefineById(processInstance.getProcessDefinitionId())); + processService.findProcessDefineById(processInstance.getProcessDefinitionId())); sendTimeWarning = true; } for(Map.Entry> entry: activeTaskNode.entrySet()) { @@ -903,7 +904,7 @@ public class MasterExecThread implements Runnable { if(completeTask.getState()== ExecutionStatus.PAUSE){ completeTask.setState(ExecutionStatus.KILL); completeTaskList.put(entry.getKey(), completeTask); - processDao.updateTaskInstance(completeTask); + processService.updateTaskInstance(completeTask); } } } @@ -961,7 +962,7 @@ public class MasterExecThread implements Runnable { Future future = entry.getValue(); TaskInstance taskInstance = taskExecThread.getTaskInstance(); - taskInstance = processDao.findTaskInstanceById(taskInstance.getId()); + taskInstance = processService.findTaskInstanceById(taskInstance.getId()); if(taskInstance.getState().typeIsFinished()){ continue; } @@ -1031,7 +1032,7 @@ public class MasterExecThread implements Runnable { } try { Integer intId = Integer.valueOf(taskId); - TaskInstance task = processDao.findTaskInstanceById(intId); + TaskInstance task = processService.findTaskInstanceById(intId); if(task == null){ logger.error("start node id cannot be found: {}", taskId); }else { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java index a873fb786d..c0ddb1cb5c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java @@ -22,13 +22,13 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.common.zk.AbstractZKClient; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.zk.AbstractZKClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +53,7 @@ public class MasterSchedulerThread implements Runnable { /** * dolphinscheduler database interface */ - private final ProcessDao processDao; + private final ProcessService processService; /** * zookeeper master client @@ -74,11 +74,11 @@ public class MasterSchedulerThread implements Runnable { /** * constructor of MasterSchedulerThread * @param zkClient zookeeper master client - * @param processDao process dao + * @param processService process service * @param masterExecThreadNum master exec thread num */ - public MasterSchedulerThread(ZKMasterClient zkClient, ProcessDao processDao, int masterExecThreadNum){ - this.processDao = processDao; + public MasterSchedulerThread(ZKMasterClient zkClient, ProcessService processService, int masterExecThreadNum){ + this.processService = processService; this.zkMasterClient = zkClient; this.masterExecThreadNum = masterExecThreadNum; this.masterExecService = ThreadUtils.newDaemonFixedThreadExecutor("Master-Exec-Thread",masterExecThreadNum); @@ -115,19 +115,19 @@ public class MasterSchedulerThread implements Runnable { ThreadPoolExecutor poolExecutor = (ThreadPoolExecutor) masterExecService; int activeCount = poolExecutor.getActiveCount(); // make sure to scan and delete command table in one transaction - Command command = processDao.findOneCommand(); + Command command = processService.findOneCommand(); if (command != null) { logger.info(String.format("find one command: id: %d, type: %s", command.getId(),command.getCommandType().toString())); try{ - processInstance = processDao.handleCommand(logger, OSUtils.getHost(), this.masterExecThreadNum - activeCount, command); + processInstance = processService.handleCommand(logger, OSUtils.getHost(), this.masterExecThreadNum - activeCount, command); if (processInstance != null) { logger.info("start master exec thread , split DAG ..."); - masterExecService.execute(new MasterExecThread(processInstance,processDao)); + masterExecService.execute(new MasterExecThread(processInstance, processService)); } }catch (Exception e){ logger.error("scan command error ", e); - processDao.moveToErrorCommand(command, e.toString()); + processService.moveToErrorCommand(command, e.toString()); } } else{ //indicate that no command ,sleep for 1s diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index f2ee66b64a..66d1a3f4c2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -82,7 +82,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { result = waitTaskQuit(); } taskInstance.setEndTime(new Date()); - processDao.updateTaskInstance(taskInstance); + processService.updateTaskInstance(taskInstance); logger.info("task :{} id:{}, process id:{}, exec thread completed ", this.taskInstance.getName(),taskInstance.getId(), processInstance.getId() ); return result; @@ -94,7 +94,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { */ public Boolean waitTaskQuit(){ // query new state - taskInstance = processDao.findTaskInstanceById(taskInstance.getId()); + taskInstance = processService.findTaskInstanceById(taskInstance.getId()); logger.info("wait task: process id: {}, task id:{}, task name:{} complete", this.taskInstance.getProcessInstanceId(), this.taskInstance.getId(), this.taskInstance.getName()); // task time out @@ -126,15 +126,15 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { if (remainTime < 0) { logger.warn("task id: {} execution time out",taskInstance.getId()); // process define - ProcessDefinition processDefine = processDao.findProcessDefineById(processInstance.getProcessDefinitionId()); + ProcessDefinition processDefine = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); // send warn mail alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(),processDefine.getReceiversCc(),taskInstance.getId(),taskInstance.getName()); checkTimeout = false; } } // updateProcessInstance task instance - taskInstance = processDao.findTaskInstanceById(taskInstance.getId()); - processInstance = processDao.findProcessInstanceById(processInstance.getId()); + taskInstance = processService.findTaskInstanceById(taskInstance.getId()); + processInstance = processService.findProcessInstanceById(processInstance.getId()); Thread.sleep(Constants.SLEEP_TIME_MILLIS); } catch (Exception e) { logger.error("exception",e); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java index 0026de7c25..fc16b5112b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java @@ -64,7 +64,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { } setTaskInstanceState(); waitTaskQuit(); - subProcessInstance = processDao.findSubProcessInstance(processInstance.getId(), taskInstance.getId()); + subProcessInstance = processService.findSubProcessInstance(processInstance.getId(), taskInstance.getId()); // at the end of the subflow , the task state is changed to the subflow state if(subProcessInstance != null){ @@ -75,7 +75,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { } } taskInstance.setEndTime(new Date()); - processDao.updateTaskInstance(taskInstance); + processService.updateTaskInstance(taskInstance); logger.info("subflow task :{} id:{}, process id:{}, exec thread completed ", this.taskInstance.getName(),taskInstance.getId(), processInstance.getId() ); result = true; @@ -96,14 +96,14 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { * @return */ private Boolean setTaskInstanceState(){ - subProcessInstance = processDao.findSubProcessInstance(processInstance.getId(), taskInstance.getId()); + subProcessInstance = processService.findSubProcessInstance(processInstance.getId(), taskInstance.getId()); if(subProcessInstance == null || taskInstance.getState().typeIsFinished()){ return false; } taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); taskInstance.setStartTime(new Date()); - processDao.updateTaskInstance(taskInstance); + processService.updateTaskInstance(taskInstance); return true; } @@ -111,7 +111,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { * updateProcessInstance parent state */ private void updateParentProcessState(){ - ProcessInstance parentProcessInstance = processDao.findProcessInstanceById(this.processInstance.getId()); + ProcessInstance parentProcessInstance = processService.findProcessInstanceById(this.processInstance.getId()); if(parentProcessInstance == null){ logger.error("parent work flow instance is null , please check it! work flow id {}", processInstance.getId()); @@ -145,7 +145,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { continue; } } - subProcessInstance = processDao.findProcessInstanceById(subProcessInstance.getId()); + subProcessInstance = processService.findProcessInstanceById(subProcessInstance.getId()); updateParentProcessState(); if (subProcessInstance.getState().typeIsFinished()){ break; @@ -171,7 +171,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { return; } subProcessInstance.setState(ExecutionStatus.READY_STOP); - processDao.updateProcessInstance(subProcessInstance); + processService.updateProcessInstance(subProcessInstance); } /** @@ -183,6 +183,6 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { return; } subProcessInstance.setState(ExecutionStatus.READY_PAUSE); - processDao.updateProcessInstance(subProcessInstance); + processService.updateProcessInstance(subProcessInstance); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/ZKMonitorImpl.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/ZKMonitorImpl.java index 927074012d..5acc8fd931 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/ZKMonitorImpl.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/ZKMonitorImpl.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.monitor; -import org.apache.dolphinscheduler.common.zk.ZookeeperOperator; +import org.apache.dolphinscheduler.service.zk.ZookeeperOperator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java deleted file mode 100644 index 1c6c97b88f..0000000000 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.server.rpc; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import org.apache.dolphinscheduler.rpc.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; - -/** - * log client - */ -public class LogClient { - - /** - * logger of LogClient - */ - private static final Logger logger = LoggerFactory.getLogger(LogClient.class); - - /** - * managed channel - */ - private final ManagedChannel channel; - - /** - * blocking stub - */ - private final LogViewServiceGrpc.LogViewServiceBlockingStub blockingStub; - - /** - * Construct client connecting to HelloWorld server at host:port. - * - * @param host host - * @param port port - */ - public LogClient(String host, int port) { - this(ManagedChannelBuilder.forAddress(host, port) - // Channels are secure by default (via SSL/TLS). For the example we disable TLS to avoid - // needing certificates. - .usePlaintext(true)); - } - - /** - * Construct client for accessing RouteGuide server using the existing channel. - * - * @param channelBuilder channel builder - */ - LogClient(ManagedChannelBuilder channelBuilder) { - /** - * set max message read size - */ - channelBuilder.maxInboundMessageSize(Integer.MAX_VALUE); - channel = channelBuilder.build(); - blockingStub = LogViewServiceGrpc.newBlockingStub(channel); - } - - /** - * shut down channel - * - * @throws InterruptedException interrupted exception - */ - public void shutdown() throws InterruptedException { - channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); - } - - /** - * roll view log - * - * @param path log path - * @param skipLineNum skip line num - * @param limit limit - * @return log content - */ - public String rollViewLog(String path,int skipLineNum,int limit) { - logger.info("roll view log , path : {},skipLineNum : {} ,limit :{}", path, skipLineNum, limit); - LogParameter pathParameter = LogParameter - .newBuilder() - .setPath(path) - .setSkipLineNum(skipLineNum) - .setLimit(limit) - .build(); - RetStrInfo retStrInfo; - try { - retStrInfo = blockingStub.rollViewLog(pathParameter); - return retStrInfo.getMsg(); - } catch (StatusRuntimeException e) { - logger.error("roll view log failed", e); - return null; - } - } - - /** - * view all log - * - * @param path log path - * @return log content - */ - public String viewLog(String path) { - logger.info("view log path : {}",path); - - PathParameter pathParameter = PathParameter.newBuilder().setPath(path).build(); - RetStrInfo retStrInfo; - try { - retStrInfo = blockingStub.viewLog(pathParameter); - return retStrInfo.getMsg(); - } catch (StatusRuntimeException e) { - logger.error("view log failed", e); - return null; - } - } - - /** - * get log bytes - * - * @param path log path - * @return log content - */ - public byte[] getLogBytes(String path) { - logger.info("get log bytes {}",path); - - PathParameter pathParameter = PathParameter.newBuilder().setPath(path).build(); - RetByteInfo retByteInfo; - try { - retByteInfo = blockingStub.getLogBytes(pathParameter); - return retByteInfo.getData().toByteArray(); - } catch (StatusRuntimeException e) { - logger.error("get log bytes failed ", e); - return null; - } - } -} \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LoggerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LoggerServer.java deleted file mode 100644 index 5ec5df92fc..0000000000 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LoggerServer.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.server.rpc; - -import io.grpc.stub.StreamObserver; -import org.apache.dolphinscheduler.common.Constants; -import com.google.protobuf.ByteString; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import org.apache.dolphinscheduler.rpc.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * logger server - */ -public class LoggerServer { - - private static final Logger logger = LoggerFactory.getLogger(LoggerServer.class); - - /** - * server - */ - private Server server; - - /** - * server start - * @throws IOException io exception - */ - public void start() throws IOException { - /* The port on which the server should run */ - int port = Constants.RPC_PORT; - server = ServerBuilder.forPort(port) - .addService(new LogViewServiceGrpcImpl()) - .build() - .start(); - logger.info("server started, listening on port : {}" , port); - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - // Use stderr here since the logger may have been reset by its JVM shutdown hook. - logger.info("shutting down gRPC server since JVM is shutting down"); - LoggerServer.this.stop(); - logger.info("server shut down"); - } - }); - } - - /** - * stop - */ - private void stop() { - if (server != null) { - server.shutdown(); - } - } - - /** - * await termination on the main thread since the grpc library uses daemon threads. - */ - private void blockUntilShutdown() throws InterruptedException { - if (server != null) { - server.awaitTermination(); - } - } - - /** - * main launches the server from the command line. - */ - - /** - * main launches the server from the command line. - * @param args arguments - * @throws IOException io exception - * @throws InterruptedException interrupted exception - */ - public static void main(String[] args) throws IOException, InterruptedException { - final LoggerServer server = new LoggerServer(); - server.start(); - server.blockUntilShutdown(); - } - - /** - * Log View Service Grpc Implementation - */ - static class LogViewServiceGrpcImpl extends LogViewServiceGrpc.LogViewServiceImplBase { - @Override - public void rollViewLog(LogParameter request, StreamObserver responseObserver) { - - logger.info("log parameter path : {} ,skip line : {}, limit : {}", - request.getPath(), - request.getSkipLineNum(), - request.getLimit()); - List list = readFile(request.getPath(), request.getSkipLineNum(), request.getLimit()); - StringBuilder sb = new StringBuilder(); - boolean errorLineFlag = false; - for (String line : list){ - sb.append(line + "\r\n"); - } - RetStrInfo retInfoBuild = RetStrInfo.newBuilder().setMsg(sb.toString()).build(); - responseObserver.onNext(retInfoBuild); - responseObserver.onCompleted(); - } - - @Override - public void viewLog(PathParameter request, StreamObserver responseObserver) { - logger.info("task path is : {} " , request.getPath()); - RetStrInfo retInfoBuild = RetStrInfo.newBuilder().setMsg(readFile(request.getPath())).build(); - responseObserver.onNext(retInfoBuild); - responseObserver.onCompleted(); - } - - @Override - public void getLogBytes(PathParameter request, StreamObserver responseObserver) { - try { - ByteString bytes = ByteString.copyFrom(getFileBytes(request.getPath())); - RetByteInfo.Builder builder = RetByteInfo.newBuilder(); - builder.setData(bytes); - responseObserver.onNext(builder.build()); - responseObserver.onCompleted(); - }catch (Exception e){ - logger.error("get log bytes failed",e); - } - } - } - - /** - * get files bytes - * - * @param path path - * @return byte array of file - * @throws Exception exception - */ - private static byte[] getFileBytes(String path){ - InputStream in = null; - ByteArrayOutputStream bos = null; - try { - in = new FileInputStream(path); - bos = new ByteArrayOutputStream(); - byte[] buf = new byte[1024]; - int len = 0; - while ((len = in.read(buf)) != -1) { - bos.write(buf, 0, len); - } - return bos.toByteArray(); - }catch (IOException e){ - logger.error("get file bytes error",e); - }finally { - if (bos != null){ - try { - bos.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - if (in != null){ - try { - in.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - return null; - } - - /** - * read file content - * - * @param path - * @param skipLine - * @param limit - * @return - */ - private static List readFile(String path,int skipLine,int limit){ - try (Stream stream = Files.lines(Paths.get(path))) { - return stream.skip(skipLine).limit(limit).collect(Collectors.toList()); - } catch (IOException e) { - logger.error("read file failed",e); - } - return null; - } - - /** - * read file content - * - * @param path path - * @return string of file content - * @throws Exception exception - */ - private static String readFile(String path){ - BufferedReader br = null; - String line = null; - StringBuilder sb = new StringBuilder(); - try { - br = new BufferedReader(new InputStreamReader(new FileInputStream(path))); - boolean errorLineFlag = false; - while ((line = br.readLine()) != null){ - sb.append(line + "\r\n"); - } - - return sb.toString(); - }catch (IOException e){ - logger.error("read file failed",e); - }finally { - try { - if (br != null){ - br.close(); - } - } catch (IOException e) { - logger.error(e.getMessage(),e); - } - } - return null; - } - -} \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index fd0a08cd8e..90711e1d14 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -22,8 +22,8 @@ import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.rpc.LogClient; import org.apache.commons.io.FileUtils; +import org.apache.dolphinscheduler.service.log.LogClientService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -375,9 +375,16 @@ public class ProcessUtils { public static void killYarnJob(TaskInstance taskInstance) { try { Thread.sleep(Constants.SLEEP_TIME_MILLIS); - LogClient logClient = new LogClient(taskInstance.getHost(), Constants.RPC_PORT); - - String log = logClient.viewLog(taskInstance.getLogPath()); + LogClientService logClient = null; + String log = null; + try { + logClient = new LogClientService(taskInstance.getHost(), Constants.RPC_PORT); + log = logClient.viewLog(taskInstance.getLogPath()); + } finally { + if(logClient != null){ + logClient.close(); + } + } if (StringUtils.isNotEmpty(log)) { List appIds = LoggerUtils.getAppIds(log, logger); String workerDir = taskInstance.getExecutePath(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java index 7264c2f59d..5550e750b5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.utils; -import org.apache.dolphinscheduler.common.zk.ZookeeperOperator; +import org.apache.dolphinscheduler.service.zk.ZookeeperOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 99d418f048..ace93079ff 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -22,22 +22,22 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskType; -import org.apache.dolphinscheduler.common.queue.ITaskQueue; -import org.apache.dolphinscheduler.common.queue.TaskQueueFactory; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.AlertDao; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.FetchTaskThread; import org.apache.dolphinscheduler.server.zk.ZKWorkerClient; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.queue.ITaskQueue; +import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; +import org.apache.dolphinscheduler.service.zk.AbstractZKClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -73,10 +73,10 @@ public class WorkerServer implements IStoppable { /** - * process database access + * process service */ @Autowired - private ProcessDao processDao; + private ProcessService processService; /** * alert database access @@ -104,13 +104,6 @@ public class WorkerServer implements IStoppable { */ private ExecutorService fetchTaskExecutorService; - /** - * spring application context - * only use it for initialization - */ - @Autowired - private SpringApplicationContext springApplicationContext; - /** * CountDownLatch latch */ @@ -122,6 +115,13 @@ public class WorkerServer implements IStoppable { @Autowired private WorkerConfig workerConfig; + /** + * spring application context + * only use it for initialization + */ + @Autowired + private SpringApplicationContext springApplicationContext; + /** * master server startup * @@ -167,7 +167,7 @@ public class WorkerServer implements IStoppable { killExecutorService.execute(killProcessThread); // new fetch task thread - FetchTaskThread fetchTaskThread = new FetchTaskThread(zkWorkerClient, processDao, taskQueue); + FetchTaskThread fetchTaskThread = new FetchTaskThread(zkWorkerClient, processService, taskQueue); // submit fetch task thread fetchTaskExecutorService.execute(fetchTaskThread); @@ -297,7 +297,7 @@ public class WorkerServer implements IStoppable { Set taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL); if (CollectionUtils.isNotEmpty(taskInfoSet)){ for (String taskInfo : taskInfoSet){ - killTask(taskInfo, processDao); + killTask(taskInfo, processService); removeKillInfoFromQueue(taskInfo); } } @@ -319,7 +319,7 @@ public class WorkerServer implements IStoppable { * @param taskInfo task info * @param pd process dao */ - private void killTask(String taskInfo, ProcessDao pd) { + private void killTask(String taskInfo, ProcessService pd) { logger.info("get one kill command from tasks kill queue: " + taskInfo); String[] taskInfoArray = taskInfo.split("-"); if(taskInfoArray.length != 2){ @@ -357,7 +357,7 @@ public class WorkerServer implements IStoppable { * @param taskInstance * @param pd process dao */ - private void deleteTaskFromQueue(TaskInstance taskInstance, ProcessDao pd){ + private void deleteTaskFromQueue(TaskInstance taskInstance, ProcessService pd){ // creating distributed locks, lock path /dolphinscheduler/lock/worker InterProcessMutex mutex = null; logger.info("delete task from tasks queue: " + taskInstance.getId()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java index 221ad069bb..013db83761 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java @@ -19,17 +19,18 @@ package org.apache.dolphinscheduler.server.worker.runner; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.queue.ITaskQueue; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.common.zk.AbstractZKClient; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.zk.ZKWorkerClient; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.queue.ITaskQueue; +import org.apache.dolphinscheduler.service.zk.AbstractZKClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,7 +64,7 @@ public class FetchTaskThread implements Runnable{ /** * process database access */ - private final ProcessDao processDao; + private final ProcessService processService; /** * worker thread pool executor @@ -91,10 +92,10 @@ public class FetchTaskThread implements Runnable{ private WorkerConfig workerConfig; public FetchTaskThread(ZKWorkerClient zkWorkerClient, - ProcessDao processDao, + ProcessService processService, ITaskQueue taskQueue){ this.zkWorkerClient = zkWorkerClient; - this.processDao = processDao; + this.processService = processService; this.taskQueue = taskQueue; this.workerConfig = SpringApplicationContext.getBean(WorkerConfig.class); this.taskNum = workerConfig.getWorkerFetchTaskNum(); @@ -112,12 +113,12 @@ public class FetchTaskThread implements Runnable{ */ private boolean checkWorkerGroup(TaskInstance taskInstance, String host){ - int taskWorkerGroupId = processDao.getTaskWorkerGroupId(taskInstance); + int taskWorkerGroupId = processService.getTaskWorkerGroupId(taskInstance); if(taskWorkerGroupId <= 0){ return true; } - WorkerGroup workerGroup = processDao.queryWorkerGroupById(taskWorkerGroupId); + WorkerGroup workerGroup = processService.queryWorkerGroupById(taskWorkerGroupId); if(workerGroup == null ){ logger.info("task {} cannot find the worker group, use all worker instead.", taskInstance.getId()); return true; @@ -184,7 +185,7 @@ public class FetchTaskThread implements Runnable{ // mainly to wait for the master insert task to succeed waitForTaskInstance(); - taskInstance = processDao.getTaskInstanceDetailByTaskId(taskInstId); + taskInstance = processService.getTaskInstanceDetailByTaskId(taskInstId); // verify task instance is null if (verifyTaskInstanceIsNull(taskInstance)) { @@ -200,7 +201,7 @@ public class FetchTaskThread implements Runnable{ // if process definition is null ,process definition already deleted int userId = taskInstance.getProcessDefine() == null ? 0 : taskInstance.getProcessDefine().getUserId(); - Tenant tenant = processDao.getTenantForProcess( + Tenant tenant = processService.getTenantForProcess( taskInstance.getProcessInstance().getTenantId(), userId); @@ -212,7 +213,7 @@ public class FetchTaskThread implements Runnable{ } // set queue for process instance, user-specified queue takes precedence over tenant queue - String userQueue = processDao.queryUserQueueByProcessInstanceId(taskInstance.getProcessInstanceId()); + String userQueue = processService.queryUserQueueByProcessInstanceId(taskInstance.getProcessInstanceId()); taskInstance.getProcessInstance().setQueue(StringUtils.isEmpty(userQueue) ? tenant.getQueue() : userQueue); taskInstance.getProcessInstance().setTenantCode(tenant.getTenantCode()); @@ -234,7 +235,7 @@ public class FetchTaskThread implements Runnable{ logger.info("task : {} ready to submit to task scheduler thread",taskInstId); // submit task - workerExecService.submit(new TaskScheduleThread(taskInstance, processDao)); + workerExecService.submit(new TaskScheduleThread(taskInstance, processService)); // remove node from zk removeNodeFromTaskQueue(taskQueueStr); @@ -259,7 +260,7 @@ public class FetchTaskThread implements Runnable{ removeNodeFromTaskQueue(taskQueueStr); if (taskInstance != null){ - processDao.changeTaskState(ExecutionStatus.FAILURE, + processService.changeTaskState(ExecutionStatus.FAILURE, taskInstance.getStartTime(), taskInstance.getHost(), null, @@ -347,7 +348,7 @@ public class FetchTaskThread implements Runnable{ int retryTimes = 30; while (taskInstance == null && retryTimes > 0) { Thread.sleep(Constants.SLEEP_TIME_MILLIS); - taskInstance = processDao.findTaskInstanceById(taskInstId); + taskInstance = processService.findTaskInstanceById(taskInstId); retryTimes--; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java index f179d6344a..a69cffd58d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java @@ -31,15 +31,15 @@ import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.dao.permission.PermissionCheck; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.permission.PermissionCheck; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,9 +64,9 @@ public class TaskScheduleThread implements Runnable { private TaskInstance taskInstance; /** - * process database access + * process service */ - private final ProcessDao processDao; + private final ProcessService processService; /** * abstract task @@ -77,10 +77,10 @@ public class TaskScheduleThread implements Runnable { * constructor * * @param taskInstance task instance - * @param processDao process dao + * @param processService process dao */ - public TaskScheduleThread(TaskInstance taskInstance, ProcessDao processDao){ - this.processDao = processDao; + public TaskScheduleThread(TaskInstance taskInstance, ProcessService processService){ + this.processService = processService; this.taskInstance = taskInstance; } @@ -152,7 +152,7 @@ public class TaskScheduleThread implements Runnable { logger.error("task scheduler failure", e); kill(); // update task instance state - processDao.changeTaskState(ExecutionStatus.FAILURE, + processService.changeTaskState(ExecutionStatus.FAILURE, new Date(), taskInstance.getId()); } @@ -161,7 +161,7 @@ public class TaskScheduleThread implements Runnable { taskInstance.getId(), task.getExitStatus()); // update task instance state - processDao.changeTaskState(task.getExitStatus(), + processService.changeTaskState(task.getExitStatus(), new Date(), taskInstance.getId()); } @@ -191,14 +191,14 @@ public class TaskScheduleThread implements Runnable { // update task status is running if(taskType.equals(TaskType.SQL.name()) || taskType.equals(TaskType.PROCEDURE.name())){ - processDao.changeTaskState(ExecutionStatus.RUNNING_EXEUTION, + processService.changeTaskState(ExecutionStatus.RUNNING_EXEUTION, taskInstance.getStartTime(), taskInstance.getHost(), null, getTaskLogPath(), taskInstance.getId()); }else{ - processDao.changeTaskState(ExecutionStatus.RUNNING_EXEUTION, + processService.changeTaskState(ExecutionStatus.RUNNING_EXEUTION, taskInstance.getStartTime(), taskInstance.getHost(), taskInstance.getExecutePath(), @@ -311,7 +311,7 @@ public class TaskScheduleThread implements Runnable { if (!resFile.exists()) { try { // query the tenant code of the resource according to the name of the resource - String tentnCode = processDao.queryTenantCodeByResName(res); + String tentnCode = processService.queryTenantCodeByResName(res); String resHdfsPath = HadoopUtils.getHdfsFilename(tentnCode, res); logger.info("get resource file from hdfs :{}", resHdfsPath); @@ -334,7 +334,7 @@ public class TaskScheduleThread implements Runnable { private void checkDownloadPermission(List projectRes) throws Exception { int userId = taskInstance.getProcessInstance().getExecutorId(); String[] resNames = projectRes.toArray(new String[projectRes.size()]); - PermissionCheck permissionCheck = new PermissionCheck<>(AuthorizationType.RESOURCE_FILE,processDao,resNames,userId,logger); + PermissionCheck permissionCheck = new PermissionCheck<>(AuthorizationType.RESOURCE_FILE, processService,resNames,userId,logger); permissionCheck.checkPermission(); } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index f1c01aff36..c473f3a2aa 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -21,10 +21,10 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.utils.ProcessUtils; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import java.io.*; @@ -121,10 +121,10 @@ public abstract class AbstractCommandExecutor { * task specific execution logic * * @param execCommand exec command - * @param processDao process dao + * @param processService process dao * @return exit status code */ - public int run(String execCommand, ProcessDao processDao) { + public int run(String execCommand, ProcessService processService) { int exitStatusCode; try { @@ -147,7 +147,7 @@ public abstract class AbstractCommandExecutor { // get process id int pid = getProcessId(process); - processDao.updatePidByTaskInstId(taskInstId, pid, ""); + processService.updatePidByTaskInstId(taskInstId, pid, ""); logger.info("process start, process id is: {}", pid); @@ -161,10 +161,10 @@ public abstract class AbstractCommandExecutor { exitStatusCode = process.exitValue(); logger.info("process has exited, work dir:{}, pid:{} ,exitStatusCode:{}", taskDir, pid,exitStatusCode); //update process state to db - exitStatusCode = updateState(processDao, exitStatusCode, pid, taskInstId); + exitStatusCode = updateState(processService, exitStatusCode, pid, taskInstId); } else { - TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId); + TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); if (taskInstance == null) { logger.error("task instance id:{} not exist", taskInstId); } else { @@ -219,23 +219,23 @@ public abstract class AbstractCommandExecutor { /** * update process state to db * - * @param processDao process dao + * @param processService process dao * @param exitStatusCode exit status code * @param pid process id * @param taskInstId task instance id * @return exit status code */ - private int updateState(ProcessDao processDao, int exitStatusCode, int pid, int taskInstId) { + private int updateState(ProcessService processService, int exitStatusCode, int pid, int taskInstId) { //get yarn state by log if (exitStatusCode == 0) { - TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId); + TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); logger.info("process id is {}", pid); List appIds = getAppLinks(taskInstance.getLogPath()); if (appIds.size() > 0) { String appUrl = String.join(Constants.COMMA, appIds); logger.info("yarn log url:{}",appUrl); - processDao.updatePidByTaskInstId(taskInstId, pid, appUrl); + processService.updatePidByTaskInstId(taskInstId, pid, appUrl); } // check if all operations are completed diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java index 6846617408..39f4dfbb97 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.server.worker.task; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; /** @@ -39,7 +39,7 @@ public abstract class AbstractYarnTask extends AbstractTask { /** * process database access */ - protected ProcessDao processDao; + protected ProcessService processService; /** * Abstract Yarn Task @@ -48,7 +48,7 @@ public abstract class AbstractYarnTask extends AbstractTask { */ public AbstractYarnTask(TaskProps taskProps, Logger logger) { super(taskProps, logger); - this.processDao = SpringApplicationContext.getBean(ProcessDao.class); + this.processService = SpringApplicationContext.getBean(ProcessService.class); this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle, taskProps.getTaskDir(), taskProps.getTaskAppId(), @@ -64,7 +64,7 @@ public abstract class AbstractYarnTask extends AbstractTask { public void handle() throws Exception { try { // construct process - exitStatusCode = shellCommandExecutor.run(buildCommand(), processDao); + exitStatusCode = shellCommandExecutor.run(buildCommand(), processService); } catch (Exception e) { logger.error("yarn process failure", e); exitStatusCode = -1; @@ -82,7 +82,7 @@ public abstract class AbstractYarnTask extends AbstractTask { cancel = true; // cancel process shellCommandExecutor.cancelApplication(); - TaskInstance taskInstance = processDao.findTaskInstanceById(taskProps.getTaskInstId()); + TaskInstance taskInstance = processService.findTaskInstanceById(taskProps.getTaskInstId()); if (status && taskInstance != null){ ProcessUtils.killYarnJob(taskInstance); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 0de2bbc7c6..ef941cd062 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -39,23 +39,23 @@ import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.job.db.BaseDataSource; -import org.apache.dolphinscheduler.common.job.db.DataSourceFactory; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; +import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; +import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.utils.DataxUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import com.alibaba.druid.sql.ast.SQLStatement; @@ -106,9 +106,9 @@ public class DataxTask extends AbstractTask { private ShellCommandExecutor shellCommandExecutor; /** - * process database access + * process dao */ - private ProcessDao processDao; + private ProcessService processService; /** * constructor @@ -128,7 +128,7 @@ public class DataxTask extends AbstractTask { props.getTaskInstId(), props.getTenantCode(), props.getEnvFile(), props.getTaskStartTime(), props.getTaskTimeout(), logger); - this.processDao = SpringApplicationContext.getBean(ProcessDao.class); + this.processService = SpringApplicationContext.getBean(ProcessService.class); } /** @@ -160,7 +160,7 @@ public class DataxTask extends AbstractTask { // run datax process String jsonFilePath = buildDataxJsonFile(); String shellCommandFilePath = buildShellCommandFile(jsonFilePath); - exitStatusCode = shellCommandExecutor.run(shellCommandFilePath, processDao); + exitStatusCode = shellCommandExecutor.run(shellCommandFilePath, processService); } catch (Exception e) { exitStatusCode = -1; @@ -220,11 +220,11 @@ public class DataxTask extends AbstractTask { */ private List buildDataxJobContentJson() throws SQLException { - DataSource dataSource = processDao.findDataSourceById(dataXParameters.getDataSource()); + DataSource dataSource = processService.findDataSourceById(dataXParameters.getDataSource()); BaseDataSource dataSourceCfg = DataSourceFactory.getDatasource(dataSource.getType(), dataSource.getConnectionParams()); - DataSource dataTarget = processDao.findDataSourceById(dataXParameters.getDataTarget()); + DataSource dataTarget = processService.findDataSourceById(dataXParameters.getDataTarget()); BaseDataSource dataTargetCfg = DataSourceFactory.getDatasource(dataTarget.getType(), dataTarget.getConnectionParams()); @@ -355,7 +355,7 @@ public class DataxTask extends AbstractTask { String dataxCommand = sbr.toString(); // find process instance by task id - ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId()); + ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId()); // combining local and global parameters Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java index 4be65ed49d..b08cabc2e9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java @@ -23,10 +23,10 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.DependentItem; import org.apache.dolphinscheduler.common.utils.DependentUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,9 +37,9 @@ import java.util.*; */ public class DependentExecute { /** - * process dao + * process service */ - private final ProcessDao processDao = SpringApplicationContext.getBean(ProcessDao.class); + private final ProcessService processService = SpringApplicationContext.getBean(ProcessService.class); /** * depend item list @@ -108,7 +108,7 @@ public class DependentExecute { result = getDependResultByState(processInstance.getState()); }else{ TaskInstance taskInstance = null; - List taskInstanceList = processDao.findValidTaskListByProcessId(processInstance.getId()); + List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); for(TaskInstance task : taskInstanceList){ if(task.getName().equals(dependentItem.getDepTasks())){ @@ -141,16 +141,16 @@ public class DependentExecute { */ private ProcessInstance findLastProcessInterval(int definitionId, DateInterval dateInterval) { - ProcessInstance runningProcess = processDao.findLastRunningProcess(definitionId, dateInterval); + ProcessInstance runningProcess = processService.findLastRunningProcess(definitionId, dateInterval); if(runningProcess != null){ return runningProcess; } - ProcessInstance lastSchedulerProcess = processDao.findLastSchedulerProcessInterval( + ProcessInstance lastSchedulerProcess = processService.findLastSchedulerProcessInterval( definitionId, dateInterval ); - ProcessInstance lastManualProcess = processDao.findLastManualProcessInterval( + ProcessInstance lastManualProcess = processService.findLastManualProcessInterval( definitionId, dateInterval ); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java index 9af29e01dd..f074d57e6c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java @@ -25,11 +25,11 @@ import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.DependentUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import java.util.*; @@ -63,9 +63,9 @@ public class DependentTask extends AbstractTask { private Date dependentDate; /** - * process dao + * process service */ - private ProcessDao processDao; + private ProcessService processService; /** * constructor @@ -88,7 +88,7 @@ public class DependentTask extends AbstractTask { taskModel.getDependItemList(), taskModel.getRelation())); } - this.processDao = SpringApplicationContext.getBean(ProcessDao.class); + this.processService = SpringApplicationContext.getBean(ProcessService.class); if(taskProps.getScheduleTime() != null){ this.dependentDate = taskProps.getScheduleTime(); @@ -107,7 +107,7 @@ public class DependentTask extends AbstractTask { try{ TaskInstance taskInstance = null; while(Stopper.isRunning()){ - taskInstance = processDao.findTaskInstanceById(this.taskProps.getTaskInstId()); + taskInstance = processService.findTaskInstanceById(this.taskProps.getTaskInstId()); if(taskInstance == null){ exitStatusCode = -1; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java index 0fa9e11ce5..c562fbe4dd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java @@ -68,7 +68,7 @@ public class FlinkTask extends AbstractYarnTask { if (StringUtils.isNotEmpty(flinkParameters.getMainArgs())) { String args = flinkParameters.getMainArgs(); // get process instance by task instance id - ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId()); + ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId()); /** * combining local and global parameters diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index 97e6cb7bee..c925f90b9e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -28,13 +28,13 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.http.HttpEntity; import org.apache.http.ParseException; import org.apache.http.client.config.RequestConfig; @@ -66,9 +66,9 @@ public class HttpTask extends AbstractTask { private HttpParameters httpParameters; /** - * process database access + * process service */ - private ProcessDao processDao; + private ProcessService processService; /** * Convert mill seconds to second unit @@ -92,7 +92,7 @@ public class HttpTask extends AbstractTask { */ public HttpTask(TaskProps props, Logger logger) { super(props, logger); - this.processDao = SpringApplicationContext.getBean(ProcessDao.class); + this.processService = SpringApplicationContext.getBean(ProcessService.class); } @Override @@ -138,7 +138,7 @@ public class HttpTask extends AbstractTask { */ protected CloseableHttpResponse sendRequest(CloseableHttpClient client) throws IOException { RequestBuilder builder = createRequestBuilder(); - ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId()); + ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId()); Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), taskProps.getDefinedParams(), diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index 9b4952bbd2..fb881453e9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -22,19 +22,19 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; -import org.apache.dolphinscheduler.common.job.db.BaseDataSource; -import org.apache.dolphinscheduler.common.job.db.DataSourceFactory; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.dao.ProcessDao; +import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; +import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import java.sql.*; @@ -56,9 +56,9 @@ public class ProcedureTask extends AbstractTask { private ProcedureParameters procedureParameters; /** - * process database access + * process service */ - private ProcessDao processDao; + private ProcessService processService; /** * base datasource @@ -82,7 +82,7 @@ public class ProcedureTask extends AbstractTask { throw new RuntimeException("procedure task params is not valid"); } - this.processDao = SpringApplicationContext.getBean(ProcessDao.class); + this.processService = SpringApplicationContext.getBean(ProcessService.class); } @Override @@ -97,7 +97,7 @@ public class ProcedureTask extends AbstractTask { procedureParameters.getMethod(), procedureParameters.getLocalParams()); - DataSource dataSource = processDao.findDataSourceById(procedureParameters.getDatasource()); + DataSource dataSource = processService.findDataSourceById(procedureParameters.getDatasource()); if (dataSource == null){ logger.error("datasource not exists"); exitStatusCode = -1; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java index 585d62f154..fc212f866b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java @@ -22,12 +22,12 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.python.PythonParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.PythonCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import java.util.Map; @@ -53,9 +53,9 @@ public class PythonTask extends AbstractTask { private PythonCommandExecutor pythonCommandExecutor; /** - * process database access + * process service */ - private ProcessDao processDao; + private ProcessService processService; /** * constructor @@ -76,7 +76,7 @@ public class PythonTask extends AbstractTask { taskProps.getTaskStartTime(), taskProps.getTaskTimeout(), logger); - this.processDao = SpringApplicationContext.getBean(ProcessDao.class); + this.processService = SpringApplicationContext.getBean(ProcessService.class); } @Override @@ -94,7 +94,7 @@ public class PythonTask extends AbstractTask { public void handle() throws Exception { try { // construct process - exitStatusCode = pythonCommandExecutor.run(buildCommand(), processDao); + exitStatusCode = pythonCommandExecutor.run(buildCommand(), processService); } catch (Exception e) { logger.error("python task failure", e); exitStatusCode = -1; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index 789a0c5302..5704c8052e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -23,12 +23,12 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import java.io.File; @@ -64,7 +64,7 @@ public class ShellTask extends AbstractTask { /** * process database access */ - private ProcessDao processDao; + private ProcessService processService; /** * constructor @@ -84,7 +84,7 @@ public class ShellTask extends AbstractTask { taskProps.getTaskStartTime(), taskProps.getTaskTimeout(), logger); - this.processDao = SpringApplicationContext.getBean(ProcessDao.class); + this.processService = SpringApplicationContext.getBean(ProcessService.class); } @Override @@ -102,7 +102,7 @@ public class ShellTask extends AbstractTask { public void handle() throws Exception { try { // construct process - exitStatusCode = shellCommandExecutor.run(buildCommand(), processDao); + exitStatusCode = shellCommandExecutor.run(buildCommand(), processService); } catch (Exception e) { logger.error("shell task failure", e); exitStatusCode = -1; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index eba05a0d21..aae11f5530 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -27,8 +27,6 @@ import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.common.enums.UdfType; -import org.apache.dolphinscheduler.common.job.db.BaseDataSource; -import org.apache.dolphinscheduler.common.job.db.DataSourceFactory; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.sql.SqlBinds; @@ -36,16 +34,19 @@ import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sql.SqlType; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AlertDao; -import org.apache.dolphinscheduler.dao.ProcessDao; +import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; +import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.permission.PermissionCheck; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.utils.UDFUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.permission.PermissionCheck; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import java.sql.*; @@ -67,9 +68,9 @@ public class SqlTask extends AbstractTask { private SqlParameters sqlParameters; /** - * process database access + * process service */ - private ProcessDao processDao; + private ProcessService processService; /** * alert dao @@ -96,7 +97,7 @@ public class SqlTask extends AbstractTask { if (!sqlParameters.checkParameters()) { throw new RuntimeException("sql task params is not valid"); } - this.processDao = SpringApplicationContext.getBean(ProcessDao.class); + this.processService = SpringApplicationContext.getBean(ProcessService.class); this.alertDao = SpringApplicationContext.getBean(AlertDao.class); } @@ -122,7 +123,7 @@ public class SqlTask extends AbstractTask { return; } - dataSource= processDao.findDataSourceById(sqlParameters.getDatasource()); + dataSource= processService.findDataSourceById(sqlParameters.getDatasource()); // data source is null if (dataSource == null){ @@ -171,7 +172,7 @@ public class SqlTask extends AbstractTask { } // check udf permission checkUdfPermission(ArrayUtils.toObject(idsArray)); - List udfFuncList = processDao.queryUdfFunListByids(idsArray); + List udfFuncList = processService.queryUdfFunListByids(idsArray); createFuncs = UDFUtils.createFuncs(udfFuncList, taskProps.getTenantCode(), logger); } @@ -383,7 +384,7 @@ public class SqlTask extends AbstractTask { public void sendAttachment(String title,String content){ // process instance - ProcessInstance instance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId()); + ProcessInstance instance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId()); List users = alertDao.queryUserByAlertGroupId(instance.getWarningGroupId()); @@ -470,10 +471,10 @@ public class SqlTask extends AbstractTask { */ private void checkUdfPermission(Integer[] udfFunIds) throws Exception{ // process instance - ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId()); + ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId()); int userId = processInstance.getExecutorId(); - PermissionCheck permissionCheckUdf = new PermissionCheck(AuthorizationType.UDF,processDao,udfFunIds,userId,logger); + PermissionCheck permissionCheckUdf = new PermissionCheck(AuthorizationType.UDF, processService,udfFunIds,userId,logger); permissionCheckUdf.checkPermission(); } @@ -484,10 +485,10 @@ public class SqlTask extends AbstractTask { */ private void checkDataSourcePermission(int dataSourceId) throws Exception{ // process instance - ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId()); + ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId()); int userId = processInstance.getExecutorId(); - PermissionCheck permissionCheckDataSource = new PermissionCheck(AuthorizationType.DATASOURCE,processDao,new Integer[]{dataSourceId},userId,logger); + PermissionCheck permissionCheckDataSource = new PermissionCheck(AuthorizationType.DATASOURCE, processService,new Integer[]{dataSourceId},userId,logger); permissionCheckDataSource.checkPermission(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index c6a71ed066..fe4ec9130a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -21,10 +21,8 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; @@ -32,6 +30,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.utils.ThreadUtils; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.zk.AbstractZKClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -70,10 +70,10 @@ public class ZKMasterClient extends AbstractZKClient { */ private AlertDao alertDao = null; /** - * flow database access + * process service */ @Autowired - private ProcessDao processDao; + private ProcessService processService; /** * default constructor @@ -374,7 +374,7 @@ public class ZKMasterClient extends AbstractZKClient { private void failoverWorker(String workerHost, boolean needCheckWorkerAlive) throws Exception { logger.info("start worker[{}] failover ...", workerHost); - List needFailoverTaskInstanceList = processDao.queryNeedFailoverTaskInstances(workerHost); + List needFailoverTaskInstanceList = processService.queryNeedFailoverTaskInstances(workerHost); for(TaskInstance taskInstance : needFailoverTaskInstanceList){ if(needCheckWorkerAlive){ if(!checkTaskInstanceNeedFailover(taskInstance)){ @@ -382,7 +382,7 @@ public class ZKMasterClient extends AbstractZKClient { } } - ProcessInstance instance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); + ProcessInstance instance = processService.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); if(instance!=null){ taskInstance.setProcessInstance(instance); } @@ -390,7 +390,7 @@ public class ZKMasterClient extends AbstractZKClient { ProcessUtils.killYarnJob(taskInstance); taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE); - processDao.saveTaskInstance(taskInstance); + processService.saveTaskInstance(taskInstance); } logger.info("end worker[{}] failover ...", workerHost); } @@ -403,11 +403,11 @@ public class ZKMasterClient extends AbstractZKClient { private void failoverMaster(String masterHost) { logger.info("start master failover ..."); - List needFailoverProcessInstanceList = processDao.queryNeedFailoverProcessInstances(masterHost); + List needFailoverProcessInstanceList = processService.queryNeedFailoverProcessInstances(masterHost); //updateProcessInstance host is null and insert into command for(ProcessInstance processInstance : needFailoverProcessInstanceList){ - processDao.processNeedFailoverProcessInstances(processInstance); + processService.processNeedFailoverProcessInstances(processInstance); } logger.info("master failover end"); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java index 88abfa3071..7ddee3b2a1 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java @@ -19,9 +19,9 @@ package org.apache.dolphinscheduler.server.zk; import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ZKNodeType; -import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; +import org.apache.dolphinscheduler.service.zk.AbstractZKClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index d7c3de13a5..770ab3cff6 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -20,13 +20,13 @@ import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.MasterExecThread; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -57,7 +57,7 @@ public class MasterExecThreadTest { private ProcessInstance processInstance; - private ProcessDao processDao; + private ProcessService processService; private int processDefinitionId = 1; @@ -67,7 +67,7 @@ public class MasterExecThreadTest { @Before public void init() throws Exception{ - processDao = mock(ProcessDao.class); + processService = mock(ProcessService.class); applicationContext = mock(ApplicationContext.class); config = new MasterConfig(); @@ -91,7 +91,7 @@ public class MasterExecThreadTest { processDefinition.setGlobalParamList(Collections.EMPTY_LIST); Mockito.when(processInstance.getProcessDefinition()).thenReturn(processDefinition); - masterExecThread = PowerMockito.spy(new MasterExecThread(processInstance, processDao)); + masterExecThread = PowerMockito.spy(new MasterExecThread(processInstance, processService)); // prepareProcess init dag Field dag = MasterExecThread.class.getDeclaredField("dag"); dag.setAccessible(true); @@ -110,12 +110,12 @@ public class MasterExecThreadTest { @Test public void testParallelWithOutSchedule() throws ParseException { try{ - Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); + Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess"); method.setAccessible(true); method.invoke(masterExecThread); // one create save, and 1-30 for next save, and last day 31 no save - verify(processDao, times(31)).saveProcessInstance(processInstance); + verify(processService, times(31)).saveProcessInstance(processInstance); }catch (Exception e){ e.printStackTrace(); Assert.assertTrue(false); @@ -129,12 +129,12 @@ public class MasterExecThreadTest { @Test public void testParallelWithSchedule() throws ParseException { try{ - Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(oneSchedulerList()); + Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(oneSchedulerList()); Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess"); method.setAccessible(true); method.invoke(masterExecThread); // one create save, and 15(1 to 31 step 2) for next save, and last day 31 no save - verify(processDao, times(15)).saveProcessInstance(processInstance); + verify(processService, times(15)).saveProcessInstance(processInstance); }catch (Exception e){ Assert.assertTrue(false); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 04c844827f..5d4263644b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -20,13 +20,13 @@ import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -43,11 +43,11 @@ public class ShellCommandExecutorTest { private static final Logger logger = LoggerFactory.getLogger(ShellCommandExecutorTest.class); - private ProcessDao processDao = null; + private ProcessService processService = null; @Before public void before(){ - processDao = SpringApplicationContext.getBean(ProcessDao.class); + processService = SpringApplicationContext.getBean(ProcessService.class); } @Test @@ -65,7 +65,7 @@ public class ShellCommandExecutorTest { - TaskInstance taskInstance = processDao.findTaskInstanceById(7657); + TaskInstance taskInstance = processService.findTaskInstanceById(7657); String taskJson = taskInstance.getTaskJson(); TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java index 7da3f710b6..c395eabe51 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java @@ -21,13 +21,13 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -44,11 +44,11 @@ public class SqlExecutorTest { private static final Logger logger = LoggerFactory.getLogger(SqlExecutorTest.class); - private ProcessDao processDao = null; + private ProcessService processService = null; @Before public void before(){ - processDao = SpringApplicationContext.getBean(ProcessDao.class); + processService = SpringApplicationContext.getBean(ProcessService.class); } @Test @@ -109,7 +109,7 @@ public class SqlExecutorTest { taskProps.setCmdTypeIfComplement(CommandType.START_PROCESS); - TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId); + TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); String taskJson = taskInstance.getTaskJson(); TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index 7a6073e05d..bd7f27530a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -25,15 +25,15 @@ import java.util.List; import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.job.db.BaseDataSource; -import org.apache.dolphinscheduler.common.job.db.DataSourceFactory; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; -import org.apache.dolphinscheduler.dao.ProcessDao; +import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; +import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.utils.DataxUtils; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -53,7 +53,7 @@ public class DataxTaskTest { private DataxTask dataxTask; - private ProcessDao processDao; + private ProcessService processService; private ShellCommandExecutor shellCommandExecutor; @@ -62,13 +62,13 @@ public class DataxTaskTest { @Before public void before() throws Exception { - processDao = Mockito.mock(ProcessDao.class); + processService = Mockito.mock(ProcessService.class); shellCommandExecutor = Mockito.mock(ShellCommandExecutor.class); applicationContext = Mockito.mock(ApplicationContext.class); SpringApplicationContext springApplicationContext = new SpringApplicationContext(); springApplicationContext.setApplicationContext(applicationContext); - Mockito.when(applicationContext.getBean(ProcessDao.class)).thenReturn(processDao); + Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); TaskProps props = new TaskProps(); props.setTaskDir("/tmp"); @@ -83,12 +83,12 @@ public class DataxTaskTest { dataxTask = PowerMockito.spy(new DataxTask(props, logger)); dataxTask.init(); - Mockito.when(processDao.findDataSourceById(1)).thenReturn(getDataSource()); - Mockito.when(processDao.findDataSourceById(2)).thenReturn(getDataSource()); - Mockito.when(processDao.findProcessInstanceByTaskId(1)).thenReturn(getProcessInstance()); + Mockito.when(processService.findDataSourceById(1)).thenReturn(getDataSource()); + Mockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource()); + Mockito.when(processService.findProcessInstanceByTaskId(1)).thenReturn(getProcessInstance()); String fileName = String.format("%s/%s_node.sh", props.getTaskDir(), props.getTaskAppId()); - Mockito.when(shellCommandExecutor.run(fileName, processDao)).thenReturn(0); + Mockito.when(shellCommandExecutor.run(fileName, processService)).thenReturn(0); } private DataSource getDataSource() { diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml new file mode 100644 index 0000000000..7d775d5497 --- /dev/null +++ b/dolphinscheduler-service/pom.xml @@ -0,0 +1,56 @@ + + + + + dolphinscheduler + org.apache.dolphinscheduler + 1.2.1-SNAPSHOT + + 4.0.0 + + dolphinscheduler-service + + dolphinscheduler-service + + + + org.apache.dolphinscheduler + dolphinscheduler-remote + + + org.apache.dolphinscheduler + dolphinscheduler-dao + + + org.apache.curator + curator-client + ${curator.version} + + + log4j-1.2-api + org.apache.logging.log4j + + + io.netty + netty + + + + + org.quartz-scheduler + quartz + + + c3p0 + c3p0 + + + + + + org.quartz-scheduler + quartz-jobs + + + diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java similarity index 96% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java index 97618e1b39..ddf1fecf76 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java @@ -14,14 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.service.bean; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.stereotype.Component; - @Component public class SpringApplicationContext implements ApplicationContextAware { diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java new file mode 100644 index 0000000000..aa6999ef0d --- /dev/null +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.service.log; + +import io.netty.channel.Channel; +import org.apache.dolphinscheduler.remote.NettyRemotingClient; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.command.log.*; +import org.apache.dolphinscheduler.remote.config.NettyClientConfig; +import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; +import org.apache.dolphinscheduler.remote.utils.Address; +import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * log client + */ +public class LogClientService implements NettyRequestProcessor { + + private static final Logger logger = LoggerFactory.getLogger(LogClientService.class); + + private final NettyClientConfig clientConfig; + + private final NettyRemotingClient client; + + private final Address address; + + /** + * request time out + */ + private final long logRequestTimeout = 10 * 1000; + + /** + * construct client + * @param host host + * @param port port + */ + public LogClientService(String host, int port) { + this.address = new Address(host, port); + this.clientConfig = new NettyClientConfig(); + this.clientConfig.setWorkerThreads(1); + this.client = new NettyRemotingClient(clientConfig); + this.client.registerProcessor(CommandType.ROLL_VIEW_LOG_RESPONSE,this); + this.client.registerProcessor(CommandType.VIEW_WHOLE_LOG_RESPONSE, this); + this.client.registerProcessor(CommandType.GET_LOG_BYTES_RESPONSE, this); + + } + + /** + * close + */ + public void close() { + this.client.close(); + logger.info("logger client closed"); + } + + /** + * roll view log + * @param path path + * @param skipLineNum skip line number + * @param limit limit + * @return log content + */ + public String rollViewLog(String path,int skipLineNum,int limit) { + logger.info("roll view log, path {}, skipLineNum {} ,limit {}", path, skipLineNum, limit); + RollViewLogRequestCommand request = new RollViewLogRequestCommand(path, skipLineNum, limit); + String result = ""; + try { + Command command = request.convert2Command(); + this.client.send(address, command); + LogPromise promise = new LogPromise(command.getOpaque(), logRequestTimeout); + result = ((String)promise.getResult()); + } catch (Exception e) { + logger.error("roll view log error", e); + } + return result; + } + + /** + * view log + * @param path path + * @return log content + */ + public String viewLog(String path) { + logger.info("view log path {}", path); + ViewLogRequestCommand request = new ViewLogRequestCommand(path); + String result = ""; + try { + Command command = request.convert2Command(); + this.client.send(address, command); + LogPromise promise = new LogPromise(command.getOpaque(), logRequestTimeout); + result = ((String)promise.getResult()); + } catch (Exception e) { + logger.error("view log error", e); + } + return result; + } + + /** + * get log size + * @param path log path + * @return log content bytes + */ + public byte[] getLogBytes(String path) { + logger.info("log path {}", path); + GetLogBytesRequestCommand request = new GetLogBytesRequestCommand(path); + byte[] result = null; + try { + Command command = request.convert2Command(); + this.client.send(address, command); + LogPromise promise = new LogPromise(command.getOpaque(), logRequestTimeout); + result = (byte[])promise.getResult(); + } catch (Exception e) { + logger.error("get log size error", e); + } + return result; + } + + @Override + public void process(Channel channel, Command command) { + logger.info("received log response : {}", command); + switch (command.getType()){ + case ROLL_VIEW_LOG_RESPONSE: + RollViewLogResponseCommand rollReviewLog = FastJsonSerializer.deserialize( + command.getBody(), RollViewLogResponseCommand.class); + LogPromise.notify(command.getOpaque(), rollReviewLog.getMsg()); + break; + case VIEW_WHOLE_LOG_RESPONSE: + ViewLogResponseCommand viewLog = FastJsonSerializer.deserialize( + command.getBody(), ViewLogResponseCommand.class); + LogPromise.notify(command.getOpaque(), viewLog.getMsg()); + break; + case GET_LOG_BYTES_RESPONSE: + GetLogBytesResponseCommand getLog = FastJsonSerializer.deserialize( + command.getBody(), GetLogBytesResponseCommand.class); + LogPromise.notify(command.getOpaque(), getLog.getData()); + break; + default: + throw new UnsupportedOperationException(String.format("command type : %s is not supported ", command.getType())); + } + } + + public static void main(String[] args) throws Exception{ + LogClientService logClient = new LogClientService("192.168.220.247", 50051); + byte[] logBytes = logClient.getLogBytes("/opt/program/incubator-dolphinscheduler/logs/1/463/540.log"); + System.out.println(new String(logBytes)); + } + +} \ No newline at end of file diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java new file mode 100644 index 0000000000..8920b8a527 --- /dev/null +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.service.log; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * log asyc callback + */ +public class LogPromise { + + private static final ConcurrentHashMap PROMISES = new ConcurrentHashMap<>(); + + private long opaque; + + private final long start; + + private final long timeout; + + private final CountDownLatch latch; + + private Object result; + + public LogPromise(long opaque, long timeout){ + this.opaque = opaque; + this.timeout = timeout; + this.start = System.currentTimeMillis(); + this.latch = new CountDownLatch(1); + PROMISES.put(opaque, this); + } + + + /** + * notify client finish + * @param opaque unique identification + * @param result result + */ + public static void notify(long opaque, Object result){ + LogPromise promise = PROMISES.remove(opaque); + if(promise != null){ + promise.doCountDown(result); + } + } + + private void doCountDown(Object result){ + this.result = result; + this.latch.countDown(); + } + + public boolean isTimeout(){ + return System.currentTimeMillis() - start > timeout; + } + + public Object getResult(){ + try { + latch.await(timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException ignore) { + } + PROMISES.remove(opaque); + return this.result; + } + + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/permission/PermissionCheck.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java similarity index 80% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/permission/PermissionCheck.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java index 63d4c1c8af..027666f053 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/permission/PermissionCheck.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.permission; +package org.apache.dolphinscheduler.service.permission; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import java.util.List; @@ -38,7 +38,7 @@ public class PermissionCheck { /** * Authorization Type */ - private ProcessDao processDao; + private ProcessService processService; /** * need check array @@ -53,23 +53,23 @@ public class PermissionCheck { /** * permission check * @param authorizationType authorization type - * @param processDao process dao + * @param processService process dao */ - public PermissionCheck(AuthorizationType authorizationType, ProcessDao processDao) { + public PermissionCheck(AuthorizationType authorizationType, ProcessService processService) { this.authorizationType = authorizationType; - this.processDao = processDao; + this.processService = processService; } /** * permission check * @param authorizationType - * @param processDao + * @param processService * @param needChecks * @param userId */ - public PermissionCheck(AuthorizationType authorizationType, ProcessDao processDao, T[] needChecks, int userId) { + public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId) { this.authorizationType = authorizationType; - this.processDao = processDao; + this.processService = processService; this.needChecks = needChecks; this.userId = userId; } @@ -77,14 +77,14 @@ public class PermissionCheck { /** * permission check * @param authorizationType - * @param processDao + * @param processService * @param needChecks * @param userId * @param logger */ - public PermissionCheck(AuthorizationType authorizationType, ProcessDao processDao, T[] needChecks, int userId,Logger logger) { + public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId, Logger logger) { this.authorizationType = authorizationType; - this.processDao = processDao; + this.processService = processService; this.needChecks = needChecks; this.userId = userId; this.logger = logger; @@ -98,12 +98,12 @@ public class PermissionCheck { this.authorizationType = authorizationType; } - public ProcessDao getProcessDao() { - return processDao; + public ProcessService getProcessService() { + return processService; } - public void setProcessDao(ProcessDao processDao) { - this.processDao = processDao; + public void setProcessService(ProcessService processService) { + this.processService = processService; } public T[] getNeedChecks() { @@ -142,9 +142,9 @@ public class PermissionCheck { public void checkPermission() throws Exception{ if(this.needChecks.length > 0){ // get user type in order to judge whether the user is admin - User user = processDao.getUserById(userId); + User user = processService.getUserById(userId); if (user.getUserType() != UserType.ADMIN_USER){ - List unauthorizedList = processDao.listUnauthorized(userId,needChecks,authorizationType); + List unauthorizedList = processService.listUnauthorized(userId,needChecks,authorizationType); // if exist unauthorized resource if(CollectionUtils.isNotEmpty(unauthorizedList)){ logger.error("user {} didn't has permission of {}: {}", user.getUserName(), authorizationType.getDescp(),unauthorizedList.toString()); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java similarity index 99% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 820b2fdaf4..a26044e417 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao; +package org.apache.dolphinscheduler.service.process; import com.alibaba.fastjson.JSONObject; import com.cronutils.model.Cron; @@ -24,16 +24,12 @@ import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.process.Property; -import org.apache.dolphinscheduler.common.queue.ITaskQueue; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.IpUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; -import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; +import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; +import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,7 +47,7 @@ import static org.apache.dolphinscheduler.common.Constants.*; * process relative dao that some mappers in this. */ @Component -public class ProcessDao { +public class ProcessService { private final Logger logger = LoggerFactory.getLogger(getClass()); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/DruidConnectionProvider.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/DruidConnectionProvider.java similarity index 99% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/DruidConnectionProvider.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/DruidConnectionProvider.java index 8a4ceba927..d51e8e82bf 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/DruidConnectionProvider.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/DruidConnectionProvider.java @@ -14,11 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.quartz; +package org.apache.dolphinscheduler.service.quartz; import com.alibaba.druid.pool.DruidDataSource; import org.quartz.SchedulerException; import org.quartz.utils.ConnectionProvider; + import java.sql.Connection; import java.sql.SQLException; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/ProcessScheduleJob.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java similarity index 83% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/ProcessScheduleJob.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java index ac461296a9..69a80e65f5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/ProcessScheduleJob.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java @@ -14,17 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.quartz; +package org.apache.dolphinscheduler.service.quartz; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ReleaseState; -import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Schedule; -import org.quartz.*; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; @@ -42,17 +45,17 @@ public class ProcessScheduleJob implements Job { private static final Logger logger = LoggerFactory.getLogger(ProcessScheduleJob.class); /** - * process dao + * process service */ - private static ProcessDao processDao; + private static ProcessService processService; /** * init - * @param processDao process dao + * @param processService process dao */ - public static void init(ProcessDao processDao) { - ProcessScheduleJob.processDao = processDao; + public static void init(ProcessService processService) { + ProcessScheduleJob.processService = processService; } /** @@ -64,7 +67,7 @@ public class ProcessScheduleJob implements Job { @Override public void execute(JobExecutionContext context) throws JobExecutionException { - Assert.notNull(processDao, "please call init() method first"); + Assert.notNull(processService, "please call init() method first"); JobDataMap dataMap = context.getJobDetail().getJobDataMap(); @@ -80,7 +83,7 @@ public class ProcessScheduleJob implements Job { logger.info("scheduled fire time :{}, fire time :{}, process id :{}", scheduledFireTime, fireTime, scheduleId); // query schedule - Schedule schedule = processDao.querySchedule(scheduleId); + Schedule schedule = processService.querySchedule(scheduleId); if (schedule == null) { logger.warn("process schedule does not exist in db，delete schedule job in quartz, projectId:{}, scheduleId:{}", projectId, scheduleId); deleteJob(projectId, scheduleId); @@ -88,7 +91,7 @@ public class ProcessScheduleJob implements Job { } - ProcessDefinition processDefinition = processDao.findProcessDefineById(schedule.getProcessDefinitionId()); + ProcessDefinition processDefinition = processService.findProcessDefineById(schedule.getProcessDefinitionId()); // release state : online/offline ReleaseState releaseState = processDefinition.getReleaseState(); if (processDefinition == null || releaseState == ReleaseState.OFFLINE) { @@ -108,7 +111,7 @@ public class ProcessScheduleJob implements Job { command.setWarningType(schedule.getWarningType()); command.setProcessInstancePriority(schedule.getProcessInstancePriority()); - processDao.createCommand(command); + processService.createCommand(command); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java similarity index 99% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/QuartzExecutors.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java index 054d7903fc..9d96264a60 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/QuartzExecutors.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.quartz; +package org.apache.dolphinscheduler.service.quartz; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.Schedule; -import org.apache.commons.lang.StringUtils; import org.quartz.*; import org.quartz.impl.StdSchedulerFactory; import org.quartz.impl.matchers.GroupMatcher; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/AbstractCycle.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java similarity index 99% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/AbstractCycle.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java index 0cda336d7d..0a2e31b610 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/AbstractCycle.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.utils.cron; +package org.apache.dolphinscheduler.service.quartz.cron; -import org.apache.dolphinscheduler.common.enums.CycleEnum; import com.cronutils.model.Cron; import com.cronutils.model.field.CronField; import com.cronutils.model.field.CronFieldName; import com.cronutils.model.field.expression.*; +import org.apache.dolphinscheduler.common.enums.CycleEnum; /** * Cycle diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java similarity index 98% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java index 2ad029064e..d03a4a5cdc 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java @@ -14,15 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.utils.cron; +package org.apache.dolphinscheduler.service.quartz.cron; -import org.apache.dolphinscheduler.common.enums.CycleEnum; -import org.apache.dolphinscheduler.common.thread.Stopper; -import org.apache.dolphinscheduler.common.utils.DateUtils; import com.cronutils.model.Cron; import com.cronutils.model.definition.CronDefinitionBuilder; import com.cronutils.parser.CronParser; +import org.apache.dolphinscheduler.common.enums.CycleEnum; +import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +31,7 @@ import java.text.ParseException; import java.util.*; import static com.cronutils.model.CronType.QUARTZ; -import static org.apache.dolphinscheduler.dao.utils.cron.CycleFactory.*; +import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.*; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleFactory.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java similarity index 99% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleFactory.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java index b2f52566fc..1f807dce7f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleFactory.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.utils.cron; +package org.apache.dolphinscheduler.service.quartz.cron; -import org.apache.dolphinscheduler.common.enums.CycleEnum; import com.cronutils.model.Cron; import com.cronutils.model.field.expression.Always; import com.cronutils.model.field.expression.QuestionMark; +import org.apache.dolphinscheduler.common.enums.CycleEnum; /** * Crontab Cycle Tool Factory diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleLinks.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleLinks.java similarity index 97% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleLinks.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleLinks.java index 63824bda8e..9f01b18868 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleLinks.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleLinks.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.utils.cron; +package org.apache.dolphinscheduler.service.quartz.cron; -import org.apache.dolphinscheduler.common.enums.CycleEnum; import com.cronutils.model.Cron; +import org.apache.dolphinscheduler.common.enums.CycleEnum; import java.util.ArrayList; import java.util.List; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/ITaskQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/ITaskQueue.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/ITaskQueue.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/ITaskQueue.java index 5beb8111ad..bed8a11247 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/ITaskQueue.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/ITaskQueue.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.queue; +package org.apache.dolphinscheduler.service.queue; import java.util.List; import java.util.Set; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueFactory.java similarity index 93% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueFactory.java index 0a2d943118..6be419f5a9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueFactory.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.queue; +package org.apache.dolphinscheduler.service.queue; -import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.commons.lang.StringUtils; -import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueZkImpl.java similarity index 99% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueZkImpl.java index d442c13ebc..874512c361 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueZkImpl.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.queue; +package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.IpUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.zk.ZookeeperOperator; +import org.apache.dolphinscheduler.service.zk.ZookeeperOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java similarity index 99% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java index f62e106680..135bfdabc6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.zk; +package org.apache.dolphinscheduler.service.zk; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.imps.CuratorFrameworkState; @@ -31,12 +31,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; + import static org.apache.dolphinscheduler.common.Constants.*; /** * abstract zookeeper client */ -public abstract class AbstractZKClient extends ZookeeperCachedOperator{ +public abstract class AbstractZKClient extends ZookeeperCachedOperator { private static final Logger logger = LoggerFactory.getLogger(AbstractZKClient.class); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/DefaultEnsembleProvider.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java similarity index 96% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/DefaultEnsembleProvider.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java index 0cf06c0503..9eedf7a4ca 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/DefaultEnsembleProvider.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.zk; +package org.apache.dolphinscheduler.service.zk; import org.apache.curator.ensemble.EnsembleProvider; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java similarity index 90% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java index 5aa25552d7..dccb768f8b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java @@ -14,22 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.zk; +package org.apache.dolphinscheduler.service.zk; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.TreeCache; import org.apache.curator.framework.recipes.cache.TreeCacheEvent; -import org.apache.curator.framework.recipes.cache.TreeCacheListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import java.nio.charset.StandardCharsets; -import java.util.concurrent.ConcurrentHashMap; - -import static org.apache.dolphinscheduler.common.utils.Preconditions.*; -import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull; @Component public class ZookeeperCachedOperator extends ZookeeperOperator { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperConfig.java similarity index 98% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperConfig.java index 75a9f6c5f4..c6bdfc3b02 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperConfig.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.zk; +package org.apache.dolphinscheduler.service.zk; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.PropertySource; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java similarity index 98% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java index 9442afd7a0..a2cabce805 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.zk; +package org.apache.dolphinscheduler.service.zk; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; @@ -33,12 +33,10 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; import java.nio.charset.StandardCharsets; import java.util.List; -import static org.apache.dolphinscheduler.common.utils.Preconditions.*; import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull; /** diff --git a/dolphinscheduler-common/src/main/resources/quartz.properties b/dolphinscheduler-service/src/main/resources/quartz.properties similarity index 96% rename from dolphinscheduler-common/src/main/resources/quartz.properties rename to dolphinscheduler-service/src/main/resources/quartz.properties index 2e3a2a0dc1..9c8930b647 100644 --- a/dolphinscheduler-common/src/main/resources/quartz.properties +++ b/dolphinscheduler-service/src/main/resources/quartz.properties @@ -59,6 +59,6 @@ org.quartz.jobStore.dataSource = myDs #============================================================================ # Configure Datasources #============================================================================ -org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.dao.quartz.DruidConnectionProvider +org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider org.quartz.dataSource.myDs.maxConnections = 10 org.quartz.dataSource.myDs.validationQuery = select 1 \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java b/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java similarity index 98% rename from dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java rename to dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java index 5ecc6620dd..6a402b5e67 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java @@ -14,11 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.cron; +package cron; -import org.apache.dolphinscheduler.common.enums.CycleEnum; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; import com.cronutils.builder.CronBuilder; import com.cronutils.model.Cron; import com.cronutils.model.CronType; @@ -26,6 +23,9 @@ import com.cronutils.model.definition.CronDefinitionBuilder; import com.cronutils.model.field.CronField; import com.cronutils.model.field.CronFieldName; import com.cronutils.model.field.expression.*; +import org.apache.dolphinscheduler.common.enums.CycleEnum; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/BaseTaskQueueTest.java b/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java similarity index 90% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/BaseTaskQueueTest.java rename to dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java index 433e4fa30f..a0cc457e22 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/BaseTaskQueueTest.java +++ b/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java @@ -14,9 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.queue; +package queue; -import org.apache.dolphinscheduler.common.zk.ZKServer; +import org.apache.dolphinscheduler.service.queue.ITaskQueue; +import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; import org.junit.*; /** diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java b/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java similarity index 99% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java rename to dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java index b34a7d6924..d29c5aa610 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java +++ b/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.queue; +package queue; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.IpUtils; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/ZKServer.java b/dolphinscheduler-service/src/test/java/queue/ZKServer.java similarity index 99% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/ZKServer.java rename to dolphinscheduler-service/src/test/java/queue/ZKServer.java index fc39e62ed8..65fb95c02b 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/ZKServer.java +++ b/dolphinscheduler-service/src/test/java/queue/ZKServer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.zk; +package queue; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java b/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java similarity index 97% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java rename to dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java index dcb0e1370e..a1b85f1b12 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java +++ b/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java @@ -14,8 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package utils; +import org.apache.dolphinscheduler.common.utils.Preconditions; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -35,7 +36,7 @@ public class PreconditionsTest { public void testCheckNotNull() throws Exception { String testReference = "test reference"; //test reference is not null - Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference)); + Assert.assertEquals(testReference, Preconditions.checkNotNull(testReference)); Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"reference is null")); Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"%s is null",testReference)); diff --git a/pom.xml b/pom.xml index de7ee81403..47e0f3ec79 100644 --- a/pom.xml +++ b/pom.xml @@ -229,7 +229,12 @@ org.apache.dolphinscheduler - dolphinscheduler-rpc + dolphinscheduler-remote + ${project.version} + + + org.apache.dolphinscheduler + dolphinscheduler-service ${project.version} @@ -774,9 +779,6 @@ **/dolphinscheduler-ui/src/view/common/outro.inc **/dolphinscheduler-ui/src/view/common/meta.inc **/dolphinscheduler-ui/src/combo/1.0.0/3rd.css - - **/dolphinscheduler-rpc/src/main/java/org/apache/dolphinscheduler/rpc/LogViewServiceGrpc.java - true @@ -863,8 +865,9 @@ dolphinscheduler-api dolphinscheduler-dao dolphinscheduler-alert - dolphinscheduler-rpc dolphinscheduler-dist + dolphinscheduler-remote + dolphinscheduler-service From 99ac7398518ba5f0970b192cecd40024d8006075 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sat, 15 Feb 2020 18:55:16 +0800 Subject: [PATCH 42/43] rename vaild to valid (#1961) --- .../dolphinscheduler/api/service/ExecutorService.java | 6 +++--- .../org/apache/dolphinscheduler/api/utils/CheckUtils.java | 2 +- .../org/apache/dolphinscheduler/common/utils/JSONUtils.java | 2 +- .../apache/dolphinscheduler/common/utils/JSONUtilsTest.java | 6 +++--- .../dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 6edd48d499..0389890691 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -117,7 +117,7 @@ public class ExecutorService extends BaseService{ } if (!checkTenantSuitable(processDefinition)){ - logger.error("there is not any vaild tenant for the process definition: id:{},name:{}, ", + logger.error("there is not any valid tenant for the process definition: id:{},name:{}, ", processDefinition.getId(), processDefinition.getName()); putMsg(result, Status.TENANT_NOT_SUITABLE); return result; @@ -206,7 +206,7 @@ public class ExecutorService extends BaseService{ return checkResult; } if (!checkTenantSuitable(processDefinition)){ - logger.error("there is not any vaild tenant for the process definition: id:{},name:{}, ", + logger.error("there is not any valid tenant for the process definition: id:{},name:{}, ", processDefinition.getId(), processDefinition.getName()); putMsg(result, Status.TENANT_NOT_SUITABLE); } @@ -539,7 +539,7 @@ public class ExecutorService extends BaseService{ } } }else{ - logger.error("there is not vaild schedule date for the process definition: id:{},date:{}", + logger.error("there is not valid schedule date for the process definition: id:{},date:{}", processDefineId, schedule); } }else{ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index c5c702404d..a888712511 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -86,7 +86,7 @@ public class CheckUtils { * @return true if other parameters are valid, otherwise return false */ public static boolean checkOtherParams(String otherParams) { - return StringUtils.isNotEmpty(otherParams) && !JSONUtils.checkJsonVaild(otherParams); + return StringUtils.isNotEmpty(otherParams) && !JSONUtils.checkJsonValid(otherParams); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 9e9e4f6546..ec523b1ff2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -126,7 +126,7 @@ public class JSONUtils { * @param json json * @return true if valid */ - public static boolean checkJsonVaild(String json) { + public static boolean checkJsonValid(String json) { if (StringUtils.isEmpty(json)) { return false; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index 799874ad71..bd924e4852 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -117,9 +117,9 @@ public class JSONUtilsTest { } @Test - public void testCheckJsonVaild() { - Assert.assertTrue(JSONUtils.checkJsonVaild("3")); - Assert.assertFalse(JSONUtils.checkJsonVaild("")); + public void testCheckJsonValid() { + Assert.assertTrue(JSONUtils.checkJsonValid("3")); + Assert.assertFalse(JSONUtils.checkJsonValid("")); } @Test diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java index c60cc3a655..16ba4b06c4 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java @@ -121,7 +121,7 @@ public class TaskInstanceMapperTest { } /** - * test find vaild task list by process instance id + * test find valid task list by process instance id */ @Test public void testFindValidTaskListByProcessId() { From c6f252451bd68fe54f4da95eeeb0c76fbf3c6c0b Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Sat, 15 Feb 2020 19:58:45 +0800 Subject: [PATCH 43/43] =?UTF-8?q?1=EF=BC=8Cquartz.properties=20add=20conf?= =?UTF-8?q?=20category=202=EF=BC=8Cdolphinscheduler-daemon.sh=20modify=20(?= =?UTF-8?q?#1960)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 1，remove dolphinscheduler-rpc module 2，add dolphinscheduler-remote module 3，add dolphinscheduler-service module 4，refactor LoggerServer module (#1925) * 1，remove dolphinscheduler-rpc module 2，add dolphinscheduler-remote module 3，add dolphinscheduler-service module 4，refactor LoggerServer module * ProcessUtils modify * Refactor architecture (#1926) * move version to parent pom * move version properties to parent pom for easy management * remove freemarker dependency * delete CombinedApplicationServer * #1871 correct spelling * #1873 some updates for TaskQueueZkImpl * #1875 remove unused properties in pom * #1878 1. remove tomcat dependency 2. remove combined_logback.xml in api module 3. format pom.xml for not aligning * #1885 fix api server startup failure 1. add jsp-2.1 dependency 2. remove jasper-runtime dependency * add stringutils ut (#1921) * add stringutils ut * Newfeature for #1675. (#1908) Continue to finish the rest works, add the cache feature for dependence,mr,python,sub_process,procedure and shell. * Add modify user name for process definition (#1919) * class overrides equals() and should therefore also override hashCode() * #1862 add modify user in process difinition list * #1862 add pg-1.2.2 ddl.sql * modify ScriptRunnerTest * add updateProessDifinition UT * modify updateProcessDifinition UT * modify updateProcessDifinition UT * modify mysql 1.2.2 ddl.sql&dml.sql * add scope test to mysql in pom * modify pg-1.2.2 ddl.sql * refactor module * updates Co-authored-by: khadgarmage Co-authored-by: zhukai Co-authored-by: Yelli * dolphinscheduler-common remove spring (#1931) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * SpringApplicationContext class title add license (#1932) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * add license (#1934) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * Refactor architecture (#1936) * move datasource classes to dao module * fix send4LetterWord bug * LoggerServiceTest remove ProcessDao (#1944) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * LoggerServiceTest remove ProcessDao * exclude jasper-compiler in case of runtime conflict (#1938) * move datasource classes to dao module * fix send4LetterWord bug * exclude jasper-compiler in case of runtime conflict * DataAnaylysisServiceTest and ProcessDefinitionService modify * remote module add comment * OSUtilsTest modify * add finally block to close channel (#1951) * move datasource classes to dao module * fix send4LetterWord bug * exclude jasper-compiler in case of runtime conflict * add finally block to close channel * 1，quartz.properties add conf category 2，dolphinscheduler-daemon.sh modify * dolphinscheduler-binary.xml modify Co-authored-by: Tboy Co-authored-by: khadgarmage Co-authored-by: zhukai Co-authored-by: Yelli --- .../src/main/assembly/dolphinscheduler-binary.xml | 15 +++++++++++++++ script/dolphinscheduler-daemon.sh | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml index b4326c6795..28bbb361cd 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml @@ -112,6 +112,21 @@ + + + + ${basedir}/../dolphinscheduler-service/src/main/resources + + **/*.properties + **/*.xml + **/*.json + **/*.yml + + conf + + + + ${basedir}/../dolphinscheduler-server/target/dolphinscheduler-server-${project.version} diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index d4db103fe1..d942bca7d2 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -69,7 +69,7 @@ elif [ "$command" = "alert-server" ]; then LOG_FILE="-Dserver=alert-server" CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then - CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer + CLASS=org.apache.dolphinscheduler.server.log.LoggerServer elif [ "$command" = "combined-server" ]; then LOG_FILE="-Dlogging.config=classpath:combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true" CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer