From 6dc500915c0a45d087c4adc2466d6d2cd1f96e7c Mon Sep 17 00:00:00 2001 From: sky <740051880@qq.com> Date: Wed, 5 Aug 2020 17:27:55 +0800 Subject: [PATCH 01/24] [Feature-3392][api-server] (#3403) * feature user register fix bug fix security problem fix security problem * activate user * fix confilct * fix confilct and fix some bug * fix cr problem Co-authored-by: dev_sky --- .../api/controller/UsersController.java | 30 ++++++++-- .../api/service/UsersService.java | 49 +++++++++++++++- .../api/controller/UsersControllerTest.java | 17 +++++- .../api/service/UsersServiceTest.java | 57 +++++++++++++++++-- 4 files changed, 138 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java index 39b9b06337..ab4dce972d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java @@ -432,14 +432,34 @@ public class UsersController extends BaseController { @RequestParam(value = "userPassword") String userPassword, @RequestParam(value = "repeatPassword") String repeatPassword, @RequestParam(value = "email") String email) throws Exception { - userName = userName.replaceAll("[\n|\r|\t]", ""); - userPassword = userPassword.replaceAll("[\n|\r|\t]", ""); - repeatPassword = repeatPassword.replaceAll("[\n|\r|\t]", ""); - email = email.replaceAll("[\n|\r|\t]", ""); + userName = ParameterUtils.handleEscapes(userName); + userPassword = ParameterUtils.handleEscapes(userPassword); + repeatPassword = ParameterUtils.handleEscapes(repeatPassword); + email = ParameterUtils.handleEscapes(email); logger.info("user self-register, userName: {}, userPassword {}, repeatPassword {}, eamil {}", - userName, userPassword, repeatPassword, email); + userName, Constants.PASSWORD_DEFAULT, Constants.PASSWORD_DEFAULT, email); Map result = usersService.registerUser(userName, userPassword, repeatPassword, email); return returnDataList(result); } + /** + * user activate + * + * @param userName user name + */ + @ApiOperation(value="activateUser",notes = "ACTIVATE_USER_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "userName", value = "USER_NAME", type = "String"), + }) + @PostMapping("/activate") + @ResponseStatus(HttpStatus.OK) + @ApiException(UPDATE_USER_ERROR) + public Result activateUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "userName") String userName) { + userName = ParameterUtils.handleEscapes(userName); + logger.info("login user {}, activate user, userName: {}", + loginUser.getUserName(), userName); + Map result = usersService.activateUser(loginUser, userName); + return returnDataList(result); + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index cbd795cce4..6dcb327597 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.api.utils.CheckUtils; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.*; @@ -917,10 +918,11 @@ public class UsersService extends BaseService { * @param repeatPassword repeat password * @param email email * @return register result code + * @throws Exception exception */ @Transactional(rollbackFor = RuntimeException.class) public Map registerUser(String userName, String userPassword, String repeatPassword, String email) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //check user params String msg = this.checkUserParams(userName, userPassword, email, ""); @@ -934,10 +936,51 @@ public class UsersService extends BaseService { putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "two passwords are not same"); return result; } - - createUser(userName, userPassword, email, 1, "", "", 0); + User user = createUser(userName, userPassword, email, 1, "", "", Flag.NO.ordinal()); putMsg(result, Status.SUCCESS); + result.put(Constants.DATA_LIST, user); return result; } + /** + * activate user, only system admin have permission, change user state code 0 to 1 + * + * @param loginUser login user + * @return create result code + */ + public Map activateUser(User loginUser, String userName) { + Map result = new HashMap<>(); + result.put(Constants.STATUS, false); + + if (!isAdmin(loginUser)) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + + if (!CheckUtils.checkUserName(userName)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, userName); + return result; + } + + User user = userMapper.queryByUserNameAccurately(userName); + + if (user == null) { + putMsg(result, Status.USER_NOT_EXIST, userName); + return result; + } + + if (user.getState() != Flag.NO.ordinal()) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, userName); + return result; + } + + user.setState(Flag.YES.ordinal()); + Date now = new Date(); + user.setUpdateTime(now); + userMapper.updateById(user); + User responseUser = userMapper.queryByUserNameAccurately(userName); + putMsg(result, Status.SUCCESS); + result.put(Constants.DATA_LIST, responseUser); + return result; + } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java index fc86632ed7..e6796d8c47 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java @@ -285,6 +285,21 @@ public class UsersControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + } + + @Test + public void testActivateUser() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("userName","user_test"); + + MvcResult mvcResult = mockMvc.perform(post("/users/activate") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index 6939e6a280..19562229c6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -462,42 +462,87 @@ public class UsersServiceTest { try { //userName error Map result = usersService.registerUser(userName, userPassword, repeatPassword, email); - logger.info(result.toString()); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); userName = "userTest0002"; userPassword = "userTest000111111111111111"; //password error result = usersService.registerUser(userName, userPassword, repeatPassword, email); - logger.info(result.toString()); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); userPassword = "userTest0002"; email = "1q.com"; //email error result = usersService.registerUser(userName, userPassword, repeatPassword, email); - logger.info(result.toString()); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); //repeatPassword error email = "7400@qq.com"; repeatPassword = "userPassword"; result = usersService.registerUser(userName, userPassword, repeatPassword, email); - logger.info(result.toString()); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); //success repeatPassword = "userTest0002"; result = usersService.registerUser(userName, userPassword, repeatPassword, email); - logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } catch (Exception e) { - logger.error(Status.CREATE_USER_ERROR.getMsg(),e); Assert.assertTrue(false); } } + + @Test + public void testActivateUser() { + User user = new User(); + user.setUserType(UserType.GENERAL_USER); + String userName = "userTest0002~"; + try { + //not admin + Map result = usersService.activateUser(user, userName); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + //userName error + user.setUserType(UserType.ADMIN_USER); + result = usersService.activateUser(user, userName); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + //user not exist + userName = "userTest10013"; + result = usersService.activateUser(user, userName); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + + //user state error + userName = "userTest0001"; + when(userMapper.queryByUserNameAccurately(userName)).thenReturn(getUser()); + result = usersService.activateUser(user, userName); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + //success + when(userMapper.queryByUserNameAccurately(userName)).thenReturn(getDisabledUser()); + result = usersService.activateUser(user, userName); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } catch (Exception e) { + Assert.assertTrue(false); + } + } + + /** + * get disabled user + * @return + */ + private User getDisabledUser() { + + User user = new User(); + user.setUserType(UserType.GENERAL_USER); + user.setUserName("userTest0001"); + user.setUserPassword("userTest0001"); + user.setState(0); + return user; + } + + /** * get user * @return From 2b990e1f000ba583504585624591e8403958c374 Mon Sep 17 00:00:00 2001 From: JinyLeeChina <42576980+JinyLeeChina@users.noreply.github.com> Date: Wed, 5 Aug 2020 18:59:49 +0800 Subject: [PATCH 02/24] Update actions.js (#3401) --- .../src/js/conf/home/store/dag/actions.js | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js index f933eaede4..a549aafaa2 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js @@ -19,6 +19,25 @@ import _ from 'lodash' import io from '@/module/io' import { tasksState } from '@/conf/home/pages/dag/_source/config' +// delete 'definitionList' from tasks +const deleteDefinitionList = (tasks) => { + const newTasks = []; + tasks.forEach(item => { + const newItem = Object.assign({}, item); + if(newItem.dependence && newItem.dependence.dependTaskList) { + newItem.dependence.dependTaskList.forEach(dependTaskItem => { + if (dependTaskItem.dependItemList) { + dependTaskItem.dependItemList.forEach(dependItem => { + Reflect.deleteProperty(dependItem, 'definitionList'); + }) + } + }) + } + newTasks.push(newItem); + }); + return newTasks; +} + export default { /** * Task status acquisition @@ -193,7 +212,7 @@ export default { return new Promise((resolve, reject) => { const data = { globalParams: state.globalParams, - tasks: state.tasks, + tasks: deleteDefinitionList(state.tasks), tenantId: state.tenantId, timeout: state.timeout } @@ -217,7 +236,7 @@ export default { return new Promise((resolve, reject) => { const data = { globalParams: state.globalParams, - tasks: state.tasks, + tasks: deleteDefinitionList(state.tasks), tenantId: state.tenantId, timeout: state.timeout } From f784ce504f06d8a6b444a98ead01c6aa8b184679 Mon Sep 17 00:00:00 2001 From: wuchunfu <319355703@qq.com> Date: Fri, 7 Aug 2020 10:20:56 +0800 Subject: [PATCH 03/24] [Fix-3256][ui] Fix admin user info update error (#3425) * [PROPOSAL-3139] Datasource selection changes from radio to select * [PROPOSAL-3139] Datasource selection changes from radio to select * [BUG FIX] issues #3256 * [BUG FIX] issues #3256 * [BUG FIX] issues #3256 * [Fix-3256][ui] Fix admin user info update error * [Fix-3256][ui] Fix admin user info update error * [Fix-3256][ui] Fix admin user info update error * [Fix-3256][ui] Fix admin user info update error * reset createUser.vue * [Fix-3256][ui] Fix admin user info update error Co-authored-by: dailidong --- sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql | 3 ++- sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql | 3 ++- sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql index 6aeafcd700..e2a08756a6 100644 --- a/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql +++ b/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql @@ -22,4 +22,5 @@ UPDATE QRTZ_FIRED_TRIGGERS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='E UPDATE QRTZ_JOB_DETAILS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; UPDATE QRTZ_JOB_DETAILS SET JOB_CLASS_NAME='org.apache.dolphinscheduler.dao.quartz.ProcessScheduleJob' WHERE JOB_CLASS_NAME='cn.escheduler.server.quartz.ProcessScheduleJob'; UPDATE QRTZ_LOCKS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; -UPDATE QRTZ_SCHEDULER_STATE SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; \ No newline at end of file +UPDATE QRTZ_SCHEDULER_STATE SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file diff --git a/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql index 6f0e145176..661ed9e827 100644 --- a/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql +++ b/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql @@ -23,4 +23,5 @@ UPDATE t_ds_process_instance instance SET `worker_group`=IFNULL((SELECT name fro UPDATE t_ds_task_instance instance SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE instance.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_schedules schedule SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE schedule.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); -UPDATE t_ds_error_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); \ No newline at end of file +UPDATE t_ds_error_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file diff --git a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql index fba03152ee..a748eae6cf 100644 --- a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql +++ b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql @@ -21,4 +21,5 @@ UPDATE t_ds_process_instance instance SET worker_group=COALESCE((SELECT name fro UPDATE t_ds_task_instance instance SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE instance.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_schedules schedule SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE schedule.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); -UPDATE t_ds_error_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); \ No newline at end of file +UPDATE t_ds_error_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file From 40a21816abc0b8d767dacacb674340c9fda09f9c Mon Sep 17 00:00:00 2001 From: zixi0825 <649790970@qq.com> Date: Fri, 7 Aug 2020 13:58:14 +0800 Subject: [PATCH 04/24] [Fix] [Server] Fix the spell error and add log in SqlTask (#3243) * fix the spell and modify logger print format --- .../impl/TaskInstanceCacheManagerImpl.java | 2 +- .../TaskExecutionContextCacheManagerImpl.java | 2 +- .../server/worker/task/sql/SqlTask.java | 24 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/TaskInstanceCacheManagerImpl.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/TaskInstanceCacheManagerImpl.java index c149ac3335..4d55490a8d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/TaskInstanceCacheManagerImpl.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/TaskInstanceCacheManagerImpl.java @@ -36,7 +36,7 @@ import java.util.concurrent.ConcurrentHashMap; public class TaskInstanceCacheManagerImpl implements TaskInstanceCacheManager { /** - * taskInstance caceh + * taskInstance cache */ private Map taskInstanceCache = new ConcurrentHashMap<>(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java index 009332f05c..9c92fb2d64 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java @@ -32,7 +32,7 @@ public class TaskExecutionContextCacheManagerImpl implements TaskExecutionContex /** - * taskInstance caceh + * taskInstance cache */ private Map taskExecutionContextCache = new ConcurrentHashMap<>(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index acc75d70d2..939426b3bf 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -147,8 +147,8 @@ public class SqlTask extends AbstractTask { } /** - * ready to execute SQL and parameter entity Map - * @return + * ready to execute SQL and parameter entity Map + * @return SqlBinds */ private SqlBinds getSqlAndSqlParamsMap(String sql) { Map sqlParamsMap = new HashMap<>(); @@ -250,7 +250,7 @@ public class SqlTask extends AbstractTask { * result process * * @param resultSet resultSet - * @throws Exception + * @throws Exception Exception */ private void resultProcess(ResultSet resultSet) throws Exception{ ArrayNode resultJSONArray = JSONUtils.createArrayNode(); @@ -293,7 +293,7 @@ public class SqlTask extends AbstractTask { } /** - * post psql + * post sql * * @param connection connection * @param postStatementsBinds postStatementsBinds @@ -329,7 +329,7 @@ public class SqlTask extends AbstractTask { * create connection * * @return connection - * @throws Exception + * @throws Exception Exception */ private Connection createConnection() throws Exception{ // if hive , load connection params if exists @@ -367,7 +367,7 @@ public class SqlTask extends AbstractTask { try { resultSet.close(); } catch (SQLException e) { - + logger.error("close result set error : {}",e.getMessage(),e); } } @@ -375,7 +375,7 @@ public class SqlTask extends AbstractTask { try { pstmt.close(); } catch (SQLException e) { - + logger.error("close prepared statement error : {}",e.getMessage(),e); } } @@ -383,17 +383,17 @@ public class SqlTask extends AbstractTask { try { connection.close(); } catch (SQLException e) { - + logger.error("close connection error : {}",e.getMessage(),e); } } } /** * preparedStatement bind - * @param connection - * @param sqlBinds - * @return - * @throws Exception + * @param connection connection + * @param sqlBinds sqlBinds + * @return PreparedStatement + * @throws Exception Exception */ private PreparedStatement prepareStatementAndBind(Connection connection, SqlBinds sqlBinds) throws Exception { // is the timeout set From 4fb06a736dd6b3f420dca354bbf32262dbe98452 Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Mon, 10 Aug 2020 14:36:31 +0800 Subject: [PATCH 05/24] [Test][api] Introduce api controller unit test example (#3447) * [Test][api] Introduce api controller test example * Add controller test --- .../ProcessInstanceControllerTest.java | 83 +++++++++---------- pom.xml | 1 + 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java index 5189097e68..bdd762afa8 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java @@ -16,29 +16,27 @@ */ package org.apache.dolphinscheduler.api.controller; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + import org.junit.Assert; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - /** * process instance controller test */ public class ProcessInstanceControllerTest extends AbstractControllerTest { - private static Logger logger = LoggerFactory.getLogger(ProcessInstanceControllerTest.class); @Test public void testQueryProcessInstanceList() throws Exception { @@ -52,31 +50,30 @@ public class ProcessInstanceControllerTest extends AbstractControllerTest { paramsMap.add("pageNo", "2"); paramsMap.add("pageSize", "2"); - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/list-paging","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/list-paging", "cxc_1113") .header("sessionId", sessionId) .params(paramsMap)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertNotNull(result); Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); } @Test public void testQueryTaskListByProcessId() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/task-list-by-process-id","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/task-list-by-process-id", "cxc_1113") .header(SESSION_ID, sessionId) - .param("processInstanceId","1203")) + .param("processInstanceId", "1203")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - assert result != null; - Assert.assertEquals(Status.PROJECT_NOT_FOUNT.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT.getCode(), result.getCode().intValue()); } @Test @@ -91,110 +88,108 @@ public class ProcessInstanceControllerTest extends AbstractControllerTest { paramsMap.add("syncDefine", "false"); paramsMap.add("locations", locations); paramsMap.add("connects", "[]"); -// paramsMap.add("flag", "2"); - MvcResult mvcResult = mockMvc.perform(post("/projects/{projectName}/instance/update","cxc_1113") + MvcResult mvcResult = mockMvc.perform(post("/projects/{projectName}/instance/update", "cxc_1113") .header("sessionId", sessionId) .params(paramsMap)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertNotNull(result); Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); } @Test public void testQueryProcessInstanceById() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-by-id","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-by-id", "cxc_1113") .header(SESSION_ID, sessionId) - .param("processInstanceId","1203")) + .param("processInstanceId", "1203")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } - @Test public void testQuerySubProcessInstanceByTaskId() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-sub-process","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-sub-process", "cxc_1113") .header(SESSION_ID, sessionId) - .param("taskId","1203")) + .param("taskId", "1203")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.TASK_INSTANCE_NOT_EXISTS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.TASK_INSTANCE_NOT_EXISTS.getCode(), result.getCode().intValue()); } @Test public void testQueryParentInstanceBySubId() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-parent-process","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-parent-process", "cxc_1113") .header(SESSION_ID, sessionId) - .param("subId","1204")) + .param("subId", "1204")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.PROCESS_INSTANCE_NOT_SUB_PROCESS_INSTANCE.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.PROCESS_INSTANCE_NOT_SUB_PROCESS_INSTANCE.getCode(), result.getCode().intValue()); } @Test public void testViewVariables() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/view-variables","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/view-variables", "cxc_1113") .header(SESSION_ID, sessionId) - .param("processInstanceId","1204")) + .param("processInstanceId", "1204")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } @Test public void testDeleteProcessInstanceById() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/delete","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/delete", "cxc_1113") .header(SESSION_ID, sessionId) - .param("processInstanceId","1204")) + .param("processInstanceId", "1204")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } @Test public void testBatchDeleteProcessInstanceByIds() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/batch-delete","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/batch-delete", "cxc_1113") .header(SESSION_ID, sessionId) - .param("processInstanceIds","1205，1206")) + .param("processInstanceIds", "1205，1206")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR.getCode(), result.getCode().intValue()); } } diff --git a/pom.xml b/pom.xml index b469f38c1d..0b546b3ec1 100644 --- a/pom.xml +++ b/pom.xml @@ -702,6 +702,7 @@ **/alert/utils/JSONUtilsTest.java **/alert/plugin/EmailAlertPluginTest.java + **/api/controller/ProcessDefinitionControllerTest.java **/api/dto/resources/filter/ResourceFilterTest.java **/api/dto/resources/visitor/ResourceTreeVisitorTest.java **/api/enums/testGetEnum.java From f7f07608ed68010af884d7ccd4a346b0422e3d3a Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Mon, 10 Aug 2020 14:46:59 +0800 Subject: [PATCH 06/24] [Improvement][api] Introduce access token service interface for clear code (#3438) --- .../api/controller/AccessTokenController.java | 30 ++- .../api/service/AccessTokenService.java | 126 ++---------- .../service/impl/AccessTokenServiceImpl.java | 186 ++++++++++++++++++ .../api/service/AccessTokenServiceTest.java | 109 +++++----- 4 files changed, 266 insertions(+), 185 deletions(-) create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java index 8731b264e9..2457177cdf 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java @@ -17,6 +17,12 @@ package org.apache.dolphinscheduler.api.controller; +import static org.apache.dolphinscheduler.api.enums.Status.CREATE_ACCESS_TOKEN_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_ACCESS_TOKEN_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.GENERATE_TOKEN_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ACCESSTOKEN_LIST_PAGING_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ACCESS_TOKEN_ERROR; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AccessTokenService; @@ -24,20 +30,26 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiImplicitParam; -import io.swagger.annotations.ApiImplicitParams; -import io.swagger.annotations.ApiOperation; + +import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; -import springfox.documentation.annotations.ApiIgnore; - -import java.util.Map; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; -import static org.apache.dolphinscheduler.api.enums.Status.*; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import springfox.documentation.annotations.ApiIgnore; /** * access token controller diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java index 5d176961bb..98eef47090 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java @@ -16,35 +16,14 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.dao.entity.AccessToken; import org.apache.dolphinscheduler.dao.entity.User; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.EncryptionUtils; -import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import java.util.*; +import java.util.Map; /** - * user service + * access token service */ -@Service -public class AccessTokenService extends BaseService { - - private static final Logger logger = LoggerFactory.getLogger(AccessTokenService.class); - - @Autowired - private AccessTokenMapper accessTokenMapper; - +public interface AccessTokenService { /** * query access token list @@ -55,123 +34,44 @@ public class AccessTokenService extends BaseService { * @param pageSize page size * @return token list for page number and page size */ - public Map queryAccessTokenList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(5); - - PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - Page page = new Page(pageNo, pageSize); - int userId = loginUser.getId(); - if (loginUser.getUserType() == UserType.ADMIN_USER){ - userId = 0; - } - IPage accessTokenList = accessTokenMapper.selectAccessTokenPage(page, searchVal, userId); - pageInfo.setTotalCount((int)accessTokenList.getTotal()); - pageInfo.setLists(accessTokenList.getRecords()); - result.put(Constants.DATA_LIST, pageInfo); - putMsg(result, Status.SUCCESS); - - return result; - } + Map queryAccessTokenList(User loginUser, String searchVal, Integer pageNo, Integer pageSize); /** * create token + * * @param userId token for user * @param expireTime token expire time * @param token token string * @return create result code */ - public Map createToken(int userId, String expireTime, String token) { - Map result = new HashMap<>(5); - - if (userId <= 0) { - throw new IllegalArgumentException("User id should not less than or equals to 0."); - } - AccessToken accessToken = new AccessToken(); - accessToken.setUserId(userId); - accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); - accessToken.setToken(token); - accessToken.setCreateTime(new Date()); - accessToken.setUpdateTime(new Date()); - - // insert - int insert = accessTokenMapper.insert(accessToken); - - if (insert > 0) { - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.CREATE_ACCESS_TOKEN_ERROR); - } - - return result; - } + Map createToken(int userId, String expireTime, String token); /** * generate token + * * @param userId token for user * @param expireTime token expire time * @return token string */ - public Map generateToken(int userId, String expireTime) { - Map result = new HashMap<>(5); - String token = EncryptionUtils.getMd5(userId + expireTime + String.valueOf(System.currentTimeMillis())); - result.put(Constants.DATA_LIST, token); - putMsg(result, Status.SUCCESS); - return result; - } + Map generateToken(int userId, String expireTime); /** - * delete access token + * delete access token + * * @param loginUser login user * @param id token id * @return delete result code */ - public Map delAccessTokenById(User loginUser, int id) { - Map result = new HashMap<>(5); - - AccessToken accessToken = accessTokenMapper.selectById(id); - - if (accessToken == null) { - logger.error("access token not exist, access token id {}", id); - putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); - return result; - } - - if (loginUser.getId() != accessToken.getUserId() && - loginUser.getUserType() != UserType.ADMIN_USER) { - putMsg(result, Status.USER_NO_OPERATION_PERM); - return result; - } - - accessTokenMapper.deleteById(id); - putMsg(result, Status.SUCCESS); - return result; - } + Map delAccessTokenById(User loginUser, int id); /** * update token by id + * * @param id token id * @param userId token for user * @param expireTime token expire time * @param token token string * @return update result code */ - public Map updateToken(int id,int userId, String expireTime, String token) { - Map result = new HashMap<>(5); - - AccessToken accessToken = accessTokenMapper.selectById(id); - if (accessToken == null) { - logger.error("access token not exist, access token id {}", id); - putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); - return result; - } - accessToken.setUserId(userId); - accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); - accessToken.setToken(token); - accessToken.setUpdateTime(new Date()); - - accessTokenMapper.updateById(accessToken); - - putMsg(result, Status.SUCCESS); - return result; - } + Map updateToken(int id, int userId, String expireTime, String token); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java new file mode 100644 index 0000000000..7e0b11780c --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service.impl; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.AccessTokenService; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.EncryptionUtils; +import org.apache.dolphinscheduler.dao.entity.AccessToken; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +/** + * access token service impl + */ +@Service +public class AccessTokenServiceImpl extends BaseService implements AccessTokenService { + + private static final Logger logger = LoggerFactory.getLogger(AccessTokenServiceImpl.class); + + @Autowired + private AccessTokenMapper accessTokenMapper; + + /** + * query access token list + * + * @param loginUser login user + * @param searchVal search value + * @param pageNo page number + * @param pageSize page size + * @return token list for page number and page size + */ + public Map queryAccessTokenList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { + Map result = new HashMap<>(5); + + PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); + Page page = new Page<>(pageNo, pageSize); + int userId = loginUser.getId(); + if (loginUser.getUserType() == UserType.ADMIN_USER) { + userId = 0; + } + IPage accessTokenList = accessTokenMapper.selectAccessTokenPage(page, searchVal, userId); + pageInfo.setTotalCount((int) accessTokenList.getTotal()); + pageInfo.setLists(accessTokenList.getRecords()); + result.put(Constants.DATA_LIST, pageInfo); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * create token + * + * @param userId token for user + * @param expireTime token expire time + * @param token token string + * @return create result code + */ + public Map createToken(int userId, String expireTime, String token) { + Map result = new HashMap<>(5); + + if (userId <= 0) { + throw new IllegalArgumentException("User id should not less than or equals to 0."); + } + AccessToken accessToken = new AccessToken(); + accessToken.setUserId(userId); + accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); + accessToken.setToken(token); + accessToken.setCreateTime(new Date()); + accessToken.setUpdateTime(new Date()); + + // insert + int insert = accessTokenMapper.insert(accessToken); + + if (insert > 0) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.CREATE_ACCESS_TOKEN_ERROR); + } + + return result; + } + + /** + * generate token + * + * @param userId token for user + * @param expireTime token expire time + * @return token string + */ + public Map generateToken(int userId, String expireTime) { + Map result = new HashMap<>(5); + String token = EncryptionUtils.getMd5(userId + expireTime + String.valueOf(System.currentTimeMillis())); + result.put(Constants.DATA_LIST, token); + putMsg(result, Status.SUCCESS); + return result; + } + + /** + * delete access token + * + * @param loginUser login user + * @param id token id + * @return delete result code + */ + public Map delAccessTokenById(User loginUser, int id) { + Map result = new HashMap<>(5); + + AccessToken accessToken = accessTokenMapper.selectById(id); + + if (accessToken == null) { + logger.error("access token not exist, access token id {}", id); + putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); + return result; + } + + if (loginUser.getId() != accessToken.getUserId() && + loginUser.getUserType() != UserType.ADMIN_USER) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + + accessTokenMapper.deleteById(id); + putMsg(result, Status.SUCCESS); + return result; + } + + /** + * update token by id + * + * @param id token id + * @param userId token for user + * @param expireTime token expire time + * @param token token string + * @return update result code + */ + public Map updateToken(int id, int userId, String expireTime, String token) { + Map result = new HashMap<>(5); + + AccessToken accessToken = accessTokenMapper.selectById(id); + if (accessToken == null) { + logger.error("access token not exist, access token id {}", id); + putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); + return result; + } + accessToken.setUserId(userId); + accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); + accessToken.setToken(token); + accessToken.setUpdateTime(new Date()); + + accessTokenMapper.updateById(accessToken); + + putMsg(result, Status.SUCCESS); + return result; + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java index f388445f0c..f5543487ea 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java @@ -16,10 +16,12 @@ */ package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import java.util.Calendar; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.AccessTokenServiceImpl; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; @@ -27,9 +29,14 @@ import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.AccessToken; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; -import org.junit.After; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Map; + import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -38,131 +45,109 @@ import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(MockitoJUnitRunner.class) public class AccessTokenServiceTest { - private static final Logger logger = LoggerFactory.getLogger(AccessTokenServiceTest.class); - @InjectMocks - private AccessTokenService accessTokenService ; + private AccessTokenServiceImpl accessTokenService; @Mock private AccessTokenMapper accessTokenMapper; - @Before - public void setUp() { - - } - - - @After - public void after(){ - - } - - @Test - public void testQueryAccessTokenList(){ + @SuppressWarnings("unchecked") + public void testQueryAccessTokenList() { IPage tokenPage = new Page<>(); tokenPage.setRecords(getList()); tokenPage.setTotal(1L); - when(accessTokenMapper.selectAccessTokenPage(any(Page.class),eq("zhangsan"),eq(0))).thenReturn(tokenPage); + when(accessTokenMapper.selectAccessTokenPage(any(Page.class), eq("zhangsan"), eq(0))).thenReturn(tokenPage); - User user =new User(); - Map result = accessTokenService.queryAccessTokenList(user,"zhangsan",1,10); + User user = new User(); + Map result = accessTokenService.queryAccessTokenList(user, "zhangsan", 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); - Assert.assertTrue(pageInfo.getTotalCount()>0); + Assert.assertTrue(pageInfo.getTotalCount() > 0); } @Test - public void testCreateToken(){ - + public void testCreateToken() { - when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2); - Map result = accessTokenService.createToken(1,getDate(),"AccessTokenServiceTest"); + when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2); + Map result = accessTokenService.createToken(1, getDate(), "AccessTokenServiceTest"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } @Test - public void testGenerateToken(){ + public void testGenerateToken() { - Map result = accessTokenService.generateToken(Integer.MAX_VALUE,getDate()); + Map result = accessTokenService.generateToken(Integer.MAX_VALUE, getDate()); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); String token = (String) result.get(Constants.DATA_LIST); Assert.assertNotNull(token); } @Test - public void testDelAccessTokenById(){ + public void testDelAccessTokenById() { when(accessTokenMapper.selectById(1)).thenReturn(getEntity()); User userLogin = new User(); // not exist - Map result = accessTokenService.delAccessTokenById(userLogin,0); + Map result = accessTokenService.delAccessTokenById(userLogin, 0); logger.info(result.toString()); - Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS)); + Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST, result.get(Constants.STATUS)); // no operate - result = accessTokenService.delAccessTokenById(userLogin,1); + result = accessTokenService.delAccessTokenById(userLogin, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //success userLogin.setId(1); userLogin.setUserType(UserType.ADMIN_USER); - result = accessTokenService.delAccessTokenById(userLogin,1); + result = accessTokenService.delAccessTokenById(userLogin, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } @Test - public void testUpdateToken(){ + public void testUpdateToken() { when(accessTokenMapper.selectById(1)).thenReturn(getEntity()); - Map result = accessTokenService.updateToken(1,Integer.MAX_VALUE,getDate(),"token"); + Map result = accessTokenService.updateToken(1, Integer.MAX_VALUE, getDate(), "token"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); // not exist - result = accessTokenService.updateToken(2,Integer.MAX_VALUE,getDate(),"token"); + result = accessTokenService.updateToken(2, Integer.MAX_VALUE, getDate(), "token"); logger.info(result.toString()); - Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS)); + Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST, result.get(Constants.STATUS)); } /** * create entity - * @return */ - private AccessToken getEntity(){ + private AccessToken getEntity() { AccessToken accessToken = new AccessToken(); accessToken.setId(1); accessToken.setUserId(1); accessToken.setToken("AccessTokenServiceTest"); - Date date = DateUtils.add(new Date(),Calendar.DAY_OF_MONTH, 30); + Date date = DateUtils.add(new Date(), Calendar.DAY_OF_MONTH, 30); accessToken.setExpireTime(date); return accessToken; } /** * entity list - * @return */ - private List getList(){ + private List getList() { List list = new ArrayList<>(); list.add(getEntity()); @@ -170,13 +155,11 @@ public class AccessTokenServiceTest { } - /** * get dateStr - * @return */ - private String getDate(){ + private String getDate() { Date date = DateUtils.add(new Date(), Calendar.DAY_OF_MONTH, 30); - return DateUtils.dateToString(date); + return DateUtils.dateToString(date); } } From dce403d062c1a113a40d5576396a5fd41bc90c9d Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Mon, 10 Aug 2020 14:47:16 +0800 Subject: [PATCH 07/24] [Improvement-3369][api] Introduce logger service interface for clear code (#3437) * [Improvement][api] Introduce logger service interface for clear code * Remove the code smell --- .../api/controller/LoggerController.java | 25 ++- .../api/exceptions/ApiExceptionHandler.java | 5 +- .../api/service/LoggerService.java | 127 +++------------ .../api/service/impl/LoggerServiceImpl.java | 146 ++++++++++++++++++ .../api/service/LoggerServiceTest.java | 44 ++++-- .../service/ProcessInstanceServiceTest.java | 60 ++++--- .../service/log/LogClientService.java | 37 +++-- 7 files changed, 281 insertions(+), 163 deletions(-) create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java index a5b8176a48..7d612b8b1d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java @@ -17,25 +17,34 @@ package org.apache.dolphinscheduler.api.controller; +import static org.apache.dolphinscheduler.api.enums.Status.DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_INSTANCE_LOG_ERROR; + import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.LoggerService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.User; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiImplicitParam; -import io.swagger.annotations.ApiImplicitParams; -import io.swagger.annotations.ApiOperation; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; -import springfox.documentation.annotations.ApiIgnore; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; -import static org.apache.dolphinscheduler.api.enums.Status.*; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import springfox.documentation.annotations.ApiIgnore; /** @@ -70,7 +79,7 @@ public class LoggerController extends BaseController { @GetMapping(value = "/detail") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_TASK_INSTANCE_LOG_ERROR) - public Result queryLog(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryLog(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "taskInstanceId") int taskInstanceId, @RequestParam(value = "skipLineNum") int skipNum, @RequestParam(value = "limit") int limit) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java index 90d1afea49..cd6ac2b622 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java @@ -18,17 +18,18 @@ package org.apache.dolphinscheduler.api.exceptions; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestControllerAdvice; import org.springframework.web.method.HandlerMethod; /** * Exception Handler */ -@ControllerAdvice +@RestControllerAdvice @ResponseBody public class ApiExceptionHandler { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java index 3c7b421d5e..14440ee61e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java @@ -16,117 +16,30 @@ */ package org.apache.dolphinscheduler.api.service; -import java.nio.charset.StandardCharsets; -import javax.annotation.PreDestroy; -import org.apache.commons.lang.ArrayUtils; -import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.service.log.LogClientService; -import org.apache.dolphinscheduler.service.process.ProcessService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; /** * log service */ -@Service -public class LoggerService { +public interface LoggerService { + + /** + * view log + * + * @param taskInstId task instance id + * @param skipLineNum skip line number + * @param limit limit + * @return log string data + */ + Result queryLog(int taskInstId, int skipLineNum, int limit); + + + /** + * get log size + * + * @param taskInstId task instance id + * @return log byte array + */ + byte[] getLogBytes(int taskInstId); - private static final Logger logger = LoggerFactory.getLogger(LoggerService.class); - - private static final String LOG_HEAD_FORMAT = "[LOG-PATH]: %s, [HOST]: %s%s"; - - @Autowired - private ProcessService processService; - - private final LogClientService logClient; - - public LoggerService() { - logClient = new LogClientService(); - } - - @PreDestroy - public void close() { - logClient.close(); - } - - /** - * view log - * - * @param taskInstId task instance id - * @param skipLineNum skip line number - * @param limit limit - * @return log string data - */ - public Result queryLog(int taskInstId, int skipLineNum, int limit) { - - TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); - - if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) { - return Result.error(Status.TASK_INSTANCE_NOT_FOUND); - } - - String host = getHost(taskInstance.getHost()); - - Result result = new Result(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); - - logger.info("log host : {} , logPath : {} , logServer port : {}", host, taskInstance.getLogPath(), - Constants.RPC_PORT); - - StringBuilder log = new StringBuilder(); - if (skipLineNum == 0) { - String head = String.format(LOG_HEAD_FORMAT, - taskInstance.getLogPath(), - host, - Constants.SYSTEM_LINE_SEPARATOR); - log.append(head); - } - - log.append(logClient - .rollViewLog(host, Constants.RPC_PORT, taskInstance.getLogPath(), skipLineNum, limit)); - - result.setData(log); - return result; - } - - - /** - * get log size - * - * @param taskInstId task instance id - * @return log byte array - */ - public byte[] getLogBytes(int taskInstId) { - TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); - if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) { - throw new RuntimeException("task instance is null or host is null"); - } - String host = getHost(taskInstance.getHost()); - byte[] head = String.format(LOG_HEAD_FORMAT, - taskInstance.getLogPath(), - host, - Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); - return ArrayUtils.addAll(head, - logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath())); - } - - - /** - * get host - * - * @param address address - * @return old version return true ,otherwise return false - */ - private String getHost(String address) { - if (Host.isOldVersion(address)) { - return address; - } - return Host.of(address).getIp(); - } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java new file mode 100644 index 0000000000..c71f2980f5 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service.impl; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.api.service.LoggerService; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.service.log.LogClientService; +import org.apache.dolphinscheduler.service.process.ProcessService; + +import org.apache.commons.lang.ArrayUtils; + +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * log service + */ +@Service +public class LoggerServiceImpl implements LoggerService { + + private static final Logger logger = LoggerFactory.getLogger(LoggerServiceImpl.class); + + private static final String LOG_HEAD_FORMAT = "[LOG-PATH]: %s, [HOST]: %s%s"; + + @Autowired + private ProcessService processService; + + private LogClientService logClient; + + @PostConstruct + public void init() { + if (Objects.isNull(this.logClient)) { + this.logClient = new LogClientService(); + } + } + + @PreDestroy + public void close() { + if (Objects.nonNull(this.logClient) && this.logClient.isRunning()) { + logClient.close(); + } + } + + /** + * view log + * + * @param taskInstId task instance id + * @param skipLineNum skip line number + * @param limit limit + * @return log string data + */ + @SuppressWarnings("unchecked") + public Result queryLog(int taskInstId, int skipLineNum, int limit) { + + TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); + + if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) { + return Result.error(Status.TASK_INSTANCE_NOT_FOUND); + } + + String host = getHost(taskInstance.getHost()); + + Result result = new Result<>(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); + + logger.info("log host : {} , logPath : {} , logServer port : {}", host, taskInstance.getLogPath(), + Constants.RPC_PORT); + + StringBuilder log = new StringBuilder(); + if (skipLineNum == 0) { + String head = String.format(LOG_HEAD_FORMAT, + taskInstance.getLogPath(), + host, + Constants.SYSTEM_LINE_SEPARATOR); + log.append(head); + } + + log.append(logClient + .rollViewLog(host, Constants.RPC_PORT, taskInstance.getLogPath(), skipLineNum, limit)); + + result.setData(log.toString()); + return result; + } + + + /** + * get log size + * + * @param taskInstId task instance id + * @return log byte array + */ + public byte[] getLogBytes(int taskInstId) { + TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); + if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) { + throw new ServiceException("task instance is null or host is null"); + } + String host = getHost(taskInstance.getHost()); + byte[] head = String.format(LOG_HEAD_FORMAT, + taskInstance.getLogPath(), + host, + Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); + return ArrayUtils.addAll(head, + logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath())); + } + + + /** + * get host + * + * @param address address + * @return old version return true ,otherwise return false + */ + private String getHost(String address) { + if (Boolean.TRUE.equals(Host.isOldVersion(address))) { + return address; + } + return Host.of(address).getIp(); + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 4e41ed39b0..3952a25542 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -17,10 +17,14 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.LoggerServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.service.process.ProcessService; + +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -32,25 +36,30 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; @RunWith(MockitoJUnitRunner.class) -@PrepareForTest({LoggerService.class}) +@PrepareForTest({LoggerServiceImpl.class}) public class LoggerServiceTest { private static final Logger logger = LoggerFactory.getLogger(LoggerServiceTest.class); @InjectMocks - private LoggerService loggerService; + private LoggerServiceImpl loggerService; @Mock private ProcessService processService; + @Before + public void init() { + this.loggerService.init(); + } + @Test - public void testQueryDataSourceList(){ + public void testQueryDataSourceList() { TaskInstance taskInstance = new TaskInstance(); Mockito.when(processService.findTaskInstanceById(1)).thenReturn(taskInstance); - Result result = loggerService.queryLog(2,1,1); + Result result = loggerService.queryLog(2, 1, 1); //TASK_INSTANCE_NOT_FOUND - Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(),result.getCode().intValue()); + Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(), result.getCode().intValue()); try { //HOST NOT FOUND OR ILLEGAL @@ -59,36 +68,36 @@ public class LoggerServiceTest { Assert.assertTrue(true); logger.error("testQueryDataSourceList error {}", e.getMessage()); } - Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(),result.getCode().intValue()); + Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(), result.getCode().intValue()); //SUCCESS taskInstance.setHost("127.0.0.1:8080"); taskInstance.setLogPath("/temp/log"); Mockito.when(processService.findTaskInstanceById(1)).thenReturn(taskInstance); - result = loggerService.queryLog(1,1,1); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + result = loggerService.queryLog(1, 1, 1); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } @Test - public void testGetLogBytes(){ + public void testGetLogBytes() { TaskInstance taskInstance = new TaskInstance(); Mockito.when(processService.findTaskInstanceById(1)).thenReturn(taskInstance); //task instance is null - try{ + try { loggerService.getLogBytes(2); - }catch (RuntimeException e){ + } catch (RuntimeException e) { Assert.assertTrue(true); - logger.error("testGetLogBytes error: {}","task instance is null"); + logger.error("testGetLogBytes error: {}", "task instance is null"); } //task instance host is null - try{ + try { loggerService.getLogBytes(1); - }catch (RuntimeException e){ + } catch (RuntimeException e) { Assert.assertTrue(true); - logger.error("testGetLogBytes error: {}","task instance host is null"); + logger.error("testGetLogBytes error: {}", "task instance host is null"); } //success @@ -100,4 +109,9 @@ public class LoggerServiceTest { } + @After + public void close() { + this.loggerService.close(); + } + } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index 651964bb16..95ce1794c3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -16,16 +16,43 @@ */ package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.LoggerServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DependResult; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.TaskType; +import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.entity.WorkerGroup; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.io.IOException; +import java.text.MessageFormat; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -33,22 +60,13 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.boot.test.context.SpringBootTest; -import java.io.IOException; -import java.text.MessageFormat; -import java.text.ParseException; -import java.util.*; - -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(MockitoJUnitRunner.Silent.class) @SpringBootTest(classes = ApiApplicationServer.class) public class ProcessInstanceServiceTest { - private static final Logger logger = LoggerFactory.getLogger(ProcessInstanceServiceTest.class); @InjectMocks ProcessInstanceService processInstanceService; @@ -78,9 +96,7 @@ public class ProcessInstanceServiceTest { TaskInstanceMapper taskInstanceMapper; @Mock - LoggerService loggerService; - - + LoggerServiceImpl loggerService; @Mock UsersService usersService; @@ -153,16 +169,16 @@ public class ProcessInstanceServiceTest { User loginUser = getAdminUser(); Map result = new HashMap<>(5); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); - int size=10; - String startTime="2020-01-01 00:00:00"; - String endTime="2020-08-02 00:00:00"; + int size = 10; + String startTime = "2020-01-01 00:00:00"; + String endTime = "2020-08-02 00:00:00"; Date start = DateUtils.getScheduleDate(startTime); Date end = DateUtils.getScheduleDate(endTime); //project auth fail when(projectMapper.queryByName(projectName)).thenReturn(null); when(projectService.checkProjectAndAuth(loginUser, null, projectName)).thenReturn(result); - Map proejctAuthFailRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser,projectName,size,startTime,endTime); + Map proejctAuthFailRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser, projectName, size, startTime, endTime); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, proejctAuthFailRes.get(Constants.STATUS)); //project auth success @@ -176,7 +192,7 @@ public class ProcessInstanceServiceTest { when(usersService.queryUser(loginUser.getId())).thenReturn(loginUser); when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(loginUser.getId()); when(usersService.queryUser(processInstance.getExecutorId())).thenReturn(loginUser); - Map successRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser,projectName,size,startTime,endTime); + Map successRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser, projectName, size, startTime, endTime); Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index 92d38d470a..474bf12c77 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -16,13 +16,20 @@ */ package org.apache.dolphinscheduler.service.log; -import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; -import org.apache.dolphinscheduler.remote.command.log.*; +import org.apache.dolphinscheduler.remote.command.log.GetLogBytesRequestCommand; +import org.apache.dolphinscheduler.remote.command.log.GetLogBytesResponseCommand; +import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogRequestCommand; +import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponseCommand; +import org.apache.dolphinscheduler.remote.command.log.RollViewLogRequestCommand; +import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand; +import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; +import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,8 +45,10 @@ public class LogClientService { private final NettyRemotingClient client; + private volatile boolean isRunning; + /** - * request time out + * request time out */ private static final long LOG_REQUEST_TIMEOUT = 10 * 1000L; @@ -50,18 +59,21 @@ public class LogClientService { this.clientConfig = new NettyClientConfig(); this.clientConfig.setWorkerThreads(4); this.client = new NettyRemotingClient(clientConfig); + this.isRunning = true; } /** * close */ - public void close() { + public void close() { this.client.close(); + this.isRunning = false; logger.info("logger client closed"); } /** * roll view log + * * @param host host * @param port port * @param path path @@ -69,7 +81,7 @@ public class LogClientService { * @param limit limit * @return log content */ - public String rollViewLog(String host, int port, String path,int skipLineNum,int limit) { + public String rollViewLog(String host, int port, String path, int skipLineNum, int limit) { logger.info("roll view log, host : {}, port : {}, path {}, skipLineNum {} ,limit {}", host, port, path, skipLineNum, limit); RollViewLogRequestCommand request = new RollViewLogRequestCommand(path, skipLineNum, limit); String result = ""; @@ -77,7 +89,7 @@ public class LogClientService { try { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); - if(response != null){ + if (response != null) { RollViewLogResponseCommand rollReviewLog = JsonSerializer.deserialize( response.getBody(), RollViewLogResponseCommand.class); return rollReviewLog.getMsg(); @@ -92,6 +104,7 @@ public class LogClientService { /** * view log + * * @param host host * @param port port * @param path path @@ -105,7 +118,7 @@ public class LogClientService { try { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); - if(response != null){ + if (response != null) { ViewLogResponseCommand viewLog = JsonSerializer.deserialize( response.getBody(), ViewLogResponseCommand.class); return viewLog.getMsg(); @@ -120,6 +133,7 @@ public class LogClientService { /** * get log size + * * @param host host * @param port port * @param path log path @@ -133,7 +147,7 @@ public class LogClientService { try { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); - if(response != null){ + if (response != null) { GetLogBytesResponseCommand getLog = JsonSerializer.deserialize( response.getBody(), GetLogBytesResponseCommand.class); return getLog.getData(); @@ -149,6 +163,7 @@ public class LogClientService { /** * remove task log + * * @param host host * @param port port * @param path path @@ -162,7 +177,7 @@ public class LogClientService { try { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); - if(response != null){ + if (response != null) { RemoveTaskLogResponseCommand taskLogResponse = JsonSerializer.deserialize( response.getBody(), RemoveTaskLogResponseCommand.class); return taskLogResponse.getStatus(); @@ -174,4 +189,8 @@ public class LogClientService { } return result; } + + public boolean isRunning() { + return isRunning; + } } \ No newline at end of file From 3ed0afa6acaf05cb01156c9292a2df747a37cd07 Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Mon, 10 Aug 2020 14:47:45 +0800 Subject: [PATCH 08/24] [Improvement-3369][api] Introduce data analysis service interface for clear code (#3439) * [Improvement][api] Introduce data analysis service interface for clear code * Remove the bad smell * Remove the bad smell --- .../api/service/DataAnalysisService.java | 317 +-------------- .../service/impl/DataAnalysisServiceImpl.java | 384 ++++++++++++++++++ .../api/service/DataAnalysisServiceTest.java | 73 ++-- .../common/utils/TriFunction.java | 27 ++ 4 files changed, 455 insertions(+), 346 deletions(-) create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TriFunction.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java index 39bec56357..70fb272bea 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java @@ -17,57 +17,14 @@ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.dto.CommandStateCount; -import org.apache.dolphinscheduler.api.dto.DefineUserDto; -import org.apache.dolphinscheduler.api.dto.TaskCountDto; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; -import org.apache.dolphinscheduler.service.process.ProcessService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; +import org.apache.dolphinscheduler.dao.entity.User; -import java.text.MessageFormat; -import java.util.*; +import java.util.Map; /** * data analysis service */ -@Service -public class DataAnalysisService extends BaseService{ - - private static final Logger logger = LoggerFactory.getLogger(DataAnalysisService.class); - - @Autowired - ProjectMapper projectMapper; - - @Autowired - ProjectService projectService; - - @Autowired - ProcessInstanceMapper processInstanceMapper; - - @Autowired - ProcessDefinitionMapper processDefinitionMapper; - - @Autowired - CommandMapper commandMapper; - - @Autowired - ErrorCommandMapper errorCommandMapper; - - @Autowired - TaskInstanceMapper taskInstanceMapper; - - @Autowired - ProcessService processService; +public interface DataAnalysisService { /** * statistical task instance status data @@ -78,46 +35,7 @@ public class DataAnalysisService extends BaseService{ * @param endDate end date * @return task state count data */ - public Map countTaskStateByProject(User loginUser, int projectId, String startDate, String endDate) { - - Map result = new HashMap<>(5); - boolean checkProject = checkProject(loginUser, projectId, result); - if(!checkProject){ - return result; - } - - /** - * find all the task lists in the project under the user - * statistics based on task status execution, failure, completion, wait, total - */ - Date start = null; - Date end = null; - - try { - start = DateUtils.getScheduleDate(startDate); - end = DateUtils.getScheduleDate(endDate); - } catch (Exception e) { - logger.error(e.getMessage(),e); - putErrorRequestParamsMsg(result); - return result; - } - - Integer[] projectIds = getProjectIdsArrays(loginUser, projectId); - List taskInstanceStateCounts = - taskInstanceMapper.countTaskInstanceStateByUser(start, end, projectIds); - - if (taskInstanceStateCounts != null) { - TaskCountDto taskCountResult = new TaskCountDto(taskInstanceStateCounts); - result.put(Constants.DATA_LIST, taskCountResult); - putMsg(result, Status.SUCCESS); - } - return result; - } - - private void putErrorRequestParamsMsg(Map result) { - result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); - result.put(Constants.MSG, MessageFormat.format(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getMsg(), "startDate,endDate")); - } + Map countTaskStateByProject(User loginUser, int projectId, String startDate, String endDate); /** * statistical process instance status data @@ -128,37 +46,7 @@ public class DataAnalysisService extends BaseService{ * @param endDate end date * @return process instance state count data */ - public Map countProcessInstanceStateByProject(User loginUser, int projectId, String startDate, String endDate) { - - Map result = new HashMap<>(5); - boolean checkProject = checkProject(loginUser, projectId, result); - if(!checkProject){ - return result; - } - - Date start = null; - Date end = null; - try { - start = DateUtils.getScheduleDate(startDate); - end = DateUtils.getScheduleDate(endDate); - } catch (Exception e) { - logger.error(e.getMessage(),e); - putErrorRequestParamsMsg(result); - return result; - } - Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); - List processInstanceStateCounts = - processInstanceMapper.countInstanceStateByUser(start, end, - projectIdArray); - - if (processInstanceStateCounts != null) { - TaskCountDto taskCountResult = new TaskCountDto(processInstanceStateCounts); - result.put(Constants.DATA_LIST, taskCountResult); - putMsg(result, Status.SUCCESS); - } - return result; - } - + Map countProcessInstanceStateByProject(User loginUser, int projectId, String startDate, String endDate); /** * statistics the process definition quantities of certain person @@ -167,20 +55,7 @@ public class DataAnalysisService extends BaseService{ * @param projectId project id * @return definition count data */ - public Map countDefinitionByUser(User loginUser, int projectId) { - Map result = new HashMap<>(); - - - Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); - List defineGroupByUsers = processDefinitionMapper.countDefinitionGroupByUser( - loginUser.getId(), projectIdArray,isAdmin(loginUser)); - - DefineUserDto dto = new DefineUserDto(defineGroupByUsers); - result.put(Constants.DATA_LIST, dto); - putMsg(result, Status.SUCCESS); - return result; - } - + Map countDefinitionByUser(User loginUser, int projectId); /** * statistical command status data @@ -191,189 +66,15 @@ public class DataAnalysisService extends BaseService{ * @param endDate end date * @return command state count data */ - public Map countCommandState(User loginUser, int projectId, String startDate, String endDate) { - - Map result = new HashMap<>(5); - boolean checkProject = checkProject(loginUser, projectId, result); - if(!checkProject){ - return result; - } - - /** - * find all the task lists in the project under the user - * statistics based on task status execution, failure, completion, wait, total - */ - Date start = null; - Date end = null; - - if (startDate != null && endDate != null){ - try { - start = DateUtils.getScheduleDate(startDate); - end = DateUtils.getScheduleDate(endDate); - } catch (Exception e) { - logger.error(e.getMessage(),e); - putErrorRequestParamsMsg(result); - return result; - } - } - - - Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); - // count command state - List commandStateCounts = - commandMapper.countCommandState( - loginUser.getId(), - start, - end, - projectIdArray); - - // count error command state - List errorCommandStateCounts = - errorCommandMapper.countCommandState( - start, end, projectIdArray); - - // - Map> dataMap = new HashMap<>(); - - Map commonCommand = new HashMap<>(); - commonCommand.put("commandState",0); - commonCommand.put("errorCommandState",0); - - - // init data map - /** - * START_PROCESS, START_CURRENT_TASK_PROCESS, RECOVER_TOLERANCE_FAULT_PROCESS, RECOVER_SUSPENDED_PROCESS, - START_FAILURE_TASK_PROCESS,COMPLEMENT_DATA,SCHEDULER, REPEAT_RUNNING,PAUSE,STOP,RECOVER_WAITTING_THREAD; - */ - dataMap.put(CommandType.START_PROCESS,commonCommand); - dataMap.put(CommandType.START_CURRENT_TASK_PROCESS,commonCommand); - dataMap.put(CommandType.RECOVER_TOLERANCE_FAULT_PROCESS,commonCommand); - dataMap.put(CommandType.RECOVER_SUSPENDED_PROCESS,commonCommand); - dataMap.put(CommandType.START_FAILURE_TASK_PROCESS,commonCommand); - dataMap.put(CommandType.COMPLEMENT_DATA,commonCommand); - dataMap.put(CommandType.SCHEDULER,commonCommand); - dataMap.put(CommandType.REPEAT_RUNNING,commonCommand); - dataMap.put(CommandType.PAUSE,commonCommand); - dataMap.put(CommandType.STOP,commonCommand); - dataMap.put(CommandType.RECOVER_WAITTING_THREAD,commonCommand); - - // put command state - for (CommandCount executeStatusCount : commandStateCounts){ - Map commandStateCountsMap = new HashMap<>(dataMap.get(executeStatusCount.getCommandType())); - commandStateCountsMap.put("commandState", executeStatusCount.getCount()); - dataMap.put(executeStatusCount.getCommandType(),commandStateCountsMap); - } - - // put error command state - for (CommandCount errorExecutionStatus : errorCommandStateCounts){ - Map errorCommandStateCountsMap = new HashMap<>(dataMap.get(errorExecutionStatus.getCommandType())); - errorCommandStateCountsMap.put("errorCommandState",errorExecutionStatus.getCount()); - dataMap.put(errorExecutionStatus.getCommandType(),errorCommandStateCountsMap); - } - - List list = new ArrayList<>(); - Iterator>> iterator = dataMap.entrySet().iterator(); - while (iterator.hasNext()){ - Map.Entry> next = iterator.next(); - CommandStateCount commandStateCount = new CommandStateCount(next.getValue().get("errorCommandState"), - next.getValue().get("commandState"),next.getKey()); - list.add(commandStateCount); - } - - result.put(Constants.DATA_LIST, list); - putMsg(result, Status.SUCCESS); - return result; - } - - private Integer[] getProjectIdsArrays(User loginUser, int projectId) { - List projectIds = new ArrayList<>(); - if(projectId !=0){ - projectIds.add(projectId); - }else if(loginUser.getUserType() == UserType.GENERAL_USER){ - projectIds = processService.getProjectIdListHavePerm(loginUser.getId()); - if(projectIds.size() ==0 ){ - projectIds.add(0); - } - } - return projectIds.toArray(new Integer[projectIds.size()]); - } + Map countCommandState(User loginUser, int projectId, String startDate, String endDate); /** * count queue state + * * @param loginUser login user * @param projectId project id * @return queue state count data */ - public Map countQueueState(User loginUser, int projectId) { - Map result = new HashMap<>(5); - - boolean checkProject = checkProject(loginUser, projectId, result); - if(!checkProject){ - return result; - } - - List tasksQueueList = new ArrayList<>(); - List tasksKillList = new ArrayList<>(); - - Map dataMap = new HashMap<>(); - if (loginUser.getUserType() == UserType.ADMIN_USER){ - dataMap.put("taskQueue",tasksQueueList.size()); - dataMap.put("taskKill",tasksKillList.size()); - - result.put(Constants.DATA_LIST, dataMap); - putMsg(result, Status.SUCCESS); - return result; - } - - int[] tasksQueueIds = new int[tasksQueueList.size()]; - int[] tasksKillIds = new int[tasksKillList.size()]; - - int i =0; - for (String taskQueueStr : tasksQueueList){ - if (StringUtils.isNotEmpty(taskQueueStr)){ - String[] splits = taskQueueStr.split("_"); - if (splits.length >= 4){ - tasksQueueIds[i++] = Integer.parseInt(splits[3]); - } - } - } - - i = 0; - for (String taskKillStr : tasksKillList){ - if (StringUtils.isNotEmpty(taskKillStr)){ - String[] splits = taskKillStr.split("-"); - if (splits.length == 2){ - tasksKillIds[i++] = Integer.parseInt(splits[1]); - } - } - } - Integer taskQueueCount = 0; - Integer taskKillCount = 0; - - Integer[] projectIds = getProjectIdsArrays(loginUser, projectId); - if (tasksQueueIds.length != 0){ - taskQueueCount = taskInstanceMapper.countTask( - projectIds, - tasksQueueIds); - } - - if (tasksKillIds.length != 0){ - taskKillCount = taskInstanceMapper.countTask(projectIds, tasksKillIds); - } - - dataMap.put("taskQueue",taskQueueCount); - dataMap.put("taskKill",taskKillCount); - - result.put(Constants.DATA_LIST, dataMap); - putMsg(result, Status.SUCCESS); - return result; - } + Map countQueueState(User loginUser, int projectId); - private boolean checkProject(User loginUser, int projectId, Map result){ - if(projectId != 0){ - Project project = projectMapper.selectById(projectId); - return projectService.hasProjectAndPerm(loginUser, project, result); - } - return true; - } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java new file mode 100644 index 0000000000..21313b96d3 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service.impl; + + +import org.apache.dolphinscheduler.api.dto.CommandStateCount; +import org.apache.dolphinscheduler.api.dto.DefineUserDto; +import org.apache.dolphinscheduler.api.dto.TaskCountDto; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.service.DataAnalysisService; +import org.apache.dolphinscheduler.api.service.ProjectService; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.TriFunction; +import org.apache.dolphinscheduler.dao.entity.CommandCount; +import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser; +import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * data analysis service impl + */ +@Service +public class DataAnalysisServiceImpl extends BaseService implements DataAnalysisService { + + private static final Logger logger = LoggerFactory.getLogger(DataAnalysisServiceImpl.class); + + @Autowired + private ProjectMapper projectMapper; + + @Autowired + private ProjectService projectService; + + @Autowired + private ProcessInstanceMapper processInstanceMapper; + + @Autowired + private ProcessDefinitionMapper processDefinitionMapper; + + @Autowired + private CommandMapper commandMapper; + + @Autowired + private ErrorCommandMapper errorCommandMapper; + + @Autowired + private TaskInstanceMapper taskInstanceMapper; + + @Autowired + private ProcessService processService; + + private static final String COMMAND_STATE = "commandState"; + + private static final String ERROR_COMMAND_STATE = "errorCommandState"; + + /** + * statistical task instance status data + * + * @param loginUser login user + * @param projectId project id + * @param startDate start date + * @param endDate end date + * @return task state count data + */ + public Map countTaskStateByProject(User loginUser, int projectId, String startDate, String endDate) { + + return countStateByProject( + loginUser, + projectId, + startDate, + endDate, + (start, end, projectIds) -> this.taskInstanceMapper.countTaskInstanceStateByUser(start, end, projectIds)); + } + + /** + * statistical process instance status data + * + * @param loginUser login user + * @param projectId project id + * @param startDate start date + * @param endDate end date + * @return process instance state count data + */ + public Map countProcessInstanceStateByProject(User loginUser, int projectId, String startDate, String endDate) { + return this.countStateByProject( + loginUser, + projectId, + startDate, + endDate, + (start, end, projectIds) -> this.processInstanceMapper.countInstanceStateByUser(start, end, projectIds)); + } + + private Map countStateByProject(User loginUser, int projectId, String startDate, String endDate + , TriFunction> instanceStateCounter) { + Map result = new HashMap<>(5); + boolean checkProject = checkProject(loginUser, projectId, result); + if (!checkProject) { + return result; + } + + Date start; + Date end; + try { + start = DateUtils.getScheduleDate(startDate); + end = DateUtils.getScheduleDate(endDate); + } catch (Exception e) { + logger.error(e.getMessage(), e); + putErrorRequestParamsMsg(result); + return result; + } + Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); + List processInstanceStateCounts = + instanceStateCounter.apply(start, end, projectIdArray); + + if (processInstanceStateCounts != null) { + TaskCountDto taskCountResult = new TaskCountDto(processInstanceStateCounts); + result.put(Constants.DATA_LIST, taskCountResult); + putMsg(result, Status.SUCCESS); + } + return result; + } + + + /** + * statistics the process definition quantities of certain person + * + * @param loginUser login user + * @param projectId project id + * @return definition count data + */ + public Map countDefinitionByUser(User loginUser, int projectId) { + Map result = new HashMap<>(); + + + Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); + List defineGroupByUsers = processDefinitionMapper.countDefinitionGroupByUser( + loginUser.getId(), projectIdArray, isAdmin(loginUser)); + + DefineUserDto dto = new DefineUserDto(defineGroupByUsers); + result.put(Constants.DATA_LIST, dto); + putMsg(result, Status.SUCCESS); + return result; + } + + + /** + * statistical command status data + * + * @param loginUser login user + * @param projectId project id + * @param startDate start date + * @param endDate end date + * @return command state count data + */ + public Map countCommandState(User loginUser, int projectId, String startDate, String endDate) { + + Map result = new HashMap<>(5); + boolean checkProject = checkProject(loginUser, projectId, result); + if (!checkProject) { + return result; + } + + /** + * find all the task lists in the project under the user + * statistics based on task status execution, failure, completion, wait, total + */ + Date start = null; + Date end = null; + + if (startDate != null && endDate != null) { + try { + start = DateUtils.getScheduleDate(startDate); + end = DateUtils.getScheduleDate(endDate); + } catch (Exception e) { + logger.error(e.getMessage(), e); + putErrorRequestParamsMsg(result); + return result; + } + } + + + Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); + // count command state + List commandStateCounts = + commandMapper.countCommandState( + loginUser.getId(), + start, + end, + projectIdArray); + + // count error command state + List errorCommandStateCounts = + errorCommandMapper.countCommandState( + start, end, projectIdArray); + + // enumMap + Map> dataMap = new EnumMap<>(CommandType.class); + + Map commonCommand = new HashMap<>(); + commonCommand.put(COMMAND_STATE, 0); + commonCommand.put(ERROR_COMMAND_STATE, 0); + + + // init data map + /** + * START_PROCESS, START_CURRENT_TASK_PROCESS, RECOVER_TOLERANCE_FAULT_PROCESS, RECOVER_SUSPENDED_PROCESS, + START_FAILURE_TASK_PROCESS,COMPLEMENT_DATA,SCHEDULER, REPEAT_RUNNING,PAUSE,STOP,RECOVER_WAITTING_THREAD; + */ + dataMap.put(CommandType.START_PROCESS, commonCommand); + dataMap.put(CommandType.START_CURRENT_TASK_PROCESS, commonCommand); + dataMap.put(CommandType.RECOVER_TOLERANCE_FAULT_PROCESS, commonCommand); + dataMap.put(CommandType.RECOVER_SUSPENDED_PROCESS, commonCommand); + dataMap.put(CommandType.START_FAILURE_TASK_PROCESS, commonCommand); + dataMap.put(CommandType.COMPLEMENT_DATA, commonCommand); + dataMap.put(CommandType.SCHEDULER, commonCommand); + dataMap.put(CommandType.REPEAT_RUNNING, commonCommand); + dataMap.put(CommandType.PAUSE, commonCommand); + dataMap.put(CommandType.STOP, commonCommand); + dataMap.put(CommandType.RECOVER_WAITTING_THREAD, commonCommand); + + // put command state + for (CommandCount executeStatusCount : commandStateCounts) { + Map commandStateCountsMap = new HashMap<>(dataMap.get(executeStatusCount.getCommandType())); + commandStateCountsMap.put(COMMAND_STATE, executeStatusCount.getCount()); + dataMap.put(executeStatusCount.getCommandType(), commandStateCountsMap); + } + + // put error command state + for (CommandCount errorExecutionStatus : errorCommandStateCounts) { + Map errorCommandStateCountsMap = new HashMap<>(dataMap.get(errorExecutionStatus.getCommandType())); + errorCommandStateCountsMap.put(ERROR_COMMAND_STATE, errorExecutionStatus.getCount()); + dataMap.put(errorExecutionStatus.getCommandType(), errorCommandStateCountsMap); + } + + List list = new ArrayList<>(); + for (Map.Entry> next : dataMap.entrySet()) { + CommandStateCount commandStateCount = new CommandStateCount(next.getValue().get(ERROR_COMMAND_STATE), + next.getValue().get(COMMAND_STATE), next.getKey()); + list.add(commandStateCount); + } + + result.put(Constants.DATA_LIST, list); + putMsg(result, Status.SUCCESS); + return result; + } + + private Integer[] getProjectIdsArrays(User loginUser, int projectId) { + List projectIds = new ArrayList<>(); + if (projectId != 0) { + projectIds.add(projectId); + } else if (loginUser.getUserType() == UserType.GENERAL_USER) { + projectIds = processService.getProjectIdListHavePerm(loginUser.getId()); + if (projectIds.isEmpty()) { + projectIds.add(0); + } + } + return projectIds.toArray(new Integer[0]); + } + + /** + * count queue state + * + * @param loginUser login user + * @param projectId project id + * @return queue state count data + */ + public Map countQueueState(User loginUser, int projectId) { + Map result = new HashMap<>(5); + + boolean checkProject = checkProject(loginUser, projectId, result); + if (!checkProject) { + return result; + } + + // TODO tasksQueueList and tasksKillList is never updated. + List tasksQueueList = new ArrayList<>(); + List tasksKillList = new ArrayList<>(); + + Map dataMap = new HashMap<>(); + if (loginUser.getUserType() == UserType.ADMIN_USER) { + dataMap.put("taskQueue", tasksQueueList.size()); + dataMap.put("taskKill", tasksKillList.size()); + + result.put(Constants.DATA_LIST, dataMap); + putMsg(result, Status.SUCCESS); + return result; + } + + int[] tasksQueueIds = new int[tasksQueueList.size()]; + int[] tasksKillIds = new int[tasksKillList.size()]; + + int i = 0; + for (String taskQueueStr : tasksQueueList) { + if (StringUtils.isNotEmpty(taskQueueStr)) { + String[] splits = taskQueueStr.split("_"); + if (splits.length >= 4) { + tasksQueueIds[i++] = Integer.parseInt(splits[3]); + } + } + } + + i = 0; + for (String taskKillStr : tasksKillList) { + if (StringUtils.isNotEmpty(taskKillStr)) { + String[] splits = taskKillStr.split("-"); + if (splits.length == 2) { + tasksKillIds[i++] = Integer.parseInt(splits[1]); + } + } + } + Integer taskQueueCount = 0; + Integer taskKillCount = 0; + + Integer[] projectIds = getProjectIdsArrays(loginUser, projectId); + if (tasksQueueIds.length != 0) { + taskQueueCount = taskInstanceMapper.countTask( + projectIds, + tasksQueueIds); + } + + if (tasksKillIds.length != 0) { + taskKillCount = taskInstanceMapper.countTask(projectIds, tasksKillIds); + } + + dataMap.put("taskQueue", taskQueueCount); + dataMap.put("taskKill", taskKillCount); + + result.put(Constants.DATA_LIST, dataMap); + putMsg(result, Status.SUCCESS); + return result; + } + + private boolean checkProject(User loginUser, int projectId, Map result) { + if (projectId != 0) { + Project project = projectMapper.selectById(projectId); + return projectService.hasProjectAndPerm(loginUser, project, result); + } + return true; + } + + private void putErrorRequestParamsMsg(Map result) { + result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); + result.put(Constants.MSG, MessageFormat.format(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getMsg(), "startDate,endDate")); + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java index 6a9e78600b..ee127d6f6f 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java @@ -17,17 +17,28 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.DataAnalysisServiceImpl; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.CommandCount; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -36,19 +47,13 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; @RunWith(PowerMockRunner.class) public class DataAnalysisServiceTest { - + @InjectMocks - private DataAnalysisService dataAnalysisService; + private DataAnalysisServiceImpl dataAnalysisService; @Mock ProjectMapper projectMapper; @@ -71,13 +76,9 @@ public class DataAnalysisServiceTest { @Mock TaskInstanceMapper taskInstanceMapper; - - @Mock ProcessService processService; - private Project project; - private Map resultMap; private User user; @@ -86,26 +87,25 @@ public class DataAnalysisServiceTest { public void setUp() { user = new User(); - project = new Project(); + Project project = new Project(); project.setId(1); resultMap = new HashMap<>(); Mockito.when(projectMapper.selectById(1)).thenReturn(project); - Mockito.when(projectService.hasProjectAndPerm(user,project,resultMap)).thenReturn(true); + Mockito.when(projectService.hasProjectAndPerm(user, project, resultMap)).thenReturn(true); } @After - public void after(){ + public void after() { user = null; projectMapper = null; resultMap = null; } - @Test - public void testCountTaskStateByProject(){ + public void testCountTaskStateByProject() { String startDate = "2020-02-11 16:02:18"; String endDate = "2020-02-11 16:03:18"; @@ -120,42 +120,40 @@ public class DataAnalysisServiceTest { DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(getTaskInstanceStateCounts()); result = dataAnalysisService.countTaskStateByProject(user, 1, startDate, endDate); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } - @Test - public void testCountProcessInstanceStateByProject(){ + public void testCountProcessInstanceStateByProject() { String startDate = "2020-02-11 16:02:18"; String endDate = "2020-02-11 16:03:18"; //checkProject false - Map result = dataAnalysisService.countProcessInstanceStateByProject(user,2,startDate,endDate); + Map result = dataAnalysisService.countProcessInstanceStateByProject(user, 2, startDate, endDate); Assert.assertTrue(result.isEmpty()); //SUCCESS Mockito.when(processInstanceMapper.countInstanceStateByUser(DateUtils.getScheduleDate(startDate), DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(getTaskInstanceStateCounts()); - result = dataAnalysisService.countProcessInstanceStateByProject(user,1,startDate,endDate); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + result = dataAnalysisService.countProcessInstanceStateByProject(user, 1, startDate, endDate); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } @Test - public void testCountDefinitionByUser(){ + public void testCountDefinitionByUser() { - Map result = dataAnalysisService.countDefinitionByUser(user,1); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Map result = dataAnalysisService.countDefinitionByUser(user, 1); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } - @Test - public void testCountCommandState(){ + public void testCountCommandState() { String startDate = "2020-02-11 16:02:18"; String endDate = "2020-02-11 16:03:18"; //checkProject false - Map result = dataAnalysisService.countCommandState(user,2,startDate,endDate); + Map result = dataAnalysisService.countCommandState(user, 2, startDate, endDate); Assert.assertTrue(result.isEmpty()); List commandCounts = new ArrayList<>(1); CommandCount commandCount = new CommandCount(); @@ -164,26 +162,25 @@ public class DataAnalysisServiceTest { Mockito.when(commandMapper.countCommandState(0, DateUtils.getScheduleDate(startDate), DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(commandCounts); - Mockito.when(errorCommandMapper.countCommandState( DateUtils.getScheduleDate(startDate), + Mockito.when(errorCommandMapper.countCommandState(DateUtils.getScheduleDate(startDate), DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(commandCounts); - result = dataAnalysisService.countCommandState(user,1,startDate,endDate); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + result = dataAnalysisService.countCommandState(user, 1, startDate, endDate); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } /** - * get list - * @return + * get list */ - private List getTaskInstanceStateCounts(){ + private List getTaskInstanceStateCounts() { List taskInstanceStateCounts = new ArrayList<>(1); ExecuteStatusCount executeStatusCount = new ExecuteStatusCount(); executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); taskInstanceStateCounts.add(executeStatusCount); - return taskInstanceStateCounts; + return taskInstanceStateCounts; } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TriFunction.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TriFunction.java new file mode 100644 index 0000000000..fe873b3475 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TriFunction.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +/** + * tri function function interface + */ +@FunctionalInterface +public interface TriFunction { + + OUT1 apply(IN1 in1, IN2 in2, IN3 in3); + +} From 8bec5eb63932d4128029cdf1d1aea66e5d9c8bd2 Mon Sep 17 00:00:00 2001 From: BoYiZhang <39816903+BoYiZhang@users.noreply.github.com> Date: Mon, 10 Aug 2020 14:50:08 +0800 Subject: [PATCH 09/24] [Feature][Queue]`Queue manage ' is renamed' yarn queue manage ' (#3430) * fix bug Delete invalid field: executorcores Modify verification prompt * fix bug Delete invalid field: executorcores Modify verification prompt * fix bug Delete invalid field: executorcores Modify verification prompt * dag add close button * reset last version * reset last version * dag add close buttion dag add close buttion * update CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * Update CreateWorkflowLocator.java modify submit workflow button * Update CreateWorkflowLocator.java * Update CreateWorkflowLocator.java modify CLICK_ADD_BUTTON * Update CreateWorkflowLocator.java delete print * Update CreateWorkflowLocator.java 1 * Update CreateWorkflowLocator.java 1 * Setting '-XX:+DisableExplicitGC ' causes netty memory leaks in addition update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M' * Update dag.vue * Update dag.vue * Update dag.vue * Update CreateWorkflowLocator.java * Revert "Setting '-XX:+DisableExplicitGC ' causes netty memory leaks" This reverts commit 3a2cba7a * Setting '-XX:+DisableExplicitGC ' causes netty memory leaks in addition update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M' * Update dolphinscheduler-daemon.sh * `Queue manage` is renamed ` yarn queue manage ` Co-authored-by: dailidong Co-authored-by: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> --- dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js | 4 ++-- dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index f5e579c088..0ef5340488 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -176,7 +176,7 @@ export default { 'Edit Tenant': 'Edit Tenant', 'Tenant Code': 'Tenant Code', 'Tenant Name': 'Tenant Name', - Queue: 'Queue', + Queue: 'Yarn Queue', 'Please select a queue': 'default is tenant association queue', 'Please enter the tenant code in English': 'Please enter the tenant code in English', 'Please enter tenant code in English': 'Please enter tenant code in English', @@ -455,7 +455,7 @@ export default { LastMonthBegin: 'LastMonthBegin', LastMonthEnd: 'LastMonthEnd', 'Refresh status succeeded': 'Refresh status succeeded', - 'Queue manage': 'Queue manage', + 'Queue manage': 'Yarn Queue manage', 'Create queue': 'Create queue', 'Edit queue': 'Edit queue', 'Datasource manage': 'Datasource', diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 8d524144e1..a352183fca 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -180,7 +180,7 @@ export default { 'Edit Tenant': '编辑租户', 'Tenant Code': '租户编码', 'Tenant Name': '租户名称', - Queue: '队列', + Queue: 'Yarn 队列', 'Please enter the tenant code in English': '请输入租户编码只允许英文', 'Please enter tenant code in English': '请输入英文租户编码', 'Edit User': '编辑用户', @@ -450,7 +450,7 @@ export default { LastMonthBegin: '上月初', LastMonthEnd: '上月末', 'Refresh status succeeded': '刷新状态成功', - 'Queue manage': '队列管理', + 'Queue manage': 'Yarn 队列管理', 'Create queue': '创建队列', 'Edit queue': '编辑队列', 'Datasource manage': '数据源中心', From f419fe6239dd619bcf8ebf03d6247d12e41748c0 Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Mon, 10 Aug 2020 15:01:20 +0800 Subject: [PATCH 10/24] [Test][server] Add LoggerServerTest (#3405) --- .../server/log/LoggerServerTest.java | 62 ++++++++++++------- pom.xml | 2 +- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java index 0da88746f5..d3b1dcf84a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java @@ -17,46 +17,62 @@ package org.apache.dolphinscheduler.server.log; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; + import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.service.log.LogClientService; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; public class LoggerServerTest { + private LoggerServer loggerServer; - @Test - public void testRollViewLog(){ - LoggerServer loggerServer = new LoggerServer(); - loggerServer.start(); + private LogClientService logClientService; - LogClientService logClientService = new LogClientService(); - logClientService.rollViewLog("localhost", Constants.RPC_PORT,"/opt/demo.txt",0,1000); + @Before + public void startServerAndClient() { + this.loggerServer = new LoggerServer(); + this.loggerServer.start(); + this.logClientService = new LogClientService(); + } - try { - Thread.sleep(5000); - } catch (InterruptedException e) { + @Test + public void testRollViewLog() throws IOException { + String expectedTmpDemoString = "testRolloViewLog"; + FileUtils.writeStringToFile(new File("/tmp/demo.txt"), expectedTmpDemoString, Charset.defaultCharset()); - } + String resultTmpDemoString = this.logClientService.rollViewLog( + "localhost", Constants.RPC_PORT,"/tmp/demo.txt", 0, 1000); - loggerServer.stop(); - logClientService.close(); + Assert.assertEquals(expectedTmpDemoString, resultTmpDemoString.replaceAll("[\r|\n|\t]", StringUtils.EMPTY)); + + FileUtils.deleteFile("/tmp/demo.txt"); } @Test - public void testRemoveTaskLog(){ - LoggerServer loggerServer = new LoggerServer(); - loggerServer.start(); + public void testRemoveTaskLog() throws IOException { + String expectedTmpRemoveString = "testRemoveTaskLog"; + FileUtils.writeStringToFile(new File("/tmp/remove.txt"), expectedTmpRemoveString, Charset.defaultCharset()); + + Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/tmp/remove.txt"); - LogClientService logClientService = new LogClientService(); - logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/opt/zhangsan"); + Assert.assertTrue(b); - try { - Thread.sleep(5000); - } catch (InterruptedException e) { + String result = this.logClientService.viewLog("localhost", Constants.RPC_PORT,"/tmp/demo.txt"); - } + Assert.assertEquals(StringUtils.EMPTY, result); + } - loggerServer.stop(); - logClientService.close(); + @After + public void stopServerAndClient() { + this.loggerServer.stop(); + this.logClientService.close(); } } diff --git a/pom.xml b/pom.xml index 0b546b3ec1..173265a396 100644 --- a/pom.xml +++ b/pom.xml @@ -795,7 +795,7 @@ **/remote/RemoveTaskLogRequestCommandTest.java **/remote/ResponseFutureTest.java - + **/server/log/LoggerServerTest.java **/server/entity/SQLTaskExecutionContextTest.java **/server/log/MasterLogFilterTest.java **/server/log/SensitiveDataConverterTest.java From 379203b306923dde71fe87f8a6f193d44f8c2284 Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Mon, 10 Aug 2020 15:02:14 +0800 Subject: [PATCH 11/24] [Improvement-3369][api] Introduce session service interface for clear code (#3408) * [Improvement][api] Introduce session service interface for clear code * Update SessionService.java * [Improvement][api] Add @Service * Add licenses --- .../interceptor/LoginHandlerInterceptor.java | 14 +- .../api/service/BaseService.java | 17 +- .../api/service/SessionService.java | 110 +----------- .../api/service/impl/SessionServiceImpl.java | 158 ++++++++++++++++++ .../api/service/SessionServiceTest.java | 11 +- 5 files changed, 184 insertions(+), 126 deletions(-) create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java index cb7a8e653f..83eb4fefce 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java @@ -16,32 +16,28 @@ */ package org.apache.dolphinscheduler.api.interceptor; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.security.Authenticator; -import org.apache.dolphinscheduler.api.service.SessionService; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.servlet.HandlerInterceptor; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - /** * login interceptor, must login first */ public class LoginHandlerInterceptor implements HandlerInterceptor { private static final Logger logger = LoggerFactory.getLogger(LoginHandlerInterceptor.class); - @Autowired - private SessionService sessionService; - @Autowired private UserMapper userMapper; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java index 646a67ab04..4b094ea494 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java @@ -16,6 +16,12 @@ */ package org.apache.dolphinscheduler.api.service; +import java.text.MessageFormat; +import java.util.Map; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; @@ -24,11 +30,6 @@ import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.User; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import java.text.MessageFormat; -import java.util.Map; - /** * base service */ @@ -96,6 +97,7 @@ public class BaseService { /** * get cookie info by name + * * @param request request * @param name 'sessionId' * @return get cookie info @@ -115,10 +117,11 @@ public class BaseService { /** * create tenant dir if not exists + * * @param tenantCode tenant code * @throws Exception if hdfs operation exception */ - protected void createTenantDirIfNotExists(String tenantCode)throws Exception{ + protected void createTenantDirIfNotExists(String tenantCode) throws Exception { String resourcePath = HadoopUtils.getHdfsResDir(tenantCode); String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); @@ -129,7 +132,7 @@ public class BaseService { HadoopUtils.getInstance().mkdir(udfsPath); } - protected boolean hasPerm(User operateUser, int createUserId){ + protected boolean hasPerm(User operateUser, int createUserId) { return operateUser.getId() == createUserId || isAdmin(operateUser); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java index b4aab962ef..dc911f51e3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java @@ -16,36 +16,15 @@ */ package org.apache.dolphinscheduler.api.service; +import javax.servlet.http.HttpServletRequest; -import org.apache.dolphinscheduler.api.controller.BaseController; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.dao.entity.Session; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.SessionMapper; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import java.util.Date; -import java.util.List; -import java.util.UUID; /** * session service */ -@Service -public class SessionService extends BaseService{ - - private static final Logger logger = LoggerFactory.getLogger(SessionService.class); - - @Autowired - private SessionMapper sessionMapper; +public interface SessionService { /** * get user session from request @@ -53,26 +32,7 @@ public class SessionService extends BaseService{ * @param request request * @return session */ - public Session getSession(HttpServletRequest request) { - String sessionId = request.getHeader(Constants.SESSION_ID); - - if(StringUtils.isBlank(sessionId)) { - Cookie cookie = getCookie(request, Constants.SESSION_ID); - - if (cookie != null) { - sessionId = cookie.getValue(); - } - } - - if(StringUtils.isBlank(sessionId)) { - return null; - } - - String ip = BaseController.getClientIpAddress(request); - logger.debug("get session: {}, ip: {}", sessionId, ip); - - return sessionMapper.selectById(sessionId); - } + Session getSession(HttpServletRequest request); /** * create session @@ -81,55 +41,7 @@ public class SessionService extends BaseService{ * @param ip ip * @return session string */ - @Transactional(rollbackFor = RuntimeException.class) - public String createSession(User user, String ip) { - Session session = null; - - // logined - List sessionList = sessionMapper.queryByUserId(user.getId()); - - Date now = new Date(); - - /** - * if you have logged in and are still valid, return directly - */ - if (CollectionUtils.isNotEmpty(sessionList)) { - // is session list greater 1 ， delete other ，get one - if (sessionList.size() > 1){ - for (int i=1 ; i < sessionList.size();i++){ - sessionMapper.deleteById(sessionList.get(i).getId()); - } - } - session = sessionList.get(0); - if (now.getTime() - session.getLastLoginTime().getTime() <= Constants.SESSION_TIME_OUT * 1000) { - /** - * updateProcessInstance the latest login time - */ - session.setLastLoginTime(now); - sessionMapper.updateById(session); - - return session.getId(); - - } else { - /** - * session expired, then delete this session first - */ - sessionMapper.deleteById(session.getId()); - } - } - - // assign new session - session = new Session(); - - session.setId(UUID.randomUUID().toString()); - session.setIp(ip); - session.setUserId(user.getId()); - session.setLastLoginTime(now); - - sessionMapper.insert(session); - - return session.getId(); - } + String createSession(User user, String ip); /** * sign out @@ -138,17 +50,5 @@ public class SessionService extends BaseService{ * @param ip no use * @param loginUser login user */ - public void signOut(String ip, User loginUser) { - try { - /** - * query session by user id and ip - */ - Session session = sessionMapper.queryByUserIdAndIp(loginUser.getId(),ip); - - //delete session - sessionMapper.deleteById(session.getId()); - }catch (Exception e){ - logger.warn("userId : {} , ip : {} , find more one session",loginUser.getId(),ip); - } - } + void signOut(String ip, User loginUser); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java new file mode 100644 index 0000000000..8aaefdadff --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service.impl; + +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; + +import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.api.controller.BaseController; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.service.SessionService; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.Session; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.SessionMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * session service implement + */ +@Service +public class SessionServiceImpl extends BaseService implements SessionService { + + private static final Logger logger = LoggerFactory.getLogger(SessionService.class); + + @Autowired + private SessionMapper sessionMapper; + + /** + * get user session from request + * + * @param request request + * @return session + */ + public Session getSession(HttpServletRequest request) { + String sessionId = request.getHeader(Constants.SESSION_ID); + + if (StringUtils.isBlank(sessionId)) { + Cookie cookie = getCookie(request, Constants.SESSION_ID); + + if (cookie != null) { + sessionId = cookie.getValue(); + } + } + + if (StringUtils.isBlank(sessionId)) { + return null; + } + + String ip = BaseController.getClientIpAddress(request); + logger.debug("get session: {}, ip: {}", sessionId, ip); + + return sessionMapper.selectById(sessionId); + } + + /** + * create session + * + * @param user user + * @param ip ip + * @return session string + */ + @Transactional(rollbackFor = RuntimeException.class) + public String createSession(User user, String ip) { + Session session = null; + + // logined + List sessionList = sessionMapper.queryByUserId(user.getId()); + + Date now = new Date(); + + /** + * if you have logged in and are still valid, return directly + */ + if (CollectionUtils.isNotEmpty(sessionList)) { + // is session list greater 1 ， delete other ，get one + if (sessionList.size() > 1) { + for (int i = 1; i < sessionList.size(); i++) { + sessionMapper.deleteById(sessionList.get(i).getId()); + } + } + session = sessionList.get(0); + if (now.getTime() - session.getLastLoginTime().getTime() <= Constants.SESSION_TIME_OUT * 1000) { + /** + * updateProcessInstance the latest login time + */ + session.setLastLoginTime(now); + sessionMapper.updateById(session); + + return session.getId(); + + } else { + /** + * session expired, then delete this session first + */ + sessionMapper.deleteById(session.getId()); + } + } + + // assign new session + session = new Session(); + + session.setId(UUID.randomUUID().toString()); + session.setIp(ip); + session.setUserId(user.getId()); + session.setLastLoginTime(now); + + sessionMapper.insert(session); + + return session.getId(); + } + + /** + * sign out + * remove ip restrictions + * + * @param ip no use + * @param loginUser login user + */ + public void signOut(String ip, User loginUser) { + try { + /** + * query session by user id and ip + */ + Session session = sessionMapper.queryByUserIdAndIp(loginUser.getId(), ip); + + //delete session + sessionMapper.deleteById(session.getId()); + } catch (Exception e) { + logger.warn("userId : {} , ip : {} , find more one session", loginUser.getId(), ip); + } + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java index 7e98721207..b51f85f456 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java @@ -16,7 +16,12 @@ */ package org.apache.dolphinscheduler.api.service; +import java.util.ArrayList; import java.util.Calendar; +import java.util.Date; +import java.util.List; + +import org.apache.dolphinscheduler.api.service.impl.SessionServiceImpl; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -38,10 +43,6 @@ import org.slf4j.LoggerFactory; import org.springframework.mock.web.MockCookie; import org.springframework.mock.web.MockHttpServletRequest; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - @RunWith(MockitoJUnitRunner.class) public class SessionServiceTest { @@ -49,7 +50,7 @@ public class SessionServiceTest { private static final Logger logger = LoggerFactory.getLogger(SessionServiceTest.class); @InjectMocks - private SessionService sessionService; + private SessionServiceImpl sessionService; @Mock private SessionMapper sessionMapper; From 133b4de129a407402d1d663b4d00d4f6a776dd28 Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Mon, 10 Aug 2020 15:02:55 +0800 Subject: [PATCH 12/24] [Improvement-3369][api] Introduce api service interface for ProcessDefinitionService (#3371) * [Improvement-3369][api] Code clean and improvement * Feature: import DATA_LIST static variable --- .../api/service/BaseDAGService.java | 54 - .../api/service/ProcessDefinitionService.java | 1342 +-------------- .../api/service/ProcessInstanceService.java | 87 +- .../impl/ProcessDefinitionServiceImpl.java | 1493 +++++++++++++++++ .../ProcessDefinitionControllerTest.java | 7 +- .../api/service/BaseDAGServiceTest.java | 50 - .../service/ProcessDefinitionServiceTest.java | 230 +-- 7 files changed, 1656 insertions(+), 1607 deletions(-) delete mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java delete mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java deleted file mode 100644 index edc115b3d4..0000000000 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.service; - -import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.model.TaskNodeRelation; -import org.apache.dolphinscheduler.common.process.ProcessDag; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.entity.ProcessData; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.apache.dolphinscheduler.dao.utils.DagHelper; - -import java.util.List; - -/** - * base DAG service - */ -public class BaseDAGService extends BaseService{ - - - /** - * process instance to DAG - * - * @param processInstance input process instance - * @return process instance dag. - */ - public static DAG processInstance2DAG(ProcessInstance processInstance) { - - String processDefinitionJson = processInstance.getProcessInstanceJson(); - - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - - List taskNodeList = processData.getTasks(); - - ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); - - return DagHelper.buildDagGraph(processDag); - } -} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index b3d56e5982..6a4eb974ce 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -16,90 +16,20 @@ */ package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.dolphinscheduler.api.dto.ProcessMeta; -import org.apache.dolphinscheduler.api.dto.treeview.Instance; -import org.apache.dolphinscheduler.api.dto.treeview.TreeViewDto; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.utils.CheckUtils; -import org.apache.dolphinscheduler.api.utils.FileUtils; -import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.api.utils.exportprocess.ProcessAddTaskParam; -import org.apache.dolphinscheduler.api.utils.exportprocess.TaskNodeParamFactory; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; -import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.model.TaskNodeRelation; -import org.apache.dolphinscheduler.common.process.ProcessDag; -import org.apache.dolphinscheduler.common.process.Property; -import org.apache.dolphinscheduler.common.task.AbstractParameters; -import org.apache.dolphinscheduler.common.thread.Stopper; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; -import org.apache.dolphinscheduler.dao.utils.DagHelper; -import org.apache.dolphinscheduler.service.permission.PermissionCheck; -import org.apache.dolphinscheduler.service.process.ProcessService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.web.multipart.MultipartFile; +import java.util.Map; -import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS_DEFINE_ID; +import org.apache.dolphinscheduler.dao.entity.ProcessData; +import org.apache.dolphinscheduler.dao.entity.User; +import org.springframework.web.multipart.MultipartFile; + +import com.fasterxml.jackson.core.JsonProcessingException; /** * process definition service */ -@Service -public class ProcessDefinitionService extends BaseDAGService { - - private static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionService.class); - - private static final String PROCESSDEFINITIONID = "processDefinitionId"; - - private static final String RELEASESTATE = "releaseState"; - - private static final String TASKS = "tasks"; - - @Autowired - private ProjectMapper projectMapper; - - @Autowired - private ProjectService projectService; - - @Autowired - private ProcessDefinitionMapper processDefineMapper; - - @Autowired - private ProcessInstanceMapper processInstanceMapper; - - - @Autowired - private TaskInstanceMapper taskInstanceMapper; - - @Autowired - private ScheduleMapper scheduleMapper; - - @Autowired - private ProcessService processService; +public interface ProcessDefinitionService { /** * create process definition @@ -114,92 +44,13 @@ public class ProcessDefinitionService extends BaseDAGService { * @return create result code * @throws JsonProcessingException JsonProcessingException */ - public Map createProcessDefinition(User loginUser, - String projectName, - String name, - String processDefinitionJson, - String desc, - String locations, - String connects) throws JsonProcessingException { - - Map result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - // check project auth - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - ProcessDefinition processDefine = new ProcessDefinition(); - Date now = new Date(); - - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - Map checkProcessJson = checkProcessNodeList(processData, processDefinitionJson); - if (checkProcessJson.get(Constants.STATUS) != Status.SUCCESS) { - return checkProcessJson; - } - - processDefine.setName(name); - processDefine.setReleaseState(ReleaseState.OFFLINE); - processDefine.setProjectId(project.getId()); - processDefine.setUserId(loginUser.getId()); - processDefine.setProcessDefinitionJson(processDefinitionJson); - processDefine.setDescription(desc); - processDefine.setLocations(locations); - processDefine.setConnects(connects); - processDefine.setTimeout(processData.getTimeout()); - processDefine.setTenantId(processData.getTenantId()); - processDefine.setModifyBy(loginUser.getUserName()); - processDefine.setResourceIds(getResourceIds(processData)); - - //custom global params - List globalParamsList = processData.getGlobalParams(); - if (CollectionUtils.isNotEmpty(globalParamsList)) { - Set globalParamsSet = new HashSet<>(globalParamsList); - globalParamsList = new ArrayList<>(globalParamsSet); - processDefine.setGlobalParamList(globalParamsList); - } - processDefine.setCreateTime(now); - processDefine.setUpdateTime(now); - processDefine.setFlag(Flag.YES); - processDefineMapper.insert(processDefine); - - // return processDefinition object with ID - result.put(Constants.DATA_LIST, processDefineMapper.selectById(processDefine.getId())); - putMsg(result, Status.SUCCESS); - result.put("processDefinitionId", processDefine.getId()); - return result; - } - - /** - * get resource ids - * - * @param processData process data - * @return resource ids - */ - private String getResourceIds(ProcessData processData) { - List tasks = processData.getTasks(); - Set resourceIds = new HashSet<>(); - for (TaskNode taskNode : tasks) { - String taskParameter = taskNode.getParams(); - AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(), taskParameter); - if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) { - Set tempSet = params.getResourceFilesList().stream().map(t -> t.getId()).collect(Collectors.toSet()); - resourceIds.addAll(tempSet); - } - } - - StringBuilder sb = new StringBuilder(); - for (int i : resourceIds) { - if (sb.length() > 0) { - sb.append(","); - } - sb.append(i); - } - return sb.toString(); - } - + Map createProcessDefinition(User loginUser, + String projectName, + String name, + String processDefinitionJson, + String desc, + String locations, + String connects) throws JsonProcessingException; /** * query process definition list @@ -208,24 +59,8 @@ public class ProcessDefinitionService extends BaseDAGService { * @param projectName project name * @return definition list */ - public Map queryProcessDefinitionList(User loginUser, String projectName) { - - HashMap result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - List resourceList = processDefineMapper.queryAllDefinitionList(project.getId()); - result.put(Constants.DATA_LIST, resourceList); - putMsg(result, Status.SUCCESS); - - return result; - } - + Map queryProcessDefinitionList(User loginUser, + String projectName); /** * query process definition list paging @@ -238,29 +73,12 @@ public class ProcessDefinitionService extends BaseDAGService { * @param userId user id * @return process definition page */ - public Map queryProcessDefinitionListPaging(User loginUser, String projectName, String searchVal, Integer pageNo, Integer pageSize, Integer userId) { - - Map result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - Page page = new Page(pageNo, pageSize); - IPage processDefinitionIPage = processDefineMapper.queryDefineListPaging( - page, searchVal, userId, project.getId(), isAdmin(loginUser)); - - PageInfo pageInfo = new PageInfo(pageNo, pageSize); - pageInfo.setTotalCount((int) processDefinitionIPage.getTotal()); - pageInfo.setLists(processDefinitionIPage.getRecords()); - result.put(Constants.DATA_LIST, pageInfo); - putMsg(result, Status.SUCCESS); - - return result; - } + Map queryProcessDefinitionListPaging(User loginUser, + String projectName, + String searchVal, + Integer pageNo, + Integer pageSize, + Integer userId); /** * query datail of process definition @@ -270,27 +88,9 @@ public class ProcessDefinitionService extends BaseDAGService { * @param processId process definition id * @return process definition detail */ - public Map queryProcessDefinitionById(User loginUser, String projectName, Integer processId) { - - - Map result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - ProcessDefinition processDefinition = processDefineMapper.selectById(processId); - if (processDefinition == null) { - putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processId); - } else { - result.put(Constants.DATA_LIST, processDefinition); - putMsg(result, Status.SUCCESS); - } - return result; - } + Map queryProcessDefinitionById(User loginUser, + String projectName, + Integer processId); /** * copy process definition @@ -300,32 +100,9 @@ public class ProcessDefinitionService extends BaseDAGService { * @param processId process definition id * @return copy result code */ - public Map copyProcessDefinition(User loginUser, String projectName, Integer processId) throws JsonProcessingException { - - Map result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - ProcessDefinition processDefinition = processDefineMapper.selectById(processId); - if (processDefinition == null) { - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId); - return result; - } else { - return createProcessDefinition( - loginUser, - projectName, - processDefinition.getName() + "_copy_" + System.currentTimeMillis(), - processDefinition.getProcessDefinitionJson(), - processDefinition.getDescription(), - processDefinition.getLocations(), - processDefinition.getConnects()); - } - } + Map copyProcessDefinition(User loginUser, + String projectName, + Integer processId) throws JsonProcessingException; /** * update process definition @@ -340,68 +117,12 @@ public class ProcessDefinitionService extends BaseDAGService { * @param connects connects for nodes * @return update result code */ - public Map updateProcessDefinition(User loginUser, String projectName, int id, String name, - String processDefinitionJson, String desc, - String locations, String connects) { - Map result = new HashMap<>(5); - - Project project = projectMapper.queryByName(projectName); - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - Map checkProcessJson = checkProcessNodeList(processData, processDefinitionJson); - if ((checkProcessJson.get(Constants.STATUS) != Status.SUCCESS)) { - return checkProcessJson; - } - ProcessDefinition processDefine = processService.findProcessDefineById(id); - if (processDefine == null) { - // check process definition exists - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, id); - return result; - } else if (processDefine.getReleaseState() == ReleaseState.ONLINE) { - // online can not permit edit - putMsg(result, Status.PROCESS_DEFINE_NOT_ALLOWED_EDIT, processDefine.getName()); - return result; - } else { - putMsg(result, Status.SUCCESS); - } - - Date now = new Date(); - - processDefine.setId(id); - processDefine.setName(name); - processDefine.setReleaseState(ReleaseState.OFFLINE); - processDefine.setProjectId(project.getId()); - processDefine.setProcessDefinitionJson(processDefinitionJson); - processDefine.setDescription(desc); - processDefine.setLocations(locations); - processDefine.setConnects(connects); - processDefine.setTimeout(processData.getTimeout()); - processDefine.setTenantId(processData.getTenantId()); - processDefine.setModifyBy(loginUser.getUserName()); - processDefine.setResourceIds(getResourceIds(processData)); - - //custom global params - List globalParamsList = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(processData.getGlobalParams())) { - Set userDefParamsSet = new HashSet<>(processData.getGlobalParams()); - globalParamsList = new ArrayList<>(userDefParamsSet); - } - processDefine.setGlobalParamList(globalParamsList); - processDefine.setUpdateTime(now); - processDefine.setFlag(Flag.YES); - if (processDefineMapper.updateById(processDefine) > 0) { - putMsg(result, Status.SUCCESS); - - } else { - putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR); - } - return result; - } + Map updateProcessDefinition(User loginUser, + String projectName, + int id, + String name, + String processDefinitionJson, String desc, + String locations, String connects); /** * verify process definition name unique @@ -411,24 +132,9 @@ public class ProcessDefinitionService extends BaseDAGService { * @param name name * @return true if process definition name not exists, otherwise false */ - public Map verifyProcessDefinitionName(User loginUser, String projectName, String name) { - - Map result = new HashMap<>(); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultEnum = (Status) checkResult.get(Constants.STATUS); - if (resultEnum != Status.SUCCESS) { - return checkResult; - } - ProcessDefinition processDefinition = processDefineMapper.queryByDefineName(project.getId(), name); - if (processDefinition == null) { - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.PROCESS_INSTANCE_EXIST, name); - } - return result; - } + Map verifyProcessDefinitionName(User loginUser, + String projectName, + String name); /** * delete process definition by id @@ -438,62 +144,9 @@ public class ProcessDefinitionService extends BaseDAGService { * @param processDefinitionId process definition id * @return delete result code */ - @Transactional(rollbackFor = RuntimeException.class) - public Map deleteProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId) { - - Map result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultEnum = (Status) checkResult.get(Constants.STATUS); - if (resultEnum != Status.SUCCESS) { - return checkResult; - } - - ProcessDefinition processDefinition = processDefineMapper.selectById(processDefinitionId); - - if (processDefinition == null) { - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processDefinitionId); - return result; - } - - // Determine if the login user is the owner of the process definition - if (loginUser.getId() != processDefinition.getUserId() && loginUser.getUserType() != UserType.ADMIN_USER) { - putMsg(result, Status.USER_NO_OPERATION_PERM); - return result; - } - - // check process definition is already online - if (processDefinition.getReleaseState() == ReleaseState.ONLINE) { - putMsg(result, Status.PROCESS_DEFINE_STATE_ONLINE, processDefinitionId); - return result; - } - - // get the timing according to the process definition - List schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId); - if (!schedules.isEmpty() && schedules.size() > 1) { - logger.warn("scheduler num is {},Greater than 1", schedules.size()); - putMsg(result, Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR); - return result; - } else if (schedules.size() == 1) { - Schedule schedule = schedules.get(0); - if (schedule.getReleaseState() == ReleaseState.OFFLINE) { - scheduleMapper.deleteById(schedule.getId()); - } else if (schedule.getReleaseState() == ReleaseState.ONLINE) { - putMsg(result, Status.SCHEDULE_CRON_STATE_ONLINE, schedule.getId()); - return result; - } - } - - int delete = processDefineMapper.deleteById(processDefinitionId); - - if (delete > 0) { - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR); - } - return result; - } + Map deleteProcessDefinitionById(User loginUser, + String projectName, + Integer processDefinitionId); /** * release process definition: online / offline @@ -504,244 +157,23 @@ public class ProcessDefinitionService extends BaseDAGService { * @param releaseState release state * @return release result code */ - @Transactional(rollbackFor = RuntimeException.class) - public Map releaseProcessDefinition(User loginUser, String projectName, int id, int releaseState) { - HashMap result = new HashMap<>(); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultEnum = (Status) checkResult.get(Constants.STATUS); - if (resultEnum != Status.SUCCESS) { - return checkResult; - } - - ReleaseState state = ReleaseState.getEnum(releaseState); - - // check state - if (null == state) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); - return result; - } - - ProcessDefinition processDefinition = processDefineMapper.selectById(id); - - switch (state) { - case ONLINE: - // To check resources whether they are already cancel authorized or deleted - String resourceIds = processDefinition.getResourceIds(); - if (StringUtils.isNotBlank(resourceIds)) { - Integer[] resourceIdArray = Arrays.stream(resourceIds.split(",")).map(Integer::parseInt).toArray(Integer[]::new); - PermissionCheck permissionCheck = new PermissionCheck<>(AuthorizationType.RESOURCE_FILE_ID, processService, resourceIdArray, loginUser.getId(), logger); - try { - permissionCheck.checkPermission(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - putMsg(result, Status.RESOURCE_NOT_EXIST_OR_NO_PERMISSION, RELEASESTATE); - return result; - } - } - - processDefinition.setReleaseState(state); - processDefineMapper.updateById(processDefinition); - break; - case OFFLINE: - processDefinition.setReleaseState(state); - processDefineMapper.updateById(processDefinition); - List scheduleList = scheduleMapper.selectAllByProcessDefineArray( - new int[]{processDefinition.getId()} - ); - - for (Schedule schedule : scheduleList) { - logger.info("set schedule offline, project id: {}, schedule id: {}, process definition id: {}", project.getId(), schedule.getId(), id); - // set status - schedule.setReleaseState(ReleaseState.OFFLINE); - scheduleMapper.updateById(schedule); - SchedulerService.deleteSchedule(project.getId(), schedule.getId()); - } - break; - default: - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); - return result; - } - - putMsg(result, Status.SUCCESS); - return result; - } + Map releaseProcessDefinition(User loginUser, + String projectName, + int id, + int releaseState); /** * batch export process definition by ids * - * @param loginUser - * @param projectName - * @param processDefinitionIds - * @param response + * @param loginUser login user + * @param projectName project name + * @param processDefinitionIds process definition ids + * @param response http servlet response */ - public void batchExportProcessDefinitionByIds(User loginUser, String projectName, String processDefinitionIds, HttpServletResponse response) { - - if (StringUtils.isEmpty(processDefinitionIds)) { - return; - } - - //export project info - Project project = projectMapper.queryByName(projectName); - - //check user access for project - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - - if (resultStatus != Status.SUCCESS) { - return; - } - - List processDefinitionList = - getProcessDefinitionList(processDefinitionIds); - - if (CollectionUtils.isNotEmpty(processDefinitionList)) { - downloadProcessDefinitionFile(response, processDefinitionList); - } - } - - /** - * get process definition list by ids - * - * @param processDefinitionIds - * @return - */ - private List getProcessDefinitionList(String processDefinitionIds) { - List processDefinitionList = new ArrayList<>(); - String[] processDefinitionIdArray = processDefinitionIds.split(","); - for (String strProcessDefinitionId : processDefinitionIdArray) { - //get workflow info - int processDefinitionId = Integer.parseInt(strProcessDefinitionId); - ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefinitionId); - if (null != processDefinition) { - processDefinitionList.add(exportProcessMetaData(processDefinitionId, processDefinition)); - } - } - - return processDefinitionList; - } - - /** - * download the process definition file - * - * @param response - * @param processDefinitionList - */ - private void downloadProcessDefinitionFile(HttpServletResponse response, List processDefinitionList) { - response.setContentType(MediaType.APPLICATION_JSON_UTF8_VALUE); - BufferedOutputStream buff = null; - ServletOutputStream out = null; - try { - out = response.getOutputStream(); - buff = new BufferedOutputStream(out); - buff.write(JSONUtils.toJsonString(processDefinitionList).getBytes(StandardCharsets.UTF_8)); - buff.flush(); - buff.close(); - } catch (IOException e) { - logger.warn("export process fail", e); - } finally { - if (null != buff) { - try { - buff.close(); - } catch (Exception e) { - logger.warn("export process buffer not close", e); - } - } - if (null != out) { - try { - out.close(); - } catch (Exception e) { - logger.warn("export process output stream not close", e); - } - } - } - } - - /** - * get export process metadata string - * - * @param processDefinitionId process definition id - * @param processDefinition process definition - * @return export process metadata string - */ - public String exportProcessMetaDataStr(Integer processDefinitionId, ProcessDefinition processDefinition) { - //create workflow json file - return JSONUtils.toJsonString(exportProcessMetaData(processDefinitionId, processDefinition)); - } - - /** - * get export process metadata string - * - * @param processDefinitionId process definition id - * @param processDefinition process definition - * @return export process metadata string - */ - public ProcessMeta exportProcessMetaData(Integer processDefinitionId, ProcessDefinition processDefinition) { - //correct task param which has data source or dependent param - String correctProcessDefinitionJson = addExportTaskNodeSpecialParam(processDefinition.getProcessDefinitionJson()); - processDefinition.setProcessDefinitionJson(correctProcessDefinitionJson); - - //export process metadata - ProcessMeta exportProcessMeta = new ProcessMeta(); - exportProcessMeta.setProjectName(processDefinition.getProjectName()); - exportProcessMeta.setProcessDefinitionName(processDefinition.getName()); - exportProcessMeta.setProcessDefinitionJson(processDefinition.getProcessDefinitionJson()); - exportProcessMeta.setProcessDefinitionLocations(processDefinition.getLocations()); - exportProcessMeta.setProcessDefinitionConnects(processDefinition.getConnects()); - - //schedule info - List schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId); - if (!schedules.isEmpty()) { - Schedule schedule = schedules.get(0); - exportProcessMeta.setScheduleWarningType(schedule.getWarningType().toString()); - exportProcessMeta.setScheduleWarningGroupId(schedule.getWarningGroupId()); - exportProcessMeta.setScheduleStartTime(DateUtils.dateToString(schedule.getStartTime())); - exportProcessMeta.setScheduleEndTime(DateUtils.dateToString(schedule.getEndTime())); - exportProcessMeta.setScheduleCrontab(schedule.getCrontab()); - exportProcessMeta.setScheduleFailureStrategy(String.valueOf(schedule.getFailureStrategy())); - exportProcessMeta.setScheduleReleaseState(String.valueOf(ReleaseState.OFFLINE)); - exportProcessMeta.setScheduleProcessInstancePriority(String.valueOf(schedule.getProcessInstancePriority())); - exportProcessMeta.setScheduleWorkerGroupName(schedule.getWorkerGroup()); - } - //create workflow json file - return exportProcessMeta; - } - - /** - * correct task param which has datasource or dependent - * - * @param processDefinitionJson processDefinitionJson - * @return correct processDefinitionJson - */ - public String addExportTaskNodeSpecialParam(String processDefinitionJson) { - ObjectNode jsonObject = JSONUtils.parseObject(processDefinitionJson); - ArrayNode jsonArray = (ArrayNode) jsonObject.path(TASKS); - - for (int i = 0; i < jsonArray.size(); i++) { - JsonNode taskNode = jsonArray.path(i); - if (StringUtils.isNotEmpty(taskNode.path("type").asText())) { - String taskType = taskNode.path("type").asText(); - - ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); - if (null != addTaskParam) { - addTaskParam.addExportSpecialParam(taskNode); - } - } - } - jsonObject.set(TASKS, jsonArray); - return jsonObject.toString(); - } - - /** - * check task if has sub process - * - * @param taskType task type - * @return if task has sub process return true else false - */ - private boolean checkTaskHasSubProcess(String taskType) { - return taskType.equals(TaskType.SUB_PROCESS.name()); - } + void batchExportProcessDefinitionByIds(User loginUser, + String projectName, + String processDefinitionIds, + HttpServletResponse response); /** * import process definition @@ -751,357 +183,9 @@ public class ProcessDefinitionService extends BaseDAGService { * @param currentProjectName current project name * @return import process */ - @Transactional(rollbackFor = RuntimeException.class) - public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { - Map result = new HashMap<>(5); - String processMetaJson = FileUtils.file2String(file); - List processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class); - - //check file content - if (CollectionUtils.isEmpty(processMetaList)) { - putMsg(result, Status.DATA_IS_NULL, "fileContent"); - return result; - } - - for (ProcessMeta processMeta : processMetaList) { - - if (!checkAndImportProcessDefinition(loginUser, currentProjectName, result, processMeta)) { - return result; - } - } - - return result; - } - - /** - * check and import process definition - * - * @param loginUser - * @param currentProjectName - * @param result - * @param processMeta - * @return - */ - private boolean checkAndImportProcessDefinition(User loginUser, String currentProjectName, Map result, ProcessMeta processMeta) { - - if (!checkImportanceParams(processMeta, result)) { - return false; - } - - //deal with process name - String processDefinitionName = processMeta.getProcessDefinitionName(); - //use currentProjectName to query - Project targetProject = projectMapper.queryByName(currentProjectName); - if (null != targetProject) { - processDefinitionName = recursionProcessDefinitionName(targetProject.getId(), - processDefinitionName, 1); - } - - //unique check - Map checkResult = verifyProcessDefinitionName(loginUser, currentProjectName, processDefinitionName); - Status status = (Status) checkResult.get(Constants.STATUS); - if (Status.SUCCESS.equals(status)) { - putMsg(result, Status.SUCCESS); - } else { - result.putAll(checkResult); - return false; - } - - // get create process result - Map createProcessResult = - getCreateProcessResult(loginUser, - currentProjectName, - result, - processMeta, - processDefinitionName, - addImportTaskNodeParam(loginUser, processMeta.getProcessDefinitionJson(), targetProject)); - - if (createProcessResult == null) { - return false; - } - - //create process definition - Integer processDefinitionId = - Objects.isNull(createProcessResult.get(PROCESSDEFINITIONID)) ? - null : Integer.parseInt(createProcessResult.get(PROCESSDEFINITIONID).toString()); - - //scheduler param - return getImportProcessScheduleResult(loginUser, - currentProjectName, - result, - processMeta, - processDefinitionName, - processDefinitionId); - - } - - /** - * get create process result - * - * @param loginUser - * @param currentProjectName - * @param result - * @param processMeta - * @param processDefinitionName - * @param importProcessParam - * @return - */ - private Map getCreateProcessResult(User loginUser, - String currentProjectName, - Map result, - ProcessMeta processMeta, - String processDefinitionName, - String importProcessParam) { - Map createProcessResult = null; - try { - createProcessResult = createProcessDefinition(loginUser - , currentProjectName, - processDefinitionName + "_import_" + System.currentTimeMillis(), - importProcessParam, - processMeta.getProcessDefinitionDescription(), - processMeta.getProcessDefinitionLocations(), - processMeta.getProcessDefinitionConnects()); - putMsg(result, Status.SUCCESS); - } catch (JsonProcessingException e) { - logger.error("import process meta json data: {}", e.getMessage(), e); - putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); - } - - return createProcessResult; - } - - /** - * get import process schedule result - * - * @param loginUser - * @param currentProjectName - * @param result - * @param processMeta - * @param processDefinitionName - * @param processDefinitionId - * @return - */ - private boolean getImportProcessScheduleResult(User loginUser, - String currentProjectName, - Map result, - ProcessMeta processMeta, - String processDefinitionName, - Integer processDefinitionId) { - if (null != processMeta.getScheduleCrontab() && null != processDefinitionId) { - int scheduleInsert = importProcessSchedule(loginUser, - currentProjectName, - processMeta, - processDefinitionName, - processDefinitionId); - - if (0 == scheduleInsert) { - putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); - return false; - } - } - return true; - } - - /** - * check importance params - * - * @param processMeta - * @param result - * @return - */ - private boolean checkImportanceParams(ProcessMeta processMeta, Map result) { - if (StringUtils.isEmpty(processMeta.getProjectName())) { - putMsg(result, Status.DATA_IS_NULL, "projectName"); - return false; - } - if (StringUtils.isEmpty(processMeta.getProcessDefinitionName())) { - putMsg(result, Status.DATA_IS_NULL, "processDefinitionName"); - return false; - } - if (StringUtils.isEmpty(processMeta.getProcessDefinitionJson())) { - putMsg(result, Status.DATA_IS_NULL, "processDefinitionJson"); - return false; - } - - return true; - } - - /** - * import process add special task param - * - * @param loginUser login user - * @param processDefinitionJson process definition json - * @param targetProject target project - * @return import process param - */ - private String addImportTaskNodeParam(User loginUser, String processDefinitionJson, Project targetProject) { - ObjectNode jsonObject = JSONUtils.parseObject(processDefinitionJson); - ArrayNode jsonArray = (ArrayNode) jsonObject.get(TASKS); - //add sql and dependent param - for (int i = 0; i < jsonArray.size(); i++) { - JsonNode taskNode = jsonArray.path(i); - String taskType = taskNode.path("type").asText(); - ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); - if (null != addTaskParam) { - addTaskParam.addImportSpecialParam(taskNode); - } - } - - //recursive sub-process parameter correction map key for old process id value for new process id - Map subProcessIdMap = new HashMap<>(20); - - List subProcessList = StreamUtils.asStream(jsonArray.elements()) - .filter(elem -> checkTaskHasSubProcess(JSONUtils.parseObject(elem.toString()).path("type").asText())) - .collect(Collectors.toList()); - - if (CollectionUtils.isNotEmpty(subProcessList)) { - importSubProcess(loginUser, targetProject, jsonArray, subProcessIdMap); - } - - jsonObject.set(TASKS, jsonArray); - return jsonObject.toString(); - } - - /** - * import process schedule - * - * @param loginUser login user - * @param currentProjectName current project name - * @param processMeta process meta data - * @param processDefinitionName process definition name - * @param processDefinitionId process definition id - * @return insert schedule flag - */ - public int importProcessSchedule(User loginUser, String currentProjectName, ProcessMeta processMeta, - String processDefinitionName, Integer processDefinitionId) { - Date now = new Date(); - Schedule scheduleObj = new Schedule(); - scheduleObj.setProjectName(currentProjectName); - scheduleObj.setProcessDefinitionId(processDefinitionId); - scheduleObj.setProcessDefinitionName(processDefinitionName); - scheduleObj.setCreateTime(now); - scheduleObj.setUpdateTime(now); - scheduleObj.setUserId(loginUser.getId()); - scheduleObj.setUserName(loginUser.getUserName()); - - scheduleObj.setCrontab(processMeta.getScheduleCrontab()); - - if (null != processMeta.getScheduleStartTime()) { - scheduleObj.setStartTime(DateUtils.stringToDate(processMeta.getScheduleStartTime())); - } - if (null != processMeta.getScheduleEndTime()) { - scheduleObj.setEndTime(DateUtils.stringToDate(processMeta.getScheduleEndTime())); - } - if (null != processMeta.getScheduleWarningType()) { - scheduleObj.setWarningType(WarningType.valueOf(processMeta.getScheduleWarningType())); - } - if (null != processMeta.getScheduleWarningGroupId()) { - scheduleObj.setWarningGroupId(processMeta.getScheduleWarningGroupId()); - } - if (null != processMeta.getScheduleFailureStrategy()) { - scheduleObj.setFailureStrategy(FailureStrategy.valueOf(processMeta.getScheduleFailureStrategy())); - } - if (null != processMeta.getScheduleReleaseState()) { - scheduleObj.setReleaseState(ReleaseState.valueOf(processMeta.getScheduleReleaseState())); - } - if (null != processMeta.getScheduleProcessInstancePriority()) { - scheduleObj.setProcessInstancePriority(Priority.valueOf(processMeta.getScheduleProcessInstancePriority())); - } - - if (null != processMeta.getScheduleWorkerGroupName()) { - scheduleObj.setWorkerGroup(processMeta.getScheduleWorkerGroupName()); - } - - return scheduleMapper.insert(scheduleObj); - } - - /** - * check import process has sub process - * recursion create sub process - * - * @param loginUser login user - * @param targetProject target project - * @param jsonArray process task array - * @param subProcessIdMap correct sub process id map - */ - public void importSubProcess(User loginUser, Project targetProject, ArrayNode jsonArray, Map subProcessIdMap) { - for (int i = 0; i < jsonArray.size(); i++) { - ObjectNode taskNode = (ObjectNode) jsonArray.path(i); - String taskType = taskNode.path("type").asText(); - - if (!checkTaskHasSubProcess(taskType)) { - continue; - } - //get sub process info - ObjectNode subParams = (ObjectNode) taskNode.path("params"); - Integer subProcessId = subParams.path(PROCESSDEFINITIONID).asInt(); - ProcessDefinition subProcess = processDefineMapper.queryByDefineId(subProcessId); - //check is sub process exist in db - if (null == subProcess) { - continue; - } - String subProcessJson = subProcess.getProcessDefinitionJson(); - //check current project has sub process - ProcessDefinition currentProjectSubProcess = processDefineMapper.queryByDefineName(targetProject.getId(), subProcess.getName()); - - if (null == currentProjectSubProcess) { - ArrayNode subJsonArray = (ArrayNode) JSONUtils.parseObject(subProcess.getProcessDefinitionJson()).get(TASKS); - - List subProcessList = StreamUtils.asStream(subJsonArray.elements()) - .filter(item -> checkTaskHasSubProcess(JSONUtils.parseObject(item.toString()).path("type").asText())) - .collect(Collectors.toList()); - - if (CollectionUtils.isNotEmpty(subProcessList)) { - importSubProcess(loginUser, targetProject, subJsonArray, subProcessIdMap); - //sub process processId correct - if (!subProcessIdMap.isEmpty()) { - - for (Map.Entry entry : subProcessIdMap.entrySet()) { - String oldSubProcessId = "\"processDefinitionId\":" + entry.getKey(); - String newSubProcessId = "\"processDefinitionId\":" + entry.getValue(); - subProcessJson = subProcessJson.replaceAll(oldSubProcessId, newSubProcessId); - } - - subProcessIdMap.clear(); - } - } - - //if sub-process recursion - Date now = new Date(); - //create sub process in target project - ProcessDefinition processDefine = new ProcessDefinition(); - processDefine.setName(subProcess.getName()); - processDefine.setVersion(subProcess.getVersion()); - processDefine.setReleaseState(subProcess.getReleaseState()); - processDefine.setProjectId(targetProject.getId()); - processDefine.setUserId(loginUser.getId()); - processDefine.setProcessDefinitionJson(subProcessJson); - processDefine.setDescription(subProcess.getDescription()); - processDefine.setLocations(subProcess.getLocations()); - processDefine.setConnects(subProcess.getConnects()); - processDefine.setTimeout(subProcess.getTimeout()); - processDefine.setTenantId(subProcess.getTenantId()); - processDefine.setGlobalParams(subProcess.getGlobalParams()); - processDefine.setCreateTime(now); - processDefine.setUpdateTime(now); - processDefine.setFlag(subProcess.getFlag()); - processDefine.setReceivers(subProcess.getReceivers()); - processDefine.setReceiversCc(subProcess.getReceiversCc()); - processDefineMapper.insert(processDefine); - - logger.info("create sub process, project: {}, process name: {}", targetProject.getName(), processDefine.getName()); - - //modify task node - ProcessDefinition newSubProcessDefine = processDefineMapper.queryByDefineName(processDefine.getProjectId(), processDefine.getName()); - - if (null != newSubProcessDefine) { - subProcessIdMap.put(subProcessId, newSubProcessDefine.getId()); - subParams.put(PROCESSDEFINITIONID, newSubProcessDefine.getId()); - taskNode.set("params", subParams); - } - } - } - } + Map importProcessDefinition(User loginUser, + MultipartFile file, + String currentProjectName); /** @@ -1111,50 +195,8 @@ public class ProcessDefinitionService extends BaseDAGService { * @param processDefinitionJson process definition json * @return check result code */ - public Map checkProcessNodeList(ProcessData processData, String processDefinitionJson) { - - Map result = new HashMap<>(5); - try { - if (processData == null) { - logger.error("process data is null"); - putMsg(result, Status.DATA_IS_NOT_VALID, processDefinitionJson); - return result; - } - - // Check whether the task node is normal - List taskNodes = processData.getTasks(); - - if (taskNodes == null) { - logger.error("process node info is empty"); - putMsg(result, Status.DATA_IS_NULL, processDefinitionJson); - return result; - } - - // check has cycle - if (graphHasCycle(taskNodes)) { - logger.error("process DAG has cycle"); - putMsg(result, Status.PROCESS_NODE_HAS_CYCLE); - return result; - } - - // check whether the process definition json is normal - for (TaskNode taskNode : taskNodes) { - if (!CheckUtils.checkTaskNodeParameters(taskNode.getParams(), taskNode.getType())) { - logger.error("task node {} parameter invalid", taskNode.getName()); - putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskNode.getName()); - return result; - } - - // check extra params - CheckUtils.checkOtherParams(taskNode.getExtras()); - } - putMsg(result, Status.SUCCESS); - } catch (Exception e) { - result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); - result.put(Constants.MSG, e.getMessage()); - } - return result; - } + Map checkProcessNodeList(ProcessData processData, + String processDefinitionJson); /** * get task node details based on process definition @@ -1162,36 +204,7 @@ public class ProcessDefinitionService extends BaseDAGService { * @param defineId define id * @return task node list */ - public Map getTaskNodeListByDefinitionId(Integer defineId) { - Map result = new HashMap<>(); - - ProcessDefinition processDefinition = processDefineMapper.selectById(defineId); - if (processDefinition == null) { - logger.info("process define not exists"); - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineId); - return result; - } - - - String processDefinitionJson = processDefinition.getProcessDefinitionJson(); - - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - - //process data check - if (null == processData) { - logger.error("process data is null"); - putMsg(result, Status.DATA_IS_NOT_VALID, processDefinitionJson); - return result; - } - - List taskNodeList = (processData.getTasks() == null) ? new ArrayList<>() : processData.getTasks(); - - result.put(Constants.DATA_LIST, taskNodeList); - putMsg(result, Status.SUCCESS); - - return result; - - } + Map getTaskNodeListByDefinitionId(Integer defineId); /** * get task node details based on process definition @@ -1199,36 +212,7 @@ public class ProcessDefinitionService extends BaseDAGService { * @param defineIdList define id list * @return task node list */ - public Map getTaskNodeListByDefinitionIdList(String defineIdList) { - Map result = new HashMap<>(); - - Map> taskNodeMap = new HashMap<>(); - String[] idList = defineIdList.split(","); - List idIntList = new ArrayList<>(); - for (String definitionId : idList) { - idIntList.add(Integer.parseInt(definitionId)); - } - Integer[] idArray = idIntList.toArray(new Integer[idIntList.size()]); - List processDefinitionList = processDefineMapper.queryDefinitionListByIdList(idArray); - if (CollectionUtils.isEmpty(processDefinitionList)) { - logger.info("process definition not exists"); - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineIdList); - return result; - } - - for (ProcessDefinition processDefinition : processDefinitionList) { - String processDefinitionJson = processDefinition.getProcessDefinitionJson(); - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - List taskNodeList = (processData.getTasks() == null) ? new ArrayList<>() : processData.getTasks(); - taskNodeMap.put(processDefinition.getId(), taskNodeList); - } - - result.put(Constants.DATA_LIST, taskNodeMap); - putMsg(result, Status.SUCCESS); - - return result; - - } + Map getTaskNodeListByDefinitionIdList(String defineIdList); /** @@ -1237,16 +221,7 @@ public class ProcessDefinitionService extends BaseDAGService { * @param projectId project id * @return process definitions in the project */ - public Map queryProcessDefinitionAllByProjectId(Integer projectId) { - - HashMap result = new HashMap<>(5); - - List resourceList = processDefineMapper.queryAllDefinitionList(projectId); - result.put(Constants.DATA_LIST, resourceList); - putMsg(result, Status.SUCCESS); - - return result; - } + Map queryProcessDefinitionAllByProjectId(Integer projectId); /** * Encapsulates the TreeView structure @@ -1256,200 +231,7 @@ public class ProcessDefinitionService extends BaseDAGService { * @return tree view json data * @throws Exception exception */ - public Map viewTree(Integer processId, Integer limit) throws Exception { - Map result = new HashMap<>(); - - ProcessDefinition processDefinition = processDefineMapper.selectById(processId); - if (null == processDefinition) { - logger.info("process define not exists"); - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processDefinition); - return result; - } - DAG dag = genDagGraph(processDefinition); - /** - * nodes that is running - */ - Map> runningNodeMap = new ConcurrentHashMap<>(); - - /** - * nodes that is waiting torun - */ - Map> waitingRunningNodeMap = new ConcurrentHashMap<>(); - - /** - * List of process instances - */ - List processInstanceList = processInstanceMapper.queryByProcessDefineId(processId, limit); - - for (ProcessInstance processInstance : processInstanceList) { - processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime())); - } - - if (limit > processInstanceList.size()) { - limit = processInstanceList.size(); - } - - TreeViewDto parentTreeViewDto = new TreeViewDto(); - parentTreeViewDto.setName("DAG"); - parentTreeViewDto.setType(""); - // Specify the process definition, because it is a TreeView for a process definition - - for (int i = limit - 1; i >= 0; i--) { - ProcessInstance processInstance = processInstanceList.get(i); - - Date endTime = processInstance.getEndTime() == null ? new Date() : processInstance.getEndTime(); - parentTreeViewDto.getInstances().add(new Instance(processInstance.getId(), processInstance.getName(), "", processInstance.getState().toString() - , processInstance.getStartTime(), endTime, processInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - processInstance.getStartTime().getTime()))); - } - - List parentTreeViewDtoList = new ArrayList<>(); - parentTreeViewDtoList.add(parentTreeViewDto); - // Here is the encapsulation task instance - for (String startNode : dag.getBeginNode()) { - runningNodeMap.put(startNode, parentTreeViewDtoList); - } - - while (Stopper.isRunning()) { - Set postNodeList = null; - Iterator>> iter = runningNodeMap.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry> en = iter.next(); - String nodeName = en.getKey(); - parentTreeViewDtoList = en.getValue(); - - TreeViewDto treeViewDto = new TreeViewDto(); - treeViewDto.setName(nodeName); - TaskNode taskNode = dag.getNode(nodeName); - treeViewDto.setType(taskNode.getType()); - - - //set treeViewDto instances - for (int i = limit - 1; i >= 0; i--) { - ProcessInstance processInstance = processInstanceList.get(i); - TaskInstance taskInstance = taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), nodeName); - if (taskInstance == null) { - treeViewDto.getInstances().add(new Instance(-1, "not running", "null")); - } else { - Date startTime = taskInstance.getStartTime() == null ? new Date() : taskInstance.getStartTime(); - Date endTime = taskInstance.getEndTime() == null ? new Date() : taskInstance.getEndTime(); - - int subProcessId = 0; - /** - * if process is sub process, the return sub id, or sub id=0 - */ - if (taskInstance.getTaskType().equals(TaskType.SUB_PROCESS.name())) { - String taskJson = taskInstance.getTaskJson(); - taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); - subProcessId = Integer.parseInt(JSONUtils.parseObject( - taskNode.getParams()).path(CMDPARAM_SUB_PROCESS_DEFINE_ID).asText()); - } - treeViewDto.getInstances().add(new Instance(taskInstance.getId(), taskInstance.getName(), taskInstance.getTaskType(), taskInstance.getState().toString() - , taskInstance.getStartTime(), taskInstance.getEndTime(), taskInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - startTime.getTime()), subProcessId)); - } - } - for (TreeViewDto pTreeViewDto : parentTreeViewDtoList) { - pTreeViewDto.getChildren().add(treeViewDto); - } - postNodeList = dag.getSubsequentNodes(nodeName); - if (CollectionUtils.isNotEmpty(postNodeList)) { - for (String nextNodeName : postNodeList) { - List treeViewDtoList = waitingRunningNodeMap.get(nextNodeName); - if (CollectionUtils.isNotEmpty(treeViewDtoList)) { - treeViewDtoList.add(treeViewDto); - waitingRunningNodeMap.put(nextNodeName, treeViewDtoList); - } else { - treeViewDtoList = new ArrayList<>(); - treeViewDtoList.add(treeViewDto); - waitingRunningNodeMap.put(nextNodeName, treeViewDtoList); - } - } - } - runningNodeMap.remove(nodeName); - } - if (waitingRunningNodeMap == null || waitingRunningNodeMap.size() == 0) { - break; - } else { - runningNodeMap.putAll(waitingRunningNodeMap); - waitingRunningNodeMap.clear(); - } - } - result.put(Constants.DATA_LIST, parentTreeViewDto); - result.put(Constants.STATUS, Status.SUCCESS); - result.put(Constants.MSG, Status.SUCCESS.getMsg()); - return result; - } - - - /** - * Generate the DAG Graph based on the process definition id - * - * @param processDefinition process definition - * @return dag graph - */ - private DAG genDagGraph(ProcessDefinition processDefinition) { - - String processDefinitionJson = processDefinition.getProcessDefinitionJson(); - - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - - //check process data - if (null != processData) { - List taskNodeList = processData.getTasks(); - processDefinition.setGlobalParamList(processData.getGlobalParams()); - ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); - - // Generate concrete Dag to be executed - return DagHelper.buildDagGraph(processDag); - } - - return new DAG<>(); - } - - - /** - * whether the graph has a ring - * - * @param taskNodeResponseList task node response list - * @return if graph has cycle flag - */ - private boolean graphHasCycle(List taskNodeResponseList) { - DAG graph = new DAG<>(); - - // Fill the vertices - for (TaskNode taskNodeResponse : taskNodeResponseList) { - graph.addNode(taskNodeResponse.getName(), taskNodeResponse); - } - - // Fill edge relations - for (TaskNode taskNodeResponse : taskNodeResponseList) { - taskNodeResponse.getPreTasks(); - List preTasks = JSONUtils.toList(taskNodeResponse.getPreTasks(), String.class); - if (CollectionUtils.isNotEmpty(preTasks)) { - for (String preTask : preTasks) { - if (!graph.addEdge(preTask, taskNodeResponse.getName())) { - return true; - } - } - } - } - - return graph.hasCycle(); - } - - private String recursionProcessDefinitionName(Integer projectId, String processDefinitionName, int num) { - ProcessDefinition processDefinition = processDefineMapper.queryByDefineName(projectId, processDefinitionName); - if (processDefinition != null) { - if (num > 1) { - String str = processDefinitionName.substring(0, processDefinitionName.length() - 3); - processDefinitionName = str + "(" + num + ")"; - } else { - processDefinitionName = processDefinition.getName() + "(" + num + ")"; - } - } else { - return processDefinitionName; - } - return recursionProcessDefinitionName(projectId, processDefinitionName, num + 1); - } - + Map viewTree(Integer processId, + Integer limit) throws Exception; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index e4a00f3895..dc3dbe976f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -16,8 +16,28 @@ */ package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +import static org.apache.dolphinscheduler.common.Constants.DATA_LIST; +import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; +import static org.apache.dolphinscheduler.common.Constants.GLOBAL_PARAMS; +import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; +import static org.apache.dolphinscheduler.common.Constants.PROCESS_INSTANCE_STATE; +import static org.apache.dolphinscheduler.common.Constants.TASK_LIST; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -31,14 +51,26 @@ import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; +import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.process.Property; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; -import org.apache.dolphinscheduler.dao.entity.*; +import org.apache.dolphinscheduler.dao.entity.ProcessData; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,22 +78,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.text.ParseException; -import java.util.*; -import java.util.stream.Collectors; - -import static org.apache.dolphinscheduler.common.Constants.*; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * process instance service */ @Service -public class ProcessInstanceService extends BaseDAGService { +public class ProcessInstanceService extends BaseService { private static final Logger logger = LoggerFactory.getLogger(ProcessInstanceService.class); @@ -167,7 +191,7 @@ public class ProcessInstanceService extends BaseDAGService { ProcessDefinition processDefinition = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); processInstance.setReceivers(processDefinition.getReceivers()); processInstance.setReceiversCc(processDefinition.getReceiversCc()); - result.put(Constants.DATA_LIST, processInstance); + result.put(DATA_LIST, processInstance); putMsg(result, Status.SUCCESS); return result; @@ -242,7 +266,7 @@ public class ProcessInstanceService extends BaseDAGService { pageInfo.setTotalCount((int) processInstanceList.getTotal()); pageInfo.setLists(processInstances); - result.put(Constants.DATA_LIST, pageInfo); + result.put(DATA_LIST, pageInfo); putMsg(result, Status.SUCCESS); return result; } @@ -273,7 +297,7 @@ public class ProcessInstanceService extends BaseDAGService { Map resultMap = new HashMap<>(); resultMap.put(PROCESS_INSTANCE_STATE, processInstance.getState().toString()); resultMap.put(TASK_LIST, taskInstanceList); - result.put(Constants.DATA_LIST, resultMap); + result.put(DATA_LIST, resultMap); putMsg(result, Status.SUCCESS); return result; @@ -362,7 +386,7 @@ public class ProcessInstanceService extends BaseDAGService { } Map dataMap = new HashMap<>(); dataMap.put("subProcessInstanceId", subWorkflowInstance.getId()); - result.put(Constants.DATA_LIST, dataMap); + result.put(DATA_LIST, dataMap); putMsg(result, Status.SUCCESS); return result; } @@ -501,7 +525,7 @@ public class ProcessInstanceService extends BaseDAGService { } Map dataMap = new HashMap<>(); dataMap.put("parentWorkflowInstance", parentWorkflowInstance.getId()); - result.put(Constants.DATA_LIST, dataMap); + result.put(DATA_LIST, dataMap); putMsg(result, Status.SUCCESS); return result; } @@ -618,7 +642,7 @@ public class ProcessInstanceService extends BaseDAGService { resultMap.put(GLOBAL_PARAMS, globalParams); resultMap.put(LOCAL_PARAMS, localUserDefParams); - result.put(Constants.DATA_LIST, resultMap); + result.put(DATA_LIST, resultMap); putMsg(result, Status.SUCCESS); return result; } @@ -668,9 +692,28 @@ public class ProcessInstanceService extends BaseDAGService { } ganttDto.setTasks(taskList); - result.put(Constants.DATA_LIST, ganttDto); + result.put(DATA_LIST, ganttDto); putMsg(result, Status.SUCCESS); return result; } + /** + * process instance to DAG + * + * @param processInstance input process instance + * @return process instance dag. + */ + private static DAG processInstance2DAG(ProcessInstance processInstance) { + + String processDefinitionJson = processInstance.getProcessInstanceJson(); + + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + + List taskNodeList = processData.getTasks(); + + ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); + + return DagHelper.buildDagGraph(processDag); + } + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java new file mode 100644 index 0000000000..ee02940c09 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -0,0 +1,1493 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service.impl; + +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS_DEFINE_ID; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; + +import org.apache.dolphinscheduler.api.dto.ProcessMeta; +import org.apache.dolphinscheduler.api.dto.treeview.Instance; +import org.apache.dolphinscheduler.api.dto.treeview.TreeViewDto; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; +import org.apache.dolphinscheduler.api.service.ProjectService; +import org.apache.dolphinscheduler.api.service.SchedulerService; +import org.apache.dolphinscheduler.api.utils.CheckUtils; +import org.apache.dolphinscheduler.api.utils.FileUtils; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.exportprocess.ProcessAddTaskParam; +import org.apache.dolphinscheduler.api.utils.exportprocess.TaskNodeParamFactory; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.AuthorizationType; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.enums.TaskType; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.graph.DAG; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.model.TaskNodeRelation; +import org.apache.dolphinscheduler.common.process.ProcessDag; +import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StreamUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessData; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.Schedule; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.utils.DagHelper; +import org.apache.dolphinscheduler.service.permission.PermissionCheck; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * process definition service + */ +@Service +public class ProcessDefinitionServiceImpl extends BaseService implements + ProcessDefinitionService { + + private static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionServiceImpl.class); + + private static final String PROCESSDEFINITIONID = "processDefinitionId"; + + private static final String RELEASESTATE = "releaseState"; + + private static final String TASKS = "tasks"; + + @Autowired + private ProjectMapper projectMapper; + + @Autowired + private ProjectService projectService; + + @Autowired + private ProcessDefinitionMapper processDefineMapper; + + @Autowired + private ProcessInstanceMapper processInstanceMapper; + + + @Autowired + private TaskInstanceMapper taskInstanceMapper; + + @Autowired + private ScheduleMapper scheduleMapper; + + @Autowired + private ProcessService processService; + + /** + * create process definition + * + * @param loginUser login user + * @param projectName project name + * @param name process definition name + * @param processDefinitionJson process definition json + * @param desc description + * @param locations locations for nodes + * @param connects connects for nodes + * @return create result code + * @throws JsonProcessingException JsonProcessingException + */ + public Map createProcessDefinition(User loginUser, + String projectName, + String name, + String processDefinitionJson, + String desc, + String locations, + String connects) throws JsonProcessingException { + + Map result = new HashMap<>(5); + Project project = projectMapper.queryByName(projectName); + // check project auth + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + ProcessDefinition processDefine = new ProcessDefinition(); + Date now = new Date(); + + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + Map checkProcessJson = checkProcessNodeList(processData, processDefinitionJson); + if (checkProcessJson.get(Constants.STATUS) != Status.SUCCESS) { + return checkProcessJson; + } + + processDefine.setName(name); + processDefine.setReleaseState(ReleaseState.OFFLINE); + processDefine.setProjectId(project.getId()); + processDefine.setUserId(loginUser.getId()); + processDefine.setProcessDefinitionJson(processDefinitionJson); + processDefine.setDescription(desc); + processDefine.setLocations(locations); + processDefine.setConnects(connects); + processDefine.setTimeout(processData.getTimeout()); + processDefine.setTenantId(processData.getTenantId()); + processDefine.setModifyBy(loginUser.getUserName()); + processDefine.setResourceIds(getResourceIds(processData)); + + //custom global params + List globalParamsList = processData.getGlobalParams(); + if (CollectionUtils.isNotEmpty(globalParamsList)) { + Set globalParamsSet = new HashSet<>(globalParamsList); + globalParamsList = new ArrayList<>(globalParamsSet); + processDefine.setGlobalParamList(globalParamsList); + } + processDefine.setCreateTime(now); + processDefine.setUpdateTime(now); + processDefine.setFlag(Flag.YES); + processDefineMapper.insert(processDefine); + + // return processDefinition object with ID + result.put(Constants.DATA_LIST, processDefineMapper.selectById(processDefine.getId())); + putMsg(result, Status.SUCCESS); + result.put("processDefinitionId", processDefine.getId()); + return result; + } + + /** + * get resource ids + * + * @param processData process data + * @return resource ids + */ + private String getResourceIds(ProcessData processData) { + List tasks = processData.getTasks(); + Set resourceIds = new HashSet<>(); + for (TaskNode taskNode : tasks) { + String taskParameter = taskNode.getParams(); + AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(), taskParameter); + if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) { + Set tempSet = params.getResourceFilesList().stream().map(t -> t.getId()).collect(Collectors.toSet()); + resourceIds.addAll(tempSet); + } + } + + StringBuilder sb = new StringBuilder(); + for (int i : resourceIds) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(i); + } + return sb.toString(); + } + + + /** + * query process definition list + * + * @param loginUser login user + * @param projectName project name + * @return definition list + */ + public Map queryProcessDefinitionList(User loginUser, String projectName) { + + HashMap result = new HashMap<>(5); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + List resourceList = processDefineMapper.queryAllDefinitionList(project.getId()); + result.put(Constants.DATA_LIST, resourceList); + putMsg(result, Status.SUCCESS); + + return result; + } + + + /** + * query process definition list paging + * + * @param loginUser login user + * @param projectName project name + * @param searchVal search value + * @param pageNo page number + * @param pageSize page size + * @param userId user id + * @return process definition page + */ + public Map queryProcessDefinitionListPaging(User loginUser, String projectName, String searchVal, Integer pageNo, Integer pageSize, Integer userId) { + + Map result = new HashMap<>(5); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + Page page = new Page(pageNo, pageSize); + IPage processDefinitionIPage = processDefineMapper.queryDefineListPaging( + page, searchVal, userId, project.getId(), isAdmin(loginUser)); + + PageInfo pageInfo = new PageInfo(pageNo, pageSize); + pageInfo.setTotalCount((int) processDefinitionIPage.getTotal()); + pageInfo.setLists(processDefinitionIPage.getRecords()); + result.put(Constants.DATA_LIST, pageInfo); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * query datail of process definition + * + * @param loginUser login user + * @param projectName project name + * @param processId process definition id + * @return process definition detail + */ + public Map queryProcessDefinitionById(User loginUser, String projectName, Integer processId) { + + + Map result = new HashMap<>(5); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + ProcessDefinition processDefinition = processDefineMapper.selectById(processId); + if (processDefinition == null) { + putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processId); + } else { + result.put(Constants.DATA_LIST, processDefinition); + putMsg(result, Status.SUCCESS); + } + return result; + } + + /** + * copy process definition + * + * @param loginUser login user + * @param projectName project name + * @param processId process definition id + * @return copy result code + */ + public Map copyProcessDefinition(User loginUser, String projectName, Integer processId) throws JsonProcessingException { + + Map result = new HashMap<>(5); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + ProcessDefinition processDefinition = processDefineMapper.selectById(processId); + if (processDefinition == null) { + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId); + return result; + } else { + return createProcessDefinition( + loginUser, + projectName, + processDefinition.getName() + "_copy_" + System.currentTimeMillis(), + processDefinition.getProcessDefinitionJson(), + processDefinition.getDescription(), + processDefinition.getLocations(), + processDefinition.getConnects()); + } + } + + /** + * update process definition + * + * @param loginUser login user + * @param projectName project name + * @param name process definition name + * @param id process definition id + * @param processDefinitionJson process definition json + * @param desc description + * @param locations locations for nodes + * @param connects connects for nodes + * @return update result code + */ + public Map updateProcessDefinition(User loginUser, String projectName, int id, String name, + String processDefinitionJson, String desc, + String locations, String connects) { + Map result = new HashMap<>(5); + + Project project = projectMapper.queryByName(projectName); + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + Map checkProcessJson = checkProcessNodeList(processData, processDefinitionJson); + if ((checkProcessJson.get(Constants.STATUS) != Status.SUCCESS)) { + return checkProcessJson; + } + ProcessDefinition processDefine = processService.findProcessDefineById(id); + if (processDefine == null) { + // check process definition exists + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, id); + return result; + } else if (processDefine.getReleaseState() == ReleaseState.ONLINE) { + // online can not permit edit + putMsg(result, Status.PROCESS_DEFINE_NOT_ALLOWED_EDIT, processDefine.getName()); + return result; + } else { + putMsg(result, Status.SUCCESS); + } + + Date now = new Date(); + + processDefine.setId(id); + processDefine.setName(name); + processDefine.setReleaseState(ReleaseState.OFFLINE); + processDefine.setProjectId(project.getId()); + processDefine.setProcessDefinitionJson(processDefinitionJson); + processDefine.setDescription(desc); + processDefine.setLocations(locations); + processDefine.setConnects(connects); + processDefine.setTimeout(processData.getTimeout()); + processDefine.setTenantId(processData.getTenantId()); + processDefine.setModifyBy(loginUser.getUserName()); + processDefine.setResourceIds(getResourceIds(processData)); + + //custom global params + List globalParamsList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(processData.getGlobalParams())) { + Set userDefParamsSet = new HashSet<>(processData.getGlobalParams()); + globalParamsList = new ArrayList<>(userDefParamsSet); + } + processDefine.setGlobalParamList(globalParamsList); + processDefine.setUpdateTime(now); + processDefine.setFlag(Flag.YES); + if (processDefineMapper.updateById(processDefine) > 0) { + putMsg(result, Status.SUCCESS); + + } else { + putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR); + } + return result; + } + + /** + * verify process definition name unique + * + * @param loginUser login user + * @param projectName project name + * @param name name + * @return true if process definition name not exists, otherwise false + */ + public Map verifyProcessDefinitionName(User loginUser, String projectName, String name) { + + Map result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultEnum = (Status) checkResult.get(Constants.STATUS); + if (resultEnum != Status.SUCCESS) { + return checkResult; + } + ProcessDefinition processDefinition = processDefineMapper.queryByDefineName(project.getId(), name); + if (processDefinition == null) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.PROCESS_INSTANCE_EXIST, name); + } + return result; + } + + /** + * delete process definition by id + * + * @param loginUser login user + * @param projectName project name + * @param processDefinitionId process definition id + * @return delete result code + */ + @Transactional(rollbackFor = RuntimeException.class) + public Map deleteProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId) { + + Map result = new HashMap<>(5); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultEnum = (Status) checkResult.get(Constants.STATUS); + if (resultEnum != Status.SUCCESS) { + return checkResult; + } + + ProcessDefinition processDefinition = processDefineMapper.selectById(processDefinitionId); + + if (processDefinition == null) { + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processDefinitionId); + return result; + } + + // Determine if the login user is the owner of the process definition + if (loginUser.getId() != processDefinition.getUserId() && loginUser.getUserType() != UserType.ADMIN_USER) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + + // check process definition is already online + if (processDefinition.getReleaseState() == ReleaseState.ONLINE) { + putMsg(result, Status.PROCESS_DEFINE_STATE_ONLINE, processDefinitionId); + return result; + } + + // get the timing according to the process definition + List schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId); + if (!schedules.isEmpty() && schedules.size() > 1) { + logger.warn("scheduler num is {},Greater than 1", schedules.size()); + putMsg(result, Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR); + return result; + } else if (schedules.size() == 1) { + Schedule schedule = schedules.get(0); + if (schedule.getReleaseState() == ReleaseState.OFFLINE) { + scheduleMapper.deleteById(schedule.getId()); + } else if (schedule.getReleaseState() == ReleaseState.ONLINE) { + putMsg(result, Status.SCHEDULE_CRON_STATE_ONLINE, schedule.getId()); + return result; + } + } + + int delete = processDefineMapper.deleteById(processDefinitionId); + + if (delete > 0) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR); + } + return result; + } + + /** + * release process definition: online / offline + * + * @param loginUser login user + * @param projectName project name + * @param id process definition id + * @param releaseState release state + * @return release result code + */ + @Transactional(rollbackFor = RuntimeException.class) + public Map releaseProcessDefinition(User loginUser, String projectName, int id, int releaseState) { + HashMap result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultEnum = (Status) checkResult.get(Constants.STATUS); + if (resultEnum != Status.SUCCESS) { + return checkResult; + } + + ReleaseState state = ReleaseState.getEnum(releaseState); + + // check state + if (null == state) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); + return result; + } + + ProcessDefinition processDefinition = processDefineMapper.selectById(id); + + switch (state) { + case ONLINE: + // To check resources whether they are already cancel authorized or deleted + String resourceIds = processDefinition.getResourceIds(); + if (StringUtils.isNotBlank(resourceIds)) { + Integer[] resourceIdArray = Arrays.stream(resourceIds.split(",")).map(Integer::parseInt).toArray(Integer[]::new); + PermissionCheck permissionCheck = new PermissionCheck<>(AuthorizationType.RESOURCE_FILE_ID, processService, resourceIdArray, loginUser.getId(), logger); + try { + permissionCheck.checkPermission(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + putMsg(result, Status.RESOURCE_NOT_EXIST_OR_NO_PERMISSION, RELEASESTATE); + return result; + } + } + + processDefinition.setReleaseState(state); + processDefineMapper.updateById(processDefinition); + break; + case OFFLINE: + processDefinition.setReleaseState(state); + processDefineMapper.updateById(processDefinition); + List scheduleList = scheduleMapper.selectAllByProcessDefineArray( + new int[]{processDefinition.getId()} + ); + + for (Schedule schedule : scheduleList) { + logger.info("set schedule offline, project id: {}, schedule id: {}, process definition id: {}", project.getId(), schedule.getId(), id); + // set status + schedule.setReleaseState(ReleaseState.OFFLINE); + scheduleMapper.updateById(schedule); + SchedulerService.deleteSchedule(project.getId(), schedule.getId()); + } + break; + default: + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); + return result; + } + + putMsg(result, Status.SUCCESS); + return result; + } + + /** + * batch export process definition by ids + * + * @param loginUser + * @param projectName + * @param processDefinitionIds + * @param response + */ + public void batchExportProcessDefinitionByIds(User loginUser, String projectName, String processDefinitionIds, HttpServletResponse response) { + + if (StringUtils.isEmpty(processDefinitionIds)) { + return; + } + + //export project info + Project project = projectMapper.queryByName(projectName); + + //check user access for project + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + + if (resultStatus != Status.SUCCESS) { + return; + } + + List processDefinitionList = + getProcessDefinitionList(processDefinitionIds); + + if (CollectionUtils.isNotEmpty(processDefinitionList)) { + downloadProcessDefinitionFile(response, processDefinitionList); + } + } + + /** + * get process definition list by ids + * + * @param processDefinitionIds + * @return + */ + private List getProcessDefinitionList(String processDefinitionIds) { + List processDefinitionList = new ArrayList<>(); + String[] processDefinitionIdArray = processDefinitionIds.split(","); + for (String strProcessDefinitionId : processDefinitionIdArray) { + //get workflow info + int processDefinitionId = Integer.parseInt(strProcessDefinitionId); + ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefinitionId); + if (null != processDefinition) { + processDefinitionList.add(exportProcessMetaData(processDefinitionId, processDefinition)); + } + } + + return processDefinitionList; + } + + /** + * download the process definition file + * + * @param response + * @param processDefinitionList + */ + private void downloadProcessDefinitionFile(HttpServletResponse response, List processDefinitionList) { + response.setContentType(MediaType.APPLICATION_JSON_UTF8_VALUE); + BufferedOutputStream buff = null; + ServletOutputStream out = null; + try { + out = response.getOutputStream(); + buff = new BufferedOutputStream(out); + buff.write(JSONUtils.toJsonString(processDefinitionList).getBytes(StandardCharsets.UTF_8)); + buff.flush(); + buff.close(); + } catch (IOException e) { + logger.warn("export process fail", e); + } finally { + if (null != buff) { + try { + buff.close(); + } catch (Exception e) { + logger.warn("export process buffer not close", e); + } + } + if (null != out) { + try { + out.close(); + } catch (Exception e) { + logger.warn("export process output stream not close", e); + } + } + } + } + + /** + * get export process metadata string + * + * @param processDefinitionId process definition id + * @param processDefinition process definition + * @return export process metadata string + */ + private String exportProcessMetaDataStr(Integer processDefinitionId, ProcessDefinition processDefinition) { + //create workflow json file + return JSONUtils.toJsonString(exportProcessMetaData(processDefinitionId, processDefinition)); + } + + /** + * get export process metadata string + * + * @param processDefinitionId process definition id + * @param processDefinition process definition + * @return export process metadata string + */ + public ProcessMeta exportProcessMetaData(Integer processDefinitionId, ProcessDefinition processDefinition) { + //correct task param which has data source or dependent param + String correctProcessDefinitionJson = addExportTaskNodeSpecialParam(processDefinition.getProcessDefinitionJson()); + processDefinition.setProcessDefinitionJson(correctProcessDefinitionJson); + + //export process metadata + ProcessMeta exportProcessMeta = new ProcessMeta(); + exportProcessMeta.setProjectName(processDefinition.getProjectName()); + exportProcessMeta.setProcessDefinitionName(processDefinition.getName()); + exportProcessMeta.setProcessDefinitionJson(processDefinition.getProcessDefinitionJson()); + exportProcessMeta.setProcessDefinitionLocations(processDefinition.getLocations()); + exportProcessMeta.setProcessDefinitionConnects(processDefinition.getConnects()); + + //schedule info + List schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId); + if (!schedules.isEmpty()) { + Schedule schedule = schedules.get(0); + exportProcessMeta.setScheduleWarningType(schedule.getWarningType().toString()); + exportProcessMeta.setScheduleWarningGroupId(schedule.getWarningGroupId()); + exportProcessMeta.setScheduleStartTime(DateUtils.dateToString(schedule.getStartTime())); + exportProcessMeta.setScheduleEndTime(DateUtils.dateToString(schedule.getEndTime())); + exportProcessMeta.setScheduleCrontab(schedule.getCrontab()); + exportProcessMeta.setScheduleFailureStrategy(String.valueOf(schedule.getFailureStrategy())); + exportProcessMeta.setScheduleReleaseState(String.valueOf(ReleaseState.OFFLINE)); + exportProcessMeta.setScheduleProcessInstancePriority(String.valueOf(schedule.getProcessInstancePriority())); + exportProcessMeta.setScheduleWorkerGroupName(schedule.getWorkerGroup()); + } + //create workflow json file + return exportProcessMeta; + } + + /** + * correct task param which has datasource or dependent + * + * @param processDefinitionJson processDefinitionJson + * @return correct processDefinitionJson + */ + private String addExportTaskNodeSpecialParam(String processDefinitionJson) { + ObjectNode jsonObject = JSONUtils.parseObject(processDefinitionJson); + ArrayNode jsonArray = (ArrayNode) jsonObject.path(TASKS); + + for (int i = 0; i < jsonArray.size(); i++) { + JsonNode taskNode = jsonArray.path(i); + if (StringUtils.isNotEmpty(taskNode.path("type").asText())) { + String taskType = taskNode.path("type").asText(); + + ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); + if (null != addTaskParam) { + addTaskParam.addExportSpecialParam(taskNode); + } + } + } + jsonObject.set(TASKS, jsonArray); + return jsonObject.toString(); + } + + /** + * check task if has sub process + * + * @param taskType task type + * @return if task has sub process return true else false + */ + private boolean checkTaskHasSubProcess(String taskType) { + return taskType.equals(TaskType.SUB_PROCESS.name()); + } + + /** + * import process definition + * + * @param loginUser login user + * @param file process metadata json file + * @param currentProjectName current project name + * @return import process + */ + @Transactional(rollbackFor = RuntimeException.class) + public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { + Map result = new HashMap<>(5); + String processMetaJson = FileUtils.file2String(file); + List processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class); + + //check file content + if (CollectionUtils.isEmpty(processMetaList)) { + putMsg(result, Status.DATA_IS_NULL, "fileContent"); + return result; + } + + for (ProcessMeta processMeta : processMetaList) { + + if (!checkAndImportProcessDefinition(loginUser, currentProjectName, result, processMeta)) { + return result; + } + } + + return result; + } + + /** + * check and import process definition + * + * @param loginUser + * @param currentProjectName + * @param result + * @param processMeta + * @return + */ + private boolean checkAndImportProcessDefinition(User loginUser, String currentProjectName, Map result, ProcessMeta processMeta) { + + if (!checkImportanceParams(processMeta, result)) { + return false; + } + + //deal with process name + String processDefinitionName = processMeta.getProcessDefinitionName(); + //use currentProjectName to query + Project targetProject = projectMapper.queryByName(currentProjectName); + if (null != targetProject) { + processDefinitionName = recursionProcessDefinitionName(targetProject.getId(), + processDefinitionName, 1); + } + + //unique check + Map checkResult = verifyProcessDefinitionName(loginUser, currentProjectName, processDefinitionName); + Status status = (Status) checkResult.get(Constants.STATUS); + if (Status.SUCCESS.equals(status)) { + putMsg(result, Status.SUCCESS); + } else { + result.putAll(checkResult); + return false; + } + + // get create process result + Map createProcessResult = + getCreateProcessResult(loginUser, + currentProjectName, + result, + processMeta, + processDefinitionName, + addImportTaskNodeParam(loginUser, processMeta.getProcessDefinitionJson(), targetProject)); + + if (createProcessResult == null) { + return false; + } + + //create process definition + Integer processDefinitionId = + Objects.isNull(createProcessResult.get(PROCESSDEFINITIONID)) ? + null : Integer.parseInt(createProcessResult.get(PROCESSDEFINITIONID).toString()); + + //scheduler param + return getImportProcessScheduleResult(loginUser, + currentProjectName, + result, + processMeta, + processDefinitionName, + processDefinitionId); + + } + + /** + * get create process result + * + * @param loginUser + * @param currentProjectName + * @param result + * @param processMeta + * @param processDefinitionName + * @param importProcessParam + * @return + */ + private Map getCreateProcessResult(User loginUser, + String currentProjectName, + Map result, + ProcessMeta processMeta, + String processDefinitionName, + String importProcessParam) { + Map createProcessResult = null; + try { + createProcessResult = createProcessDefinition(loginUser + , currentProjectName, + processDefinitionName + "_import_" + System.currentTimeMillis(), + importProcessParam, + processMeta.getProcessDefinitionDescription(), + processMeta.getProcessDefinitionLocations(), + processMeta.getProcessDefinitionConnects()); + putMsg(result, Status.SUCCESS); + } catch (JsonProcessingException e) { + logger.error("import process meta json data: {}", e.getMessage(), e); + putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); + } + + return createProcessResult; + } + + /** + * get import process schedule result + * + * @param loginUser + * @param currentProjectName + * @param result + * @param processMeta + * @param processDefinitionName + * @param processDefinitionId + * @return + */ + private boolean getImportProcessScheduleResult(User loginUser, + String currentProjectName, + Map result, + ProcessMeta processMeta, + String processDefinitionName, + Integer processDefinitionId) { + if (null != processMeta.getScheduleCrontab() && null != processDefinitionId) { + int scheduleInsert = importProcessSchedule(loginUser, + currentProjectName, + processMeta, + processDefinitionName, + processDefinitionId); + + if (0 == scheduleInsert) { + putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); + return false; + } + } + return true; + } + + /** + * check importance params + * + * @param processMeta + * @param result + * @return + */ + private boolean checkImportanceParams(ProcessMeta processMeta, Map result) { + if (StringUtils.isEmpty(processMeta.getProjectName())) { + putMsg(result, Status.DATA_IS_NULL, "projectName"); + return false; + } + if (StringUtils.isEmpty(processMeta.getProcessDefinitionName())) { + putMsg(result, Status.DATA_IS_NULL, "processDefinitionName"); + return false; + } + if (StringUtils.isEmpty(processMeta.getProcessDefinitionJson())) { + putMsg(result, Status.DATA_IS_NULL, "processDefinitionJson"); + return false; + } + + return true; + } + + /** + * import process add special task param + * + * @param loginUser login user + * @param processDefinitionJson process definition json + * @param targetProject target project + * @return import process param + */ + private String addImportTaskNodeParam(User loginUser, String processDefinitionJson, Project targetProject) { + ObjectNode jsonObject = JSONUtils.parseObject(processDefinitionJson); + ArrayNode jsonArray = (ArrayNode) jsonObject.get(TASKS); + //add sql and dependent param + for (int i = 0; i < jsonArray.size(); i++) { + JsonNode taskNode = jsonArray.path(i); + String taskType = taskNode.path("type").asText(); + ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); + if (null != addTaskParam) { + addTaskParam.addImportSpecialParam(taskNode); + } + } + + //recursive sub-process parameter correction map key for old process id value for new process id + Map subProcessIdMap = new HashMap<>(20); + + List subProcessList = StreamUtils.asStream(jsonArray.elements()) + .filter(elem -> checkTaskHasSubProcess(JSONUtils.parseObject(elem.toString()).path("type").asText())) + .collect(Collectors.toList()); + + if (CollectionUtils.isNotEmpty(subProcessList)) { + importSubProcess(loginUser, targetProject, jsonArray, subProcessIdMap); + } + + jsonObject.set(TASKS, jsonArray); + return jsonObject.toString(); + } + + /** + * import process schedule + * + * @param loginUser login user + * @param currentProjectName current project name + * @param processMeta process meta data + * @param processDefinitionName process definition name + * @param processDefinitionId process definition id + * @return insert schedule flag + */ + private int importProcessSchedule(User loginUser, String currentProjectName, ProcessMeta processMeta, + String processDefinitionName, Integer processDefinitionId) { + Date now = new Date(); + Schedule scheduleObj = new Schedule(); + scheduleObj.setProjectName(currentProjectName); + scheduleObj.setProcessDefinitionId(processDefinitionId); + scheduleObj.setProcessDefinitionName(processDefinitionName); + scheduleObj.setCreateTime(now); + scheduleObj.setUpdateTime(now); + scheduleObj.setUserId(loginUser.getId()); + scheduleObj.setUserName(loginUser.getUserName()); + + scheduleObj.setCrontab(processMeta.getScheduleCrontab()); + + if (null != processMeta.getScheduleStartTime()) { + scheduleObj.setStartTime(DateUtils.stringToDate(processMeta.getScheduleStartTime())); + } + if (null != processMeta.getScheduleEndTime()) { + scheduleObj.setEndTime(DateUtils.stringToDate(processMeta.getScheduleEndTime())); + } + if (null != processMeta.getScheduleWarningType()) { + scheduleObj.setWarningType(WarningType.valueOf(processMeta.getScheduleWarningType())); + } + if (null != processMeta.getScheduleWarningGroupId()) { + scheduleObj.setWarningGroupId(processMeta.getScheduleWarningGroupId()); + } + if (null != processMeta.getScheduleFailureStrategy()) { + scheduleObj.setFailureStrategy(FailureStrategy.valueOf(processMeta.getScheduleFailureStrategy())); + } + if (null != processMeta.getScheduleReleaseState()) { + scheduleObj.setReleaseState(ReleaseState.valueOf(processMeta.getScheduleReleaseState())); + } + if (null != processMeta.getScheduleProcessInstancePriority()) { + scheduleObj.setProcessInstancePriority(Priority.valueOf(processMeta.getScheduleProcessInstancePriority())); + } + + if (null != processMeta.getScheduleWorkerGroupName()) { + scheduleObj.setWorkerGroup(processMeta.getScheduleWorkerGroupName()); + } + + return scheduleMapper.insert(scheduleObj); + } + + /** + * check import process has sub process + * recursion create sub process + * + * @param loginUser login user + * @param targetProject target project + * @param jsonArray process task array + * @param subProcessIdMap correct sub process id map + */ + private void importSubProcess(User loginUser, Project targetProject, ArrayNode jsonArray, Map subProcessIdMap) { + for (int i = 0; i < jsonArray.size(); i++) { + ObjectNode taskNode = (ObjectNode) jsonArray.path(i); + String taskType = taskNode.path("type").asText(); + + if (!checkTaskHasSubProcess(taskType)) { + continue; + } + //get sub process info + ObjectNode subParams = (ObjectNode) taskNode.path("params"); + Integer subProcessId = subParams.path(PROCESSDEFINITIONID).asInt(); + ProcessDefinition subProcess = processDefineMapper.queryByDefineId(subProcessId); + //check is sub process exist in db + if (null == subProcess) { + continue; + } + String subProcessJson = subProcess.getProcessDefinitionJson(); + //check current project has sub process + ProcessDefinition currentProjectSubProcess = processDefineMapper.queryByDefineName(targetProject.getId(), subProcess.getName()); + + if (null == currentProjectSubProcess) { + ArrayNode subJsonArray = (ArrayNode) JSONUtils.parseObject(subProcess.getProcessDefinitionJson()).get(TASKS); + + List subProcessList = StreamUtils.asStream(subJsonArray.elements()) + .filter(item -> checkTaskHasSubProcess(JSONUtils.parseObject(item.toString()).path("type").asText())) + .collect(Collectors.toList()); + + if (CollectionUtils.isNotEmpty(subProcessList)) { + importSubProcess(loginUser, targetProject, subJsonArray, subProcessIdMap); + //sub process processId correct + if (!subProcessIdMap.isEmpty()) { + + for (Map.Entry entry : subProcessIdMap.entrySet()) { + String oldSubProcessId = "\"processDefinitionId\":" + entry.getKey(); + String newSubProcessId = "\"processDefinitionId\":" + entry.getValue(); + subProcessJson = subProcessJson.replaceAll(oldSubProcessId, newSubProcessId); + } + + subProcessIdMap.clear(); + } + } + + //if sub-process recursion + Date now = new Date(); + //create sub process in target project + ProcessDefinition processDefine = new ProcessDefinition(); + processDefine.setName(subProcess.getName()); + processDefine.setVersion(subProcess.getVersion()); + processDefine.setReleaseState(subProcess.getReleaseState()); + processDefine.setProjectId(targetProject.getId()); + processDefine.setUserId(loginUser.getId()); + processDefine.setProcessDefinitionJson(subProcessJson); + processDefine.setDescription(subProcess.getDescription()); + processDefine.setLocations(subProcess.getLocations()); + processDefine.setConnects(subProcess.getConnects()); + processDefine.setTimeout(subProcess.getTimeout()); + processDefine.setTenantId(subProcess.getTenantId()); + processDefine.setGlobalParams(subProcess.getGlobalParams()); + processDefine.setCreateTime(now); + processDefine.setUpdateTime(now); + processDefine.setFlag(subProcess.getFlag()); + processDefine.setReceivers(subProcess.getReceivers()); + processDefine.setReceiversCc(subProcess.getReceiversCc()); + processDefineMapper.insert(processDefine); + + logger.info("create sub process, project: {}, process name: {}", targetProject.getName(), processDefine.getName()); + + //modify task node + ProcessDefinition newSubProcessDefine = processDefineMapper.queryByDefineName(processDefine.getProjectId(), processDefine.getName()); + + if (null != newSubProcessDefine) { + subProcessIdMap.put(subProcessId, newSubProcessDefine.getId()); + subParams.put(PROCESSDEFINITIONID, newSubProcessDefine.getId()); + taskNode.set("params", subParams); + } + } + } + } + + + /** + * check the process definition node meets the specifications + * + * @param processData process data + * @param processDefinitionJson process definition json + * @return check result code + */ + public Map checkProcessNodeList(ProcessData processData, String processDefinitionJson) { + + Map result = new HashMap<>(5); + try { + if (processData == null) { + logger.error("process data is null"); + putMsg(result, Status.DATA_IS_NOT_VALID, processDefinitionJson); + return result; + } + + // Check whether the task node is normal + List taskNodes = processData.getTasks(); + + if (taskNodes == null) { + logger.error("process node info is empty"); + putMsg(result, Status.DATA_IS_NULL, processDefinitionJson); + return result; + } + + // check has cycle + if (graphHasCycle(taskNodes)) { + logger.error("process DAG has cycle"); + putMsg(result, Status.PROCESS_NODE_HAS_CYCLE); + return result; + } + + // check whether the process definition json is normal + for (TaskNode taskNode : taskNodes) { + if (!CheckUtils.checkTaskNodeParameters(taskNode.getParams(), taskNode.getType())) { + logger.error("task node {} parameter invalid", taskNode.getName()); + putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskNode.getName()); + return result; + } + + // check extra params + CheckUtils.checkOtherParams(taskNode.getExtras()); + } + putMsg(result, Status.SUCCESS); + } catch (Exception e) { + result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); + result.put(Constants.MSG, e.getMessage()); + } + return result; + } + + /** + * get task node details based on process definition + * + * @param defineId define id + * @return task node list + */ + public Map getTaskNodeListByDefinitionId(Integer defineId) { + Map result = new HashMap<>(); + + ProcessDefinition processDefinition = processDefineMapper.selectById(defineId); + if (processDefinition == null) { + logger.info("process define not exists"); + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineId); + return result; + } + + + String processDefinitionJson = processDefinition.getProcessDefinitionJson(); + + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + + //process data check + if (null == processData) { + logger.error("process data is null"); + putMsg(result, Status.DATA_IS_NOT_VALID, processDefinitionJson); + return result; + } + + List taskNodeList = (processData.getTasks() == null) ? new ArrayList<>() : processData.getTasks(); + + result.put(Constants.DATA_LIST, taskNodeList); + putMsg(result, Status.SUCCESS); + + return result; + + } + + /** + * get task node details based on process definition + * + * @param defineIdList define id list + * @return task node list + */ + public Map getTaskNodeListByDefinitionIdList(String defineIdList) { + Map result = new HashMap<>(); + + Map> taskNodeMap = new HashMap<>(); + String[] idList = defineIdList.split(","); + List idIntList = new ArrayList<>(); + for (String definitionId : idList) { + idIntList.add(Integer.parseInt(definitionId)); + } + Integer[] idArray = idIntList.toArray(new Integer[idIntList.size()]); + List processDefinitionList = processDefineMapper.queryDefinitionListByIdList(idArray); + if (CollectionUtils.isEmpty(processDefinitionList)) { + logger.info("process definition not exists"); + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineIdList); + return result; + } + + for (ProcessDefinition processDefinition : processDefinitionList) { + String processDefinitionJson = processDefinition.getProcessDefinitionJson(); + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + List taskNodeList = (processData.getTasks() == null) ? new ArrayList<>() : processData.getTasks(); + taskNodeMap.put(processDefinition.getId(), taskNodeList); + } + + result.put(Constants.DATA_LIST, taskNodeMap); + putMsg(result, Status.SUCCESS); + + return result; + + } + + + /** + * query process definition all by project id + * + * @param projectId project id + * @return process definitions in the project + */ + public Map queryProcessDefinitionAllByProjectId(Integer projectId) { + + HashMap result = new HashMap<>(5); + + List resourceList = processDefineMapper.queryAllDefinitionList(projectId); + result.put(Constants.DATA_LIST, resourceList); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * Encapsulates the TreeView structure + * + * @param processId process definition id + * @param limit limit + * @return tree view json data + * @throws Exception exception + */ + public Map viewTree(Integer processId, Integer limit) throws Exception { + Map result = new HashMap<>(); + + ProcessDefinition processDefinition = processDefineMapper.selectById(processId); + if (null == processDefinition) { + logger.info("process define not exists"); + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processDefinition); + return result; + } + DAG dag = genDagGraph(processDefinition); + /** + * nodes that is running + */ + Map> runningNodeMap = new ConcurrentHashMap<>(); + + /** + * nodes that is waiting torun + */ + Map> waitingRunningNodeMap = new ConcurrentHashMap<>(); + + /** + * List of process instances + */ + List processInstanceList = processInstanceMapper.queryByProcessDefineId(processId, limit); + + for (ProcessInstance processInstance : processInstanceList) { + processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime())); + } + + if (limit > processInstanceList.size()) { + limit = processInstanceList.size(); + } + + TreeViewDto parentTreeViewDto = new TreeViewDto(); + parentTreeViewDto.setName("DAG"); + parentTreeViewDto.setType(""); + // Specify the process definition, because it is a TreeView for a process definition + + for (int i = limit - 1; i >= 0; i--) { + ProcessInstance processInstance = processInstanceList.get(i); + + Date endTime = processInstance.getEndTime() == null ? new Date() : processInstance.getEndTime(); + parentTreeViewDto.getInstances().add(new Instance(processInstance.getId(), processInstance.getName(), "", processInstance.getState().toString() + , processInstance.getStartTime(), endTime, processInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - processInstance.getStartTime().getTime()))); + } + + List parentTreeViewDtoList = new ArrayList<>(); + parentTreeViewDtoList.add(parentTreeViewDto); + // Here is the encapsulation task instance + for (String startNode : dag.getBeginNode()) { + runningNodeMap.put(startNode, parentTreeViewDtoList); + } + + while (Stopper.isRunning()) { + Set postNodeList = null; + Iterator>> iter = runningNodeMap.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> en = iter.next(); + String nodeName = en.getKey(); + parentTreeViewDtoList = en.getValue(); + + TreeViewDto treeViewDto = new TreeViewDto(); + treeViewDto.setName(nodeName); + TaskNode taskNode = dag.getNode(nodeName); + treeViewDto.setType(taskNode.getType()); + + + //set treeViewDto instances + for (int i = limit - 1; i >= 0; i--) { + ProcessInstance processInstance = processInstanceList.get(i); + TaskInstance taskInstance = taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), nodeName); + if (taskInstance == null) { + treeViewDto.getInstances().add(new Instance(-1, "not running", "null")); + } else { + Date startTime = taskInstance.getStartTime() == null ? new Date() : taskInstance.getStartTime(); + Date endTime = taskInstance.getEndTime() == null ? new Date() : taskInstance.getEndTime(); + + int subProcessId = 0; + /** + * if process is sub process, the return sub id, or sub id=0 + */ + if (taskInstance.getTaskType().equals(TaskType.SUB_PROCESS.name())) { + String taskJson = taskInstance.getTaskJson(); + taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); + subProcessId = Integer.parseInt(JSONUtils.parseObject( + taskNode.getParams()).path(CMDPARAM_SUB_PROCESS_DEFINE_ID).asText()); + } + treeViewDto.getInstances().add(new Instance(taskInstance.getId(), taskInstance.getName(), taskInstance.getTaskType(), taskInstance.getState().toString() + , taskInstance.getStartTime(), taskInstance.getEndTime(), taskInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - startTime.getTime()), subProcessId)); + } + } + for (TreeViewDto pTreeViewDto : parentTreeViewDtoList) { + pTreeViewDto.getChildren().add(treeViewDto); + } + postNodeList = dag.getSubsequentNodes(nodeName); + if (CollectionUtils.isNotEmpty(postNodeList)) { + for (String nextNodeName : postNodeList) { + List treeViewDtoList = waitingRunningNodeMap.get(nextNodeName); + if (CollectionUtils.isNotEmpty(treeViewDtoList)) { + treeViewDtoList.add(treeViewDto); + waitingRunningNodeMap.put(nextNodeName, treeViewDtoList); + } else { + treeViewDtoList = new ArrayList<>(); + treeViewDtoList.add(treeViewDto); + waitingRunningNodeMap.put(nextNodeName, treeViewDtoList); + } + } + } + runningNodeMap.remove(nodeName); + } + if (waitingRunningNodeMap == null || waitingRunningNodeMap.size() == 0) { + break; + } else { + runningNodeMap.putAll(waitingRunningNodeMap); + waitingRunningNodeMap.clear(); + } + } + result.put(Constants.DATA_LIST, parentTreeViewDto); + result.put(Constants.STATUS, Status.SUCCESS); + result.put(Constants.MSG, Status.SUCCESS.getMsg()); + return result; + } + + + /** + * Generate the DAG Graph based on the process definition id + * + * @param processDefinition process definition + * @return dag graph + */ + private DAG genDagGraph(ProcessDefinition processDefinition) { + + String processDefinitionJson = processDefinition.getProcessDefinitionJson(); + + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + + //check process data + if (null != processData) { + List taskNodeList = processData.getTasks(); + processDefinition.setGlobalParamList(processData.getGlobalParams()); + ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); + + // Generate concrete Dag to be executed + return DagHelper.buildDagGraph(processDag); + } + + return new DAG<>(); + } + + + /** + * whether the graph has a ring + * + * @param taskNodeResponseList task node response list + * @return if graph has cycle flag + */ + private boolean graphHasCycle(List taskNodeResponseList) { + DAG graph = new DAG<>(); + + // Fill the vertices + for (TaskNode taskNodeResponse : taskNodeResponseList) { + graph.addNode(taskNodeResponse.getName(), taskNodeResponse); + } + + // Fill edge relations + for (TaskNode taskNodeResponse : taskNodeResponseList) { + taskNodeResponse.getPreTasks(); + List preTasks = JSONUtils.toList(taskNodeResponse.getPreTasks(), String.class); + if (CollectionUtils.isNotEmpty(preTasks)) { + for (String preTask : preTasks) { + if (!graph.addEdge(preTask, taskNodeResponse.getName())) { + return true; + } + } + } + } + + return graph.hasCycle(); + } + + private String recursionProcessDefinitionName(Integer projectId, String processDefinitionName, int num) { + ProcessDefinition processDefinition = processDefineMapper.queryByDefineName(projectId, processDefinitionName); + if (processDefinition != null) { + if (num > 1) { + String str = processDefinitionName.substring(0, processDefinitionName.length() - 3); + processDefinitionName = str + "(" + num + ")"; + } else { + processDefinitionName = processDefinition.getName() + "(" + num + ")"; + } + } else { + return processDefinitionName; + } + return recursionProcessDefinitionName(projectId, processDefinitionName, num + 1); + } + +} + diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java index 8c0d04c6c6..c5f98630a2 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; +import org.apache.dolphinscheduler.api.service.impl.ProcessDefinitionServiceImpl; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; @@ -55,7 +56,7 @@ public class ProcessDefinitionControllerTest{ private ProcessDefinitionController processDefinitionController; @Mock - private ProcessDefinitionService processDefinitionService; + private ProcessDefinitionServiceImpl processDefinitionService; protected User user; @@ -342,9 +343,7 @@ public class ProcessDefinitionControllerTest{ String processDefinitionIds = "1,2"; String projectName = "test"; HttpServletResponse response = new MockHttpServletResponse(); - ProcessDefinitionService service = new ProcessDefinitionService(); - ProcessDefinitionService spy = Mockito.spy(service); - Mockito.doNothing().when(spy).batchExportProcessDefinitionByIds(user, projectName, processDefinitionIds, response); + Mockito.doNothing().when(this.processDefinitionService).batchExportProcessDefinitionByIds(user, projectName, processDefinitionIds, response); processDefinitionController.batchExportProcessDefinitionByIds(user, projectName, processDefinitionIds, response); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java deleted file mode 100644 index bb6e3882fe..0000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.service; - -import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.model.TaskNodeRelation; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; - -@RunWith(MockitoJUnitRunner.class) -public class BaseDAGServiceTest { - - @Test - public void testProcessInstance2DAG(){ - - ProcessInstance processInstance = new ProcessInstance(); - processInstance.setProcessInstanceJson("{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-61567\"," + - "\"name\":\"开始\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo '1'\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[]},{\"type\":\"SHELL\",\"id\":\"tasks-6-3ug5ej\",\"name\":\"结束\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo '1'\"},\"description\":\"\"," + - "\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[\"开始\"]}],\"tenantId\":-1,\"timeout\":0}"); - - DAG relationDAG = BaseDAGService.processInstance2DAG(processInstance); - - Assert.assertTrue(relationDAG.containsNode("开始")); - - } -} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 8db667e28b..168fe48fc3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -16,21 +16,45 @@ */ package org.apache.dolphinscheduler.api.service; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.dolphinscheduler.api.dto.ProcessMeta; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.ProcessDefinitionServiceImpl; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.ProcessData; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.Schedule; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.http.entity.ContentType; -import org.json.JSONException; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -38,23 +62,14 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import org.skyscreamer.jsonassert.JSONAssert; -import org.springframework.boot.test.context.SpringBootTest; import org.springframework.mock.web.MockMultipartFile; import org.springframework.web.multipart.MultipartFile; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.text.MessageFormat; -import java.util.*; - @RunWith(MockitoJUnitRunner.Silent.class) -@SpringBootTest(classes = ApiApplicationServer.class) public class ProcessDefinitionServiceTest { @InjectMocks - ProcessDefinitionService processDefinitionService; + ProcessDefinitionServiceImpl processDefinitionService; @Mock private DataSourceMapper dataSourceMapper; @@ -502,157 +517,6 @@ public class ProcessDefinitionServiceTest { Assert.assertEquals(Status.SUCCESS, taskNotNuLLRes.get(Constants.STATUS)); } - /** - * add datasource param and dependent when export process - * @throws JSONException - */ - @Test - public void testAddTaskNodeSpecialParam() throws JSONException { - - Mockito.when(dataSourceMapper.selectById(1)).thenReturn(getDataSource()); - Mockito.when(processDefineMapper.queryByDefineId(2)).thenReturn(getProcessDefinition()); - - String corSqlDependentJson = processDefinitionService.addExportTaskNodeSpecialParam(sqlDependentJson); - - JSONAssert.assertEquals(sqlDependentJson,corSqlDependentJson,false); - - } - - @Test - public void testExportProcessMetaDataStr() { - Mockito.when(scheduleMapper.queryByProcessDefinitionId(46)).thenReturn(getSchedulerList()); - ProcessDefinition processDefinition = getProcessDefinition(); - processDefinition.setProcessDefinitionJson(sqlDependentJson); - - String exportProcessMetaDataStr = processDefinitionService.exportProcessMetaDataStr(46, processDefinition); - Assert.assertNotEquals(sqlDependentJson,exportProcessMetaDataStr); - } - - @Test - public void testAddExportTaskNodeSpecialParam() throws JSONException { - String shellData = shellJson; - - String resultStr = processDefinitionService.addExportTaskNodeSpecialParam(shellData); - JSONAssert.assertEquals(shellJson, resultStr, false); - } - - @Test - public void testImportProcessSchedule() { - User loginUser = new User(); - loginUser.setId(1); - loginUser.setUserType(UserType.GENERAL_USER); - - String currentProjectName = "test"; - String processDefinitionName = "test_process"; - Integer processDefinitionId = 1; - Schedule schedule = getSchedule(); - - ProcessMeta processMeta = getProcessMeta(); - - int insertFlag = processDefinitionService.importProcessSchedule(loginUser, currentProjectName, processMeta, - processDefinitionName, processDefinitionId); - Assert.assertEquals(0, insertFlag); - - ProcessMeta processMetaCron = new ProcessMeta(); - processMetaCron.setScheduleCrontab(schedule.getCrontab()); - - int insertFlagCron = processDefinitionService.importProcessSchedule(loginUser, currentProjectName, processMetaCron, - processDefinitionName, processDefinitionId); - Assert.assertEquals(0, insertFlagCron); - - WorkerGroup workerGroup = new WorkerGroup(); - workerGroup.setName("ds-test-workergroup"); - List workerGroups = new ArrayList<>(); - workerGroups.add(workerGroup); - - processMetaCron.setScheduleWorkerGroupName("ds-test"); - int insertFlagWorker = processDefinitionService.importProcessSchedule(loginUser, currentProjectName, processMetaCron, - processDefinitionName, processDefinitionId); - Assert.assertEquals(0, insertFlagWorker); - - int workerNullFlag = processDefinitionService.importProcessSchedule(loginUser, currentProjectName, processMetaCron, - processDefinitionName, processDefinitionId); - Assert.assertEquals(0, workerNullFlag); - - - } - - /** - * import sub process test - */ - @Test - public void testImportSubProcess() { - - User loginUser = new User(); - loginUser.setId(1); - loginUser.setUserType(UserType.ADMIN_USER); - - Project testProject = getProject("test"); - - //Recursive subprocess sub2 process in sub1 process and sub1process in top process - String topProcessJson = "{\"globalParams\":[]," + - "\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-38634\",\"name\":\"shell1\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," + - "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," + - "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}," + - "{\"type\":\"SUB_PROCESS\",\"id\":\"tasks-44207\",\"name\":\"shell-4\"," + - "\"params\":{\"processDefinitionId\":39},\"description\":\"\",\"runFlag\":\"NORMAL\"," + - "\"dependence\":{},\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," + - "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," + - "\"preTasks\":[\"shell1\"]}],\"tenantId\":1,\"timeout\":0}"; - - String sub1ProcessJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-84090\"," + - "\"name\":\"shell-4\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-4\\\"\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," + - "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," + - "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]},{\"type\":\"SUB_PROCESS\"," + - "\"id\":\"tasks-87364\",\"name\":\"shell-5\"," + - "\"params\":{\"processDefinitionId\":46},\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{}," + - "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[\"shell-4\"]}],\"tenantId\":1,\"timeout\":0}"; - - String sub2ProcessJson = "{\"globalParams\":[]," + - "\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-52423\",\"name\":\"shell-5\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo \\\"shell-5\\\"\"},\"description\":\"\"," + - "\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," + - "\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; - - - ObjectNode jsonObject = JSONUtils.parseObject(topProcessJson); - ArrayNode jsonArray = (ArrayNode) jsonObject.path("tasks"); - - String originSubJson = jsonArray.toString(); - - Map subProcessIdMap = new HashMap<>(20); - - ProcessDefinition shellDefinition1 = new ProcessDefinition(); - shellDefinition1.setId(39); - shellDefinition1.setName("shell-4"); - shellDefinition1.setProjectId(2); - shellDefinition1.setProcessDefinitionJson(sub1ProcessJson); - - ProcessDefinition shellDefinition2 = new ProcessDefinition(); - shellDefinition2.setId(46); - shellDefinition2.setName("shell-5"); - shellDefinition2.setProjectId(2); - shellDefinition2.setProcessDefinitionJson(sub2ProcessJson); - - Mockito.when(processDefineMapper.queryByDefineId(39)).thenReturn(shellDefinition1); - Mockito.when(processDefineMapper.queryByDefineId(46)).thenReturn(shellDefinition2); - Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "shell-5")).thenReturn(null); - Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "shell-4")).thenReturn(null); - Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "testProject")).thenReturn(shellDefinition2); - - processDefinitionService.importSubProcess(loginUser,testProject, jsonArray, subProcessIdMap); - - String correctSubJson = jsonArray.toString(); - - Assert.assertEquals(originSubJson, correctSubJson); - - } - @Test public void testImportProcessDefinitionById() throws IOException { @@ -731,34 +595,6 @@ public class ProcessDefinitionServiceTest { } - /** - * check import process metadata - * @param file file - * @param loginUser login user - * @param currentProjectName current project name - * @param processMetaJson process meta json - * @throws IOException IO exception - */ - private void improssProcessCheckData(File file, User loginUser, String currentProjectName, String processMetaJson) throws IOException { - //check null - FileUtils.writeStringToFile(new File("/tmp/task.json"),processMetaJson); - - File fileEmpty = new File("/tmp/task.json"); - - FileInputStream fileEmptyInputStream = new FileInputStream("/tmp/task.json"); - - MultipartFile multiFileEmpty = new MockMultipartFile(fileEmpty.getName(), fileEmpty.getName(), - ContentType.APPLICATION_OCTET_STREAM.toString(), fileEmptyInputStream); - - Map resEmptyProcess = processDefinitionService.importProcessDefinition(loginUser, multiFileEmpty, currentProjectName); - - Assert.assertEquals(Status.DATA_IS_NULL, resEmptyProcess.get(Constants.STATUS)); - - boolean deleteFlag = file.delete(); - - Assert.assertTrue(deleteFlag); - } - @Test public void testUpdateProcessDefinition () { User loginUser = new User(); From db39f1f38574a2da542ac44e522aeb0a8de1de51 Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Mon, 10 Aug 2020 15:03:27 +0800 Subject: [PATCH 13/24] [Improvement][style] Improve checkstyle import order and code style importer (#3368) * [Improvement][style] Add code import style automatic formatter * Update checkstyle.xml * Update checkstyle.xml * [Improvement][api] Improve the code style and checkstyle of ds * Update checkstyle.xml --- style/checkstyle.xml | 63 +++++++++++++++++++----------- style/intellij-java-code-style.xml | 16 ++++++++ 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/style/checkstyle.xml b/style/checkstyle.xml index 54188c6834..2a74b99a27 100644 --- a/style/checkstyle.xml +++ b/style/checkstyle.xml @@ -186,19 +186,51 @@ - + - - + + - + - - + + + + + + - - + + + + + + + + + + @@ -233,11 +265,6 @@ - - - - - @@ -249,16 +276,6 @@ - - - - - - - - - diff --git a/style/intellij-java-code-style.xml b/style/intellij-java-code-style.xml index ae333b1c3d..8e0cd2f720 100644 --- a/style/intellij-java-code-style.xml +++ b/style/intellij-java-code-style.xml @@ -37,8 +37,24 @@ - + - + - + + + + @@ -269,7 +272,9 @@ - + + + From ad89f433f16d65ddcab3186baa84e14762d03e52 Mon Sep 17 00:00:00 2001 From: dengc367 Date: Mon, 17 Aug 2020 17:28:46 +0800 Subject: [PATCH 24/24] [Feature]modify some cases from rockxsj:Feature-presto to add presto datasource support (#3468) * Feature presto (#1) * * add presto datasource support update .gitigonre to igonre some files * * use another presto driver * * add LICENSE files about presto-jdbc * * just for test sonar Co-authored-by: rockxsj * modify the io.prestosql.jdbc.PrestoDriver to com.facebook.presto.jdbc.PrestoDriver * add presto connection in sql node Co-authored-by: rockxsj --- .../org/apache/dolphinscheduler/common/Constants.java | 2 +- dolphinscheduler-ui/src/js/conf/home/store/dag/state.js | 5 +++++ script/scp-hosts.sh | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 4f22e01241..072a67f44f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -898,7 +898,7 @@ public final class Constants { public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.driver.OracleDriver"; public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public static final String COM_PRESTO_JDBC_DRIVER = "io.prestosql.jdbc.PrestoDriver"; + public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"; /** * database type diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js index 05dfa77161..e3c75b838f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js @@ -96,6 +96,11 @@ export default { id: 7, code: 'DB2', disabled: false + }, + { + id: 8, + code: 'PRESTO', + disabled: false } ], // Alarm interface diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index 4a94cffa29..9da94ab79c 100644 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -33,8 +33,8 @@ for workerGroup in ${workersGroup[@]} do echo $workerGroup; worker=`echo $workerGroup|awk -F':' '{print $1}'` - groupName=`echo $workerGroup|awk -F':' '{print $2}'` - workersGroupMap+=([$worker]=$groupName) + groupsName=`echo $workerGroup|awk -F':' '{print $2}'` + workersGroupMap+=([$worker]=$groupsName) done @@ -53,7 +53,7 @@ do do # if worker in workersGroupMap if [[ "${workersGroupMap[${host}]}" ]] && [[ "${dsDir}" == "conf" ]]; then - sed -i ${txt} "s#worker.group.*#worker.group=${workersGroupMap[${host}]}#g" ${dsDir}/worker.properties + sed -i ${txt} "s:.*worker.groups.*:worker.groups=${workersGroupMap[${host}]}:g" ${dsDir}/worker.properties fi echo "start to scp $dsDir to $host/$installPath" @@ -61,4 +61,4 @@ do done echo "scp dirs to $host/$installPath complete" -done \ No newline at end of file +done