From 1ea48375f49e1b86b437faffbb19c1d6b6dec032 Mon Sep 17 00:00:00 2001 From: Rubik <39549317+wenhemin@users.noreply.github.com> Date: Mon, 2 Mar 2020 10:29:42 +0800 Subject: [PATCH 01/37] Fixbug i18n #2008 (#2035) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix:add chinese message * fix:add chinese message * fix:'getLocale()' can return not null * fix:optimize message content --- .../api/controller/DataSourceController.java | 4 +- .../dolphinscheduler/api/enums/Status.java | 422 +++++++++--------- .../api/enums/StatusTest.java | 12 +- 3 files changed, 228 insertions(+), 210 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index 9c04bf7d5c..881c93f2f7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -391,8 +391,8 @@ public class DataSourceController extends BaseController { try { return dataSourceService.verifyDataSourceName(loginUser, name); } catch (Exception e) { - logger.error(VERFIY_DATASOURCE_NAME_FAILURE.getMsg(),e); - return error(VERFIY_DATASOURCE_NAME_FAILURE.getCode(), VERFIY_DATASOURCE_NAME_FAILURE.getMsg()); + logger.error(VERIFY_DATASOURCE_NAME_FAILURE.getMsg(), e); + return error(VERIFY_DATASOURCE_NAME_FAILURE.getCode(), VERIFY_DATASOURCE_NAME_FAILURE.getMsg()); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 7a87d552de..9955463f8e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -16,244 +16,248 @@ */ package org.apache.dolphinscheduler.api.enums; +import org.springframework.context.i18n.LocaleContextHolder; + +import java.util.Locale; + /** * status enum */ public enum Status { - SUCCESS(0, "success"), + SUCCESS(0, "success", "成功"), - REQUEST_PARAMS_NOT_VALID_ERROR(10001, "request parameter {0} is not valid"), - TASK_TIMEOUT_PARAMS_ERROR(10002, "task timeout parameter is not valid"), - USER_NAME_EXIST(10003, "user name already exists"), - USER_NAME_NULL(10004,"user name is null"), - HDFS_OPERATION_ERROR(10006, "hdfs operation error"), - TASK_INSTANCE_NOT_FOUND(10008, "task instance not found"), - TENANT_NAME_EXIST(10009, "tenant code already exists"), - USER_NOT_EXIST(10010, "user {0} not exists"), - ALERT_GROUP_NOT_EXIST(10011, "alarm group not found"), - ALERT_GROUP_EXIST(10012, "alarm group already exists"), - USER_NAME_PASSWD_ERROR(10013,"user name or password error"), - LOGIN_SESSION_FAILED(10014,"create session failed!"), - DATASOURCE_EXIST(10015, "data source name already exists"), - DATASOURCE_CONNECT_FAILED(10016, "data source connection failed"), - TENANT_NOT_EXIST(10017, "tenant not exists"), - PROJECT_NOT_FOUNT(10018, "project {0} not found "), - PROJECT_ALREADY_EXISTS(10019, "project {0} already exists"), - TASK_INSTANCE_NOT_EXISTS(10020, "task instance {0} does not exist"), - TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE(10021, "task instance {0} is not sub process instance"), - SCHEDULE_CRON_NOT_EXISTS(10022, "scheduler crontab {0} does not exist"), - SCHEDULE_CRON_ONLINE_FORBID_UPDATE(10023, "online status does not allow updateProcessInstance operations"), - SCHEDULE_CRON_CHECK_FAILED(10024, "scheduler crontab expression validation failure: {0}"), - MASTER_NOT_EXISTS(10025, "master does not exist"), - SCHEDULE_STATUS_UNKNOWN(10026, "unknown command: {0}"), - CREATE_ALERT_GROUP_ERROR(10027,"create alert group error"), - QUERY_ALL_ALERTGROUP_ERROR(10028,"query all alertgroup error"), - LIST_PAGING_ALERT_GROUP_ERROR(10029,"list paging alert group error"), - UPDATE_ALERT_GROUP_ERROR(10030,"updateProcessInstance alert group error"), - DELETE_ALERT_GROUP_ERROR(10031,"delete alert group error"), - ALERT_GROUP_GRANT_USER_ERROR(10032,"alert group grant user error"), - CREATE_DATASOURCE_ERROR(10033,"create datasource error"), - UPDATE_DATASOURCE_ERROR(10034,"updateProcessInstance datasource error"), - QUERY_DATASOURCE_ERROR(10035,"query datasource error"), - CONNECT_DATASOURCE_FAILURE(10036,"connect datasource failure"), - CONNECTION_TEST_FAILURE(10037,"connection test failure"), - DELETE_DATA_SOURCE_FAILURE(10038,"delete data source failure"), - VERFIY_DATASOURCE_NAME_FAILURE(10039,"verfiy datasource name failure"), - UNAUTHORIZED_DATASOURCE(10040,"unauthorized datasource"), - AUTHORIZED_DATA_SOURCE(10041,"authorized data source"), - LOGIN_SUCCESS(10042,"login success"), - USER_LOGIN_FAILURE(10043,"user login failure"), - LIST_WORKERS_ERROR(10044,"list workers error"), - LIST_MASTERS_ERROR(10045,"list masters error"), - UPDATE_PROJECT_ERROR(10046,"updateProcessInstance project error"), - QUERY_PROJECT_DETAILS_BY_ID_ERROR(10047,"query project details by id error"), - CREATE_PROJECT_ERROR(10048,"create project error"), - LOGIN_USER_QUERY_PROJECT_LIST_PAGING_ERROR(10049,"login user query project list paging error"), - DELETE_PROJECT_ERROR(10050,"delete project error"), - QUERY_UNAUTHORIZED_PROJECT_ERROR(10051,"query unauthorized project error"), - QUERY_AUTHORIZED_PROJECT(10052,"query authorized project"), - QUERY_QUEUE_LIST_ERROR(10053,"query queue list error"), - CREATE_RESOURCE_ERROR(10054,"create resource error"), - UPDATE_RESOURCE_ERROR(10055,"updateProcessInstance resource error"), - QUERY_RESOURCES_LIST_ERROR(10056,"query resources list error"), - QUERY_RESOURCES_LIST_PAGING(10057,"query resources list paging"), - DELETE_RESOURCE_ERROR(10058,"delete resource error"), - VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR(10059,"verify resource by name and type error"), - VIEW_RESOURCE_FILE_ON_LINE_ERROR(10060,"view resource file online error"), - CREATE_RESOURCE_FILE_ON_LINE_ERROR(10061,"create resource file online error"), - RESOURCE_FILE_IS_EMPTY(10062,"resource file is empty"), - EDIT_RESOURCE_FILE_ON_LINE_ERROR(10063,"edit resource file online error"), - DOWNLOAD_RESOURCE_FILE_ERROR(10064,"download resource file error"), - CREATE_UDF_FUNCTION_ERROR(10065 ,"create udf function error"), - VIEW_UDF_FUNCTION_ERROR( 10066,"view udf function error"), - UPDATE_UDF_FUNCTION_ERROR(10067,"updateProcessInstance udf function error"), - QUERY_UDF_FUNCTION_LIST_PAGING_ERROR( 10068,"query udf function list paging error"), - QUERY_DATASOURCE_BY_TYPE_ERROR( 10069,"query datasource by type error"), - VERIFY_UDF_FUNCTION_NAME_ERROR( 10070,"verify udf function name error"), - DELETE_UDF_FUNCTION_ERROR( 10071,"delete udf function error"), - AUTHORIZED_FILE_RESOURCE_ERROR( 10072,"authorized file resource error"), - UNAUTHORIZED_FILE_RESOURCE_ERROR( 10073,"unauthorized file resource error"), - UNAUTHORIZED_UDF_FUNCTION_ERROR( 10074,"unauthorized udf function error"), - AUTHORIZED_UDF_FUNCTION_ERROR(10075,"authorized udf function error"), - CREATE_SCHEDULE_ERROR(10076,"create schedule error"), - UPDATE_SCHEDULE_ERROR(10077,"updateProcessInstance schedule error"), - PUBLISH_SCHEDULE_ONLINE_ERROR(10078,"publish schedule online error"), - OFFLINE_SCHEDULE_ERROR(10079,"offline schedule error"), - QUERY_SCHEDULE_LIST_PAGING_ERROR(10080,"query schedule list paging error"), - QUERY_SCHEDULE_LIST_ERROR(10081,"query schedule list error"), - QUERY_TASK_LIST_PAGING_ERROR(10082,"query task list paging error"), - QUERY_TASK_RECORD_LIST_PAGING_ERROR(10083,"query task record list paging error"), - CREATE_TENANT_ERROR(10084,"create tenant error"), - QUERY_TENANT_LIST_PAGING_ERROR(10085,"query tenant list paging error"), - QUERY_TENANT_LIST_ERROR(10086,"query tenant list error"), - UPDATE_TENANT_ERROR(10087,"updateProcessInstance tenant error"), - DELETE_TENANT_BY_ID_ERROR(10088,"delete tenant by id error"), - VERIFY_TENANT_CODE_ERROR(10089,"verify tenant code error"), - CREATE_USER_ERROR(10090,"create user error"), - QUERY_USER_LIST_PAGING_ERROR(10091,"query user list paging error"), - UPDATE_USER_ERROR(10092,"updateProcessInstance user error"), - DELETE_USER_BY_ID_ERROR(10093,"delete user by id error"), - GRANT_PROJECT_ERROR(10094,"grant project error"), - GRANT_RESOURCE_ERROR(10095,"grant resource error"), - GRANT_UDF_FUNCTION_ERROR(10096,"grant udf function error"), - GRANT_DATASOURCE_ERROR(10097,"grant datasource error"), - GET_USER_INFO_ERROR(10098,"get user info error"), - USER_LIST_ERROR(10099,"user list error"), - VERIFY_USERNAME_ERROR(10100,"verify username error"), - UNAUTHORIZED_USER_ERROR(10101,"unauthorized user error"), - AUTHORIZED_USER_ERROR(10102,"authorized user error"), - QUERY_TASK_INSTANCE_LOG_ERROR(10103,"view task instance log error"), - DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR(10104,"download task instance log file error"), - CREATE_PROCESS_DEFINITION(10105,"create process definition"), - VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106,"verify process definition name unique error"), - UPDATE_PROCESS_DEFINITION_ERROR(10107,"updateProcessInstance process definition error"), - RELEASE_PROCESS_DEFINITION_ERROR(10108,"release process definition error"), - QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR(10109,"query datail of process definition error"), - QUERY_PROCCESS_DEFINITION_LIST(10110,"query proccess definition list"), - ENCAPSULATION_TREEVIEW_STRUCTURE_ERROR(10111,"encapsulation treeview structure error"), - GET_TASKS_LIST_BY_PROCESS_DEFINITION_ID_ERROR(10112,"get tasks list by process definition id error"), - QUERY_PROCESS_INSTANCE_LIST_PAGING_ERROR(10113,"query process instance list paging error"), - QUERY_TASK_LIST_BY_PROCESS_INSTANCE_ID_ERROR(10114,"query task list by process instance id error"), - UPDATE_PROCESS_INSTANCE_ERROR(10115,"updateProcessInstance process instance error"), - QUERY_PROCESS_INSTANCE_BY_ID_ERROR(10116,"query process instance by id error"), - DELETE_PROCESS_INSTANCE_BY_ID_ERROR(10117,"delete process instance by id error"), - QUERY_SUB_PROCESS_INSTANCE_DETAIL_INFO_BY_TASK_ID_ERROR(10118,"query sub process instance detail info by task id error"), - QUERY_PARENT_PROCESS_INSTANCE_DETAIL_INFO_BY_SUB_PROCESS_INSTANCE_ID_ERROR(10119,"query parent process instance detail info by sub process instance id error"), - QUERY_PROCESS_INSTANCE_ALL_VARIABLES_ERROR(10120,"query process instance all variables error"), - ENCAPSULATION_PROCESS_INSTANCE_GANTT_STRUCTURE_ERROR(10121,"encapsulation process instance gantt structure error"), - QUERY_PROCCESS_DEFINITION_LIST_PAGING_ERROR(10122,"query proccess definition list paging error"), - SIGN_OUT_ERROR(10123,"sign out error"), - TENANT_CODE_HAS_ALREADY_EXISTS(10124,"tenant code has already exists"), - IP_IS_EMPTY(10125,"ip is empty"), - SCHEDULE_CRON_REALEASE_NEED_NOT_CHANGE(10126, "schedule release is already {0}"), - CREATE_QUEUE_ERROR(10127, "create queue error"), - QUEUE_NOT_EXIST(10128, "queue {0} not exists"), - QUEUE_VALUE_EXIST(10129, "queue value {0} already exists"), - QUEUE_NAME_EXIST(10130, "queue name {0} already exists"), - UPDATE_QUEUE_ERROR(10131, "update queue error"), - NEED_NOT_UPDATE_QUEUE(10132, "no content changes, no updates are required"), - VERIFY_QUEUE_ERROR(10133,"verify queue error"), - NAME_NULL(10134,"name must be not null"), - NAME_EXIST(10135, "name {0} already exists"), - SAVE_ERROR(10136, "save error"), - DELETE_PROJECT_ERROR_DEFINES_NOT_NULL(10137, "please delete the process definitions in project first!"), - BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR(10117,"batch delete process instance by ids {0} error"), - PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error"), - PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error"), - SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end"), - DELETE_TENANT_BY_ID_FAIL(100142,"delete tenant by id fail, for there are {0} process instances in executing using it"), - DELETE_TENANT_BY_ID_FAIL_DEFINES(100143,"delete tenant by id fail, for there are {0} process definitions using it"), - DELETE_TENANT_BY_ID_FAIL_USERS(100144,"delete tenant by id fail, for there are {0} users using it"), + REQUEST_PARAMS_NOT_VALID_ERROR(10001, "request parameter {0} is not valid", "请求参数[{0}]无效"), + TASK_TIMEOUT_PARAMS_ERROR(10002, "task timeout parameter is not valid", "任务超时参数无效"), + USER_NAME_EXIST(10003, "user name already exists", "用户名已存在"), + USER_NAME_NULL(10004,"user name is null", "用户名不能为空"), + HDFS_OPERATION_ERROR(10006, "hdfs operation error", "hdfs操作错误"), + TASK_INSTANCE_NOT_FOUND(10008, "task instance not found", "任务实例不存在"), + TENANT_NAME_EXIST(10009, "tenant code already exists", "租户编码不能为空"), + USER_NOT_EXIST(10010, "user {0} not exists", "用户[{0}]不存在"), + ALERT_GROUP_NOT_EXIST(10011, "alarm group not found", "告警组不存在"), + ALERT_GROUP_EXIST(10012, "alarm group already exists", "告警组名称已存在"), + USER_NAME_PASSWD_ERROR(10013,"user name or password error", "用户名或密码错误"), + LOGIN_SESSION_FAILED(10014,"create session failed!", "创建session失败"), + DATASOURCE_EXIST(10015, "data source name already exists", "数据源名称已存在"), + DATASOURCE_CONNECT_FAILED(10016, "data source connection failed", "建立数据源连接失败"), + TENANT_NOT_EXIST(10017, "tenant not exists", "租户不存在"), + PROJECT_NOT_FOUNT(10018, "project {0} not found ", "项目[{0}]不存在"), + PROJECT_ALREADY_EXISTS(10019, "project {0} already exists", "项目名称[{0}]已存在"), + TASK_INSTANCE_NOT_EXISTS(10020, "task instance {0} does not exist", "任务实例[{0}]不存在"), + TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE(10021, "task instance {0} is not sub process instance", "任务实例[{0}]不是子流程实例"), + SCHEDULE_CRON_NOT_EXISTS(10022, "scheduler crontab {0} does not exist", "调度配置定时表达式[{0}]不存在"), + SCHEDULE_CRON_ONLINE_FORBID_UPDATE(10023, "online status does not allow update operations", "调度配置上线状态不允许修改"), + SCHEDULE_CRON_CHECK_FAILED(10024, "scheduler crontab expression validation failure: {0}", "调度配置定时表达式验证失败: {0}"), + MASTER_NOT_EXISTS(10025, "master does not exist", "无可用master节点"), + SCHEDULE_STATUS_UNKNOWN(10026, "unknown status: {0}", "未知状态: {0}"), + CREATE_ALERT_GROUP_ERROR(10027,"create alert group error", "创建告警组错误"), + QUERY_ALL_ALERTGROUP_ERROR(10028,"query all alertgroup error", "查询告警组错误"), + LIST_PAGING_ALERT_GROUP_ERROR(10029,"list paging alert group error", "分页查询告警组错误"), + UPDATE_ALERT_GROUP_ERROR(10030,"update alert group error", "更新告警组错误"), + DELETE_ALERT_GROUP_ERROR(10031,"delete alert group error", "删除告警组错误"), + ALERT_GROUP_GRANT_USER_ERROR(10032,"alert group grant user error", "告警组授权用户错误"), + CREATE_DATASOURCE_ERROR(10033,"create datasource error", "创建数据源错误"), + UPDATE_DATASOURCE_ERROR(10034,"update datasource error", "更新数据源错误"), + QUERY_DATASOURCE_ERROR(10035,"query datasource error", "查询数据源错误"), + CONNECT_DATASOURCE_FAILURE(10036,"connect datasource failure", "建立数据源连接失败"), + CONNECTION_TEST_FAILURE(10037,"connection test failure", "测试数据源连接失败"), + DELETE_DATA_SOURCE_FAILURE(10038,"delete data source failure", "删除数据源失败"), + VERIFY_DATASOURCE_NAME_FAILURE(10039,"verify datasource name failure", "验证数据源名称失败"), + UNAUTHORIZED_DATASOURCE(10040,"unauthorized datasource", "未经授权的数据源"), + AUTHORIZED_DATA_SOURCE(10041,"authorized data source", "授权数据源失败"), + LOGIN_SUCCESS(10042,"login success", "登录成功"), + USER_LOGIN_FAILURE(10043,"user login failure", "用户登录失败"), + LIST_WORKERS_ERROR(10044,"list workers error", "查询worker列表错误"), + LIST_MASTERS_ERROR(10045,"list masters error", "查询master列表错误"), + UPDATE_PROJECT_ERROR(10046,"update project error", "更新项目信息错误"), + QUERY_PROJECT_DETAILS_BY_ID_ERROR(10047,"query project details by id error", "查询项目详细信息错误"), + CREATE_PROJECT_ERROR(10048,"create project error", "创建项目错误"), + LOGIN_USER_QUERY_PROJECT_LIST_PAGING_ERROR(10049,"login user query project list paging error", "分页查询项目列表错误"), + DELETE_PROJECT_ERROR(10050,"delete project error", "删除项目错误"), + QUERY_UNAUTHORIZED_PROJECT_ERROR(10051,"query unauthorized project error", "查询未授权项目错误"), + QUERY_AUTHORIZED_PROJECT(10052,"query authorized project", "查询授权项目错误"), + QUERY_QUEUE_LIST_ERROR(10053,"query queue list error", "查询队列列表错误"), + CREATE_RESOURCE_ERROR(10054,"create resource error", "创建资源错误"), + UPDATE_RESOURCE_ERROR(10055,"update resource error", "更新资源错误"), + QUERY_RESOURCES_LIST_ERROR(10056,"query resources list error", "查询资源列表错误"), + QUERY_RESOURCES_LIST_PAGING(10057,"query resources list paging", "分页查询资源列表错误"), + DELETE_RESOURCE_ERROR(10058,"delete resource error", "删除资源错误"), + VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR(10059,"verify resource by name and type error", "资源名称或类型验证错误"), + VIEW_RESOURCE_FILE_ON_LINE_ERROR(10060,"view resource file online error", "查看资源文件错误"), + CREATE_RESOURCE_FILE_ON_LINE_ERROR(10061,"create resource file online error", "创建资源文件错误"), + RESOURCE_FILE_IS_EMPTY(10062,"resource file is empty", "资源文件内容不能为空"), + EDIT_RESOURCE_FILE_ON_LINE_ERROR(10063,"edit resource file online error", "更新资源文件错误"), + DOWNLOAD_RESOURCE_FILE_ERROR(10064,"download resource file error", "下载资源文件错误"), + CREATE_UDF_FUNCTION_ERROR(10065 ,"create udf function error", "创建UDF函数错误"), + VIEW_UDF_FUNCTION_ERROR( 10066,"view udf function error", "查询UDF函数错误"), + UPDATE_UDF_FUNCTION_ERROR(10067,"update udf function error", "更新UDF函数错误"), + QUERY_UDF_FUNCTION_LIST_PAGING_ERROR( 10068,"query udf function list paging error", "分页查询UDF函数列表错误"), + QUERY_DATASOURCE_BY_TYPE_ERROR( 10069,"query datasource by type error", "查询数据源信息错误"), + VERIFY_UDF_FUNCTION_NAME_ERROR( 10070,"verify udf function name error", "UDF函数名称验证错误"), + DELETE_UDF_FUNCTION_ERROR( 10071,"delete udf function error", "删除UDF函数错误"), + AUTHORIZED_FILE_RESOURCE_ERROR( 10072,"authorized file resource error", "授权资源文件错误"), + UNAUTHORIZED_FILE_RESOURCE_ERROR( 10073,"unauthorized file resource error", "查询未授权资源错误"), + UNAUTHORIZED_UDF_FUNCTION_ERROR( 10074,"unauthorized udf function error", "查询未授权UDF函数错误"), + AUTHORIZED_UDF_FUNCTION_ERROR(10075,"authorized udf function error", "授权UDF函数错误"), + CREATE_SCHEDULE_ERROR(10076,"create schedule error", "创建调度配置错误"), + UPDATE_SCHEDULE_ERROR(10077,"update schedule error", "更新调度配置错误"), + PUBLISH_SCHEDULE_ONLINE_ERROR(10078,"publish schedule online error", "上线调度配置错误"), + OFFLINE_SCHEDULE_ERROR(10079,"offline schedule error", "下线调度配置错误"), + QUERY_SCHEDULE_LIST_PAGING_ERROR(10080,"query schedule list paging error", "分页查询调度配置列表错误"), + QUERY_SCHEDULE_LIST_ERROR(10081,"query schedule list error", "查询调度配置列表错误"), + QUERY_TASK_LIST_PAGING_ERROR(10082,"query task list paging error", "分页查询任务列表错误"), + QUERY_TASK_RECORD_LIST_PAGING_ERROR(10083,"query task record list paging error", "分页查询任务记录错误"), + CREATE_TENANT_ERROR(10084,"create tenant error", "创建租户错误"), + QUERY_TENANT_LIST_PAGING_ERROR(10085,"query tenant list paging error", "分页查询租户列表错误"), + QUERY_TENANT_LIST_ERROR(10086,"query tenant list error", "查询租户列表错误"), + UPDATE_TENANT_ERROR(10087,"update tenant error", "更新租户错误"), + DELETE_TENANT_BY_ID_ERROR(10088,"delete tenant by id error", "删除租户错误"), + VERIFY_TENANT_CODE_ERROR(10089,"verify tenant code error", "租户编码验证错误"), + CREATE_USER_ERROR(10090,"create user error", "创建用户错误"), + QUERY_USER_LIST_PAGING_ERROR(10091,"query user list paging error", "分页查询用户列表错误"), + UPDATE_USER_ERROR(10092,"update user error", "更新用户错误"), + DELETE_USER_BY_ID_ERROR(10093,"delete user by id error", "删除用户错误"), + GRANT_PROJECT_ERROR(10094,"grant project error", "授权项目错误"), + GRANT_RESOURCE_ERROR(10095,"grant resource error", "授权资源错误"), + GRANT_UDF_FUNCTION_ERROR(10096,"grant udf function error", "授权UDF函数错误"), + GRANT_DATASOURCE_ERROR(10097,"grant datasource error", "授权数据源错误"), + GET_USER_INFO_ERROR(10098,"get user info error", "获取用户信息错误"), + USER_LIST_ERROR(10099,"user list error", "查询用户列表错误"), + VERIFY_USERNAME_ERROR(10100,"verify username error", "用户名验证错误"), + UNAUTHORIZED_USER_ERROR(10101,"unauthorized user error", "查询未授权用户错误"), + AUTHORIZED_USER_ERROR(10102,"authorized user error", "查询授权用户错误"), + QUERY_TASK_INSTANCE_LOG_ERROR(10103,"view task instance log error", "查询任务实例日志错误"), + DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR(10104,"download task instance log file error", "下载任务日志文件错误"), + CREATE_PROCESS_DEFINITION(10105,"create process definition", "创建工作流错误"), + VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106,"verify process definition name unique error", "工作流名称已存在"), + UPDATE_PROCESS_DEFINITION_ERROR(10107,"update process definition error", "更新工作流定义错误"), + RELEASE_PROCESS_DEFINITION_ERROR(10108,"release process definition error", "上线工作流错误"), + QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR(10109,"query datail of process definition error", "查询工作流详细信息错误"), + QUERY_PROCCESS_DEFINITION_LIST(10110,"query proccess definition list", "查询工作流列表错误"), + ENCAPSULATION_TREEVIEW_STRUCTURE_ERROR(10111,"encapsulation treeview structure error", "查询工作流树形图数据错误"), + GET_TASKS_LIST_BY_PROCESS_DEFINITION_ID_ERROR(10112,"get tasks list by process definition id error", "查询工作流定义节点信息错误"), + QUERY_PROCESS_INSTANCE_LIST_PAGING_ERROR(10113,"query process instance list paging error", "分页查询工作流实例列表错误"), + QUERY_TASK_LIST_BY_PROCESS_INSTANCE_ID_ERROR(10114,"query task list by process instance id error", "查询任务实例列表错误"), + UPDATE_PROCESS_INSTANCE_ERROR(10115,"update process instance error", "更新工作流实例错误"), + QUERY_PROCESS_INSTANCE_BY_ID_ERROR(10116,"query process instance by id error", "查询工作流实例错误"), + DELETE_PROCESS_INSTANCE_BY_ID_ERROR(10117,"delete process instance by id error", "删除工作流实例错误"), + QUERY_SUB_PROCESS_INSTANCE_DETAIL_INFO_BY_TASK_ID_ERROR(10118,"query sub process instance detail info by task id error", "查询子流程任务实例错误"), + QUERY_PARENT_PROCESS_INSTANCE_DETAIL_INFO_BY_SUB_PROCESS_INSTANCE_ID_ERROR(10119,"query parent process instance detail info by sub process instance id error", "查询子流程该工作流实例错误"), + QUERY_PROCESS_INSTANCE_ALL_VARIABLES_ERROR(10120,"query process instance all variables error", "查询工作流自定义变量信息错误"), + ENCAPSULATION_PROCESS_INSTANCE_GANTT_STRUCTURE_ERROR(10121,"encapsulation process instance gantt structure error", "查询工作流实例甘特图数据错误"), + QUERY_PROCCESS_DEFINITION_LIST_PAGING_ERROR(10122,"query proccess definition list paging error", "分页查询工作流定义列表错误"), + SIGN_OUT_ERROR(10123,"sign out error", "退出错误"), + TENANT_CODE_HAS_ALREADY_EXISTS(10124,"tenant code has already exists", "租户编码已存在"), + IP_IS_EMPTY(10125,"ip is empty", "IP地址不能为空"), + SCHEDULE_CRON_REALEASE_NEED_NOT_CHANGE(10126, "schedule release is already {0}", "调度配置上线错误[{0}]"), + CREATE_QUEUE_ERROR(10127, "create queue error", "创建队列错误"), + QUEUE_NOT_EXIST(10128, "queue {0} not exists", "队列ID[{0}]不存在"), + QUEUE_VALUE_EXIST(10129, "queue value {0} already exists", "队列值[{0}]已存在"), + QUEUE_NAME_EXIST(10130, "queue name {0} already exists", "队列名称[{0}]已存在"), + UPDATE_QUEUE_ERROR(10131, "update queue error", "更新队列信息错误"), + NEED_NOT_UPDATE_QUEUE(10132, "no content changes, no updates are required", "数据未变更,不需要更新队列信息"), + VERIFY_QUEUE_ERROR(10133,"verify queue error", "验证队列信息错误"), + NAME_NULL(10134,"name must be not null", "名称不能为空"), + NAME_EXIST(10135, "name {0} already exists", "名称[{0}]已存在"), + SAVE_ERROR(10136, "save error", "保存错误"), + DELETE_PROJECT_ERROR_DEFINES_NOT_NULL(10137, "please delete the process definitions in project first!", "请先删除全部工作流定义"), + BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR(10117,"batch delete process instance by ids {0} error", "批量删除工作流实例错误"), + PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error", "预览调度配置错误"), + PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error", "解析调度表达式错误"), + SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end", "开始时间不能和结束时间一样"), + DELETE_TENANT_BY_ID_FAIL(100142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败,有[{0}]个运行中的工作流实例正在使用"), + DELETE_TENANT_BY_ID_FAIL_DEFINES(100143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败,有[{0}]个工作流定义正在使用"), + DELETE_TENANT_BY_ID_FAIL_USERS(100144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败,有[{0}]个用户正在使用"), - DELETE_WORKER_GROUP_BY_ID_FAIL(100145,"delete worker group by id fail, for there are {0} process instances in executing using it"), + DELETE_WORKER_GROUP_BY_ID_FAIL(100145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败,有[{0}]个运行中的工作流实例正在使用"), - QUERY_WORKER_GROUP_FAIL(100146,"query worker group fail "), - DELETE_WORKER_GROUP_FAIL(100147,"delete worker group fail "), + QUERY_WORKER_GROUP_FAIL(100146,"query worker group fail ", "查询worker分组失败"), + DELETE_WORKER_GROUP_FAIL(100147,"delete worker group fail ", "删除worker分组失败"), - UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found"), - UDF_FUNCTION_EXISTS(20002, "UDF function already exists"), - RESOURCE_NOT_EXIST(20004, "resource not exist"), - RESOURCE_EXIST(20005, "resource already exists"), - RESOURCE_SUFFIX_NOT_SUPPORT_VIEW(20006, "resource suffix do not support online viewing"), - RESOURCE_SIZE_EXCEED_LIMIT(20007, "upload resource file size exceeds limit"), - RESOURCE_SUFFIX_FORBID_CHANGE(20008, "resource suffix not allowed to be modified"), - UDF_RESOURCE_SUFFIX_NOT_JAR(20009, "UDF resource suffix name must be jar"), - HDFS_COPY_FAIL(20009, "hdfs copy {0} -> {1} fail"), - RESOURCE_FILE_EXIST(20010, "resource file {0} already exists in hdfs,please delete it or change name!"), - RESOURCE_FILE_NOT_EXIST(20011, "resource file {0} not exists in hdfs!"), + UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), + UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), + RESOURCE_NOT_EXIST(20004, "resource not exist", "资源不存在"), + RESOURCE_EXIST(20005, "resource already exists", "资源已存在"), + RESOURCE_SUFFIX_NOT_SUPPORT_VIEW(20006, "resource suffix do not support online viewing", "资源文件后缀不支持查看"), + RESOURCE_SIZE_EXCEED_LIMIT(20007, "upload resource file size exceeds limit", "上传资源文件大小超过限制"), + RESOURCE_SUFFIX_FORBID_CHANGE(20008, "resource suffix not allowed to be modified", "资源文件后缀不支持修改"), + UDF_RESOURCE_SUFFIX_NOT_JAR(20009, "UDF resource suffix name must be jar", "UDF资源文件后缀名只支持[jar]"), + HDFS_COPY_FAIL(20009, "hdfs copy {0} -> {1} fail", "hdfs复制失败:[{0}] -> [{1}]"), + RESOURCE_FILE_EXIST(20010, "resource file {0} already exists in hdfs,please delete it or change name!", "资源文件[{0}]在hdfs中已存在,请删除或修改资源名"), + RESOURCE_FILE_NOT_EXIST(20011, "resource file {0} not exists in hdfs!", "资源文件[{0}]在hdfs中不存在"), - USER_NO_OPERATION_PERM(30001, "user has no operation privilege"), - USER_NO_OPERATION_PROJECT_PERM(30002, "user {0} is not has project {1} permission"), + USER_NO_OPERATION_PERM(30001, "user has no operation privilege", "当前用户没有操作权限"), + USER_NO_OPERATION_PROJECT_PERM(30002, "user {0} is not has project {1} permission", "当前用户[{0}]没有[{1}]项目的操作权限"), - PROCESS_INSTANCE_NOT_EXIST(50001, "process instance {0} does not exist"), - PROCESS_INSTANCE_EXIST(50002, "process instance {0} already exists"), - PROCESS_DEFINE_NOT_EXIST(50003, "process definition {0} does not exist"), - PROCESS_DEFINE_NOT_RELEASE(50004, "process definition {0} not on line"), - PROCESS_INSTANCE_ALREADY_CHANGED(50005, "the status of process instance {0} is already {1}"), - PROCESS_INSTANCE_STATE_OPERATION_ERROR(50006, "the status of process instance {0} is {1},Cannot perform {2} operation"), - SUB_PROCESS_INSTANCE_NOT_EXIST(50007, "the task belong to process instance does not exist"), - PROCESS_DEFINE_NOT_ALLOWED_EDIT(50008, "process definition {0} does not allow edit"), - PROCESS_INSTANCE_EXECUTING_COMMAND(50009, "process instance {0} is executing the command, please wait ..."), - PROCESS_INSTANCE_NOT_SUB_PROCESS_INSTANCE(50010, "process instance {0} is not sub process instance"), - TASK_INSTANCE_STATE_COUNT_ERROR(50011,"task instance state count error"), - COUNT_PROCESS_INSTANCE_STATE_ERROR(50012,"count process instance state error"), - COUNT_PROCESS_DEFINITION_USER_ERROR(50013,"count process definition user error"), - START_PROCESS_INSTANCE_ERROR(50014,"start process instance error"), - EXECUTE_PROCESS_INSTANCE_ERROR(50015,"execute process instance error"), - CHECK_PROCESS_DEFINITION_ERROR(50016,"check process definition error"), - QUERY_RECIPIENTS_AND_COPYERS_BY_PROCESS_DEFINITION_ERROR(50017,"query recipients and copyers by process definition error"), - DATA_IS_NOT_VALID(50017,"data %s not valid"), - DATA_IS_NULL(50018,"data %s is null"), - PROCESS_NODE_HAS_CYCLE(50019,"process node has cycle"), - PROCESS_NODE_S_PARAMETER_INVALID(50020,"process node %s parameter invalid"), - PROCESS_DEFINE_STATE_ONLINE(50021, "process definition {0} is already on line"), - DELETE_PROCESS_DEFINE_BY_ID_ERROR(50022,"delete process definition by id error"), - SCHEDULE_CRON_STATE_ONLINE(50023,"the status of schedule {0} is already on line"), - DELETE_SCHEDULE_CRON_BY_ID_ERROR(50024,"delete schedule by id error"), - BATCH_DELETE_PROCESS_DEFINE_ERROR(50025,"batch delete process definition error"), - BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR(50026,"batch delete process definition by ids {0} error"), - TENANT_NOT_SUITABLE(50027,"there is not any tenant suitable, please choose a tenant available."), - EXPORT_PROCESS_DEFINE_BY_ID_ERROR(50028,"export process definition by id error"), - IMPORT_PROCESS_DEFINE_ERROR(50029,"import process definition error"), + PROCESS_INSTANCE_NOT_EXIST(50001, "process instance {0} does not exist", "工作流实例[{0}]不存在"), + PROCESS_INSTANCE_EXIST(50002, "process instance {0} already exists", "工作流实例[{0}]已存在"), + PROCESS_DEFINE_NOT_EXIST(50003, "process definition {0} does not exist", "工作流定义[{0}]不存在"), + PROCESS_DEFINE_NOT_RELEASE(50004, "process definition {0} not on line", "工作流定义[{0}]不是上线状态"), + PROCESS_INSTANCE_ALREADY_CHANGED(50005, "the status of process instance {0} is already {1}", "工作流实例[{0}]的状态已经是[{1}]"), + PROCESS_INSTANCE_STATE_OPERATION_ERROR(50006, "the status of process instance {0} is {1},Cannot perform {2} operation", "工作流实例[{0}]的状态是[{1}],无法执行[{2}]操作"), + SUB_PROCESS_INSTANCE_NOT_EXIST(50007, "the task belong to process instance does not exist", "子工作流实例不存在"), + PROCESS_DEFINE_NOT_ALLOWED_EDIT(50008, "process definition {0} does not allow edit", "工作流定义[{0}]不允许修改"), + PROCESS_INSTANCE_EXECUTING_COMMAND(50009, "process instance {0} is executing the command, please wait ...", "工作流实例[{0}]正在执行命令,请稍等..."), + PROCESS_INSTANCE_NOT_SUB_PROCESS_INSTANCE(50010, "process instance {0} is not sub process instance", "工作流实例[{0}]不是子工作流实例"), + TASK_INSTANCE_STATE_COUNT_ERROR(50011,"task instance state count error", "查询各状态任务实例数错误"), + COUNT_PROCESS_INSTANCE_STATE_ERROR(50012,"count process instance state error", "查询各状态流程实例数错误"), + COUNT_PROCESS_DEFINITION_USER_ERROR(50013,"count process definition user error", "查询各用户流程定义数错误"), + START_PROCESS_INSTANCE_ERROR(50014,"start process instance error", "运行工作流实例错误"), + EXECUTE_PROCESS_INSTANCE_ERROR(50015,"execute process instance error", "操作工作流实例错误"), + CHECK_PROCESS_DEFINITION_ERROR(50016,"check process definition error", "检查工作流实例错误"), + QUERY_RECIPIENTS_AND_COPYERS_BY_PROCESS_DEFINITION_ERROR(50017,"query recipients and copyers by process definition error", "查询收件人和抄送人错误"), + DATA_IS_NOT_VALID(50017,"data %s not valid", "数据[%s]无效"), + DATA_IS_NULL(50018,"data %s is null", "数据[%s]不能为空"), + PROCESS_NODE_HAS_CYCLE(50019,"process node has cycle", "流程节点间存在循环依赖"), + PROCESS_NODE_S_PARAMETER_INVALID(50020,"process node %s parameter invalid", "流程节点[%s]参数无效"), + PROCESS_DEFINE_STATE_ONLINE(50021, "process definition {0} is already on line", "工作流定义[{0}]已上线"), + DELETE_PROCESS_DEFINE_BY_ID_ERROR(50022,"delete process definition by id error", "删除工作流定义错误"), + SCHEDULE_CRON_STATE_ONLINE(50023,"the status of schedule {0} is already on line", "调度配置[{0}]已上线"), + DELETE_SCHEDULE_CRON_BY_ID_ERROR(50024,"delete schedule by id error", "删除调度配置错误"), + BATCH_DELETE_PROCESS_DEFINE_ERROR(50025,"batch delete process definition error", "批量删除工作流定义错误"), + BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR(50026,"batch delete process definition by ids {0} error", "批量删除工作流定义[{0}]错误"), + TENANT_NOT_SUITABLE(50027,"there is not any tenant suitable, please choose a tenant available.", "没有合适的租户,请选择可用的租户"), + EXPORT_PROCESS_DEFINE_BY_ID_ERROR(50028,"export process definition by id error", "导出工作流定义错误"), + IMPORT_PROCESS_DEFINE_ERROR(50029,"import process definition error", "导入工作流定义错误"), - HDFS_NOT_STARTUP(60001,"hdfs not startup"), - HDFS_TERANT_RESOURCES_FILE_EXISTS(60002,"resource file exists,please delete resource first"), - HDFS_TERANT_UDFS_FILE_EXISTS(60003,"udf file exists,please delete resource first"), + HDFS_NOT_STARTUP(60001,"hdfs not startup", "hdfs未启用"), /** * for monitor */ - QUERY_DATABASE_STATE_ERROR(70001,"query database state error"), - QUERY_ZOOKEEPER_STATE_ERROR(70002,"query zookeeper state error"), + QUERY_DATABASE_STATE_ERROR(70001,"query database state error", "查询数据库状态错误"), + QUERY_ZOOKEEPER_STATE_ERROR(70002,"query zookeeper state error", "查询zookeeper状态错误"), - CREATE_ACCESS_TOKEN_ERROR(70010,"create access token error"), - GENERATE_TOKEN_ERROR(70011,"generate token error"), - QUERY_ACCESSTOKEN_LIST_PAGING_ERROR(70012,"query access token list paging error"), - UPDATE_ACCESS_TOKEN_ERROR(70013,"update access token error"), - DELETE_ACCESS_TOKEN_ERROR(70014,"delete access token error"), - ACCESS_TOKEN_NOT_EXIST(70015, "access token not exist"), + CREATE_ACCESS_TOKEN_ERROR(70010,"create access token error", "创建访问token错误"), + GENERATE_TOKEN_ERROR(70011,"generate token error", "生成token错误"), + QUERY_ACCESSTOKEN_LIST_PAGING_ERROR(70012,"query access token list paging error", "分页查询访问token列表错误"), + UPDATE_ACCESS_TOKEN_ERROR(70013,"update access token error", "更新访问token错误"), + DELETE_ACCESS_TOKEN_ERROR(70014,"delete access token error", "删除访问token错误"), + ACCESS_TOKEN_NOT_EXIST(70015, "access token not exist", "访问token不存在"), - COMMAND_STATE_COUNT_ERROR(80001,"task instance state count error"), + COMMAND_STATE_COUNT_ERROR(80001,"task instance state count error", "查询各状态任务实例数错误"), - QUEUE_COUNT_ERROR(90001,"queue count error"), + QUEUE_COUNT_ERROR(90001,"queue count error", "查询队列数据错误"), - KERBEROS_STARTUP_STATE(100001,"get kerberos startup state error"), + KERBEROS_STARTUP_STATE(100001,"get kerberos startup state error", "获取kerberos启动状态错误"), ; private final int code; - private final String msg; + private final String enMsg; + private final String zhMsg; - private Status(int code, String msg) { + private Status(int code, String enMsg, String zhMsg) { this.code = code; - this.msg = msg; + this.enMsg = enMsg; + this.zhMsg = zhMsg; } public int getCode() { @@ -261,6 +265,10 @@ public enum Status { } public String getMsg() { - return this.msg; + if (Locale.SIMPLIFIED_CHINESE.getLanguage().equals(LocaleContextHolder.getLocale().getLanguage())) { + return this.zhMsg; + } else { + return this.enMsg; + } } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/enums/StatusTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/enums/StatusTest.java index 05d785e1a1..0c9ddff791 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/enums/StatusTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/enums/StatusTest.java @@ -16,7 +16,12 @@ */ package org.apache.dolphinscheduler.api.enums; +import org.junit.Assert; import org.junit.Test; +import org.springframework.context.i18n.LocaleContextHolder; + +import java.util.Locale; + import static org.junit.Assert.*; public class StatusTest { @@ -29,6 +34,11 @@ public class StatusTest { @Test public void testGetMsg() { - assertEquals("success", Status.SUCCESS.getMsg()); + LocaleContextHolder.setLocale(Locale.US); + Assert.assertEquals("success", Status.SUCCESS.getMsg()); + + LocaleContextHolder.setLocale(Locale.SIMPLIFIED_CHINESE); + Assert.assertEquals("成功", Status.SUCCESS.getMsg()); } + } From c9c4402cb65c89fa587a0cbbb55e2ca687cc9605 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Mon, 2 Mar 2020 10:55:40 +0800 Subject: [PATCH 02/37] modify action build --- .github/workflows/ci_backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index e527c3c4a2..afd241d94e 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -51,7 +51,7 @@ jobs: with: java-version: 1.8 - name: Compile - run: mvn -U -B -T 1C clean install -Prelease -Dmaven.compile.fork=true -Dmaven.test.skip=true + run: mvn -B clean compile package -Prelease -Dmaven.test.skip=true License-check: runs-on: ubuntu-latest steps: From 3d4acbc012d186910a3a836423a28bd72251cd59 Mon Sep 17 00:00:00 2001 From: bcai Date: Mon, 2 Mar 2020 15:02:17 +0800 Subject: [PATCH 03/37] fix github action rerun failed #2052 --- .github/workflows/ci_backend.yml | 4 ++-- .github/workflows/ci_e2e.yml | 2 +- .github/workflows/ci_frontend.yml | 4 ++-- .github/workflows/ci_ut.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index afd241d94e..1ca15c2813 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -45,7 +45,7 @@ jobs: Compile-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: @@ -55,7 +55,7 @@ jobs: License-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 13f12641fe..09f66fab11 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: submodules: true - uses: actions/cache@v1 diff --git a/.github/workflows/ci_frontend.yml b/.github/workflows/ci_frontend.yml index fab75c6341..1f959b40eb 100644 --- a/.github/workflows/ci_frontend.yml +++ b/.github/workflows/ci_frontend.yml @@ -34,7 +34,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Node.js uses: actions/setup-node@v1 with: @@ -49,7 +49,7 @@ jobs: License-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 6f3eda0362..6451c2f8bb 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: submodules: true - uses: actions/cache@v1 From 8b894de3d2caf644818d7c84444327a43d864d16 Mon Sep 17 00:00:00 2001 From: bcai Date: Mon, 2 Mar 2020 15:18:35 +0800 Subject: [PATCH 04/37] remove submodules for checkout v2 --- .github/workflows/ci_e2e.yml | 2 -- .github/workflows/ci_ut.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 09f66fab11..5282681fb6 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -30,8 +30,6 @@ jobs: steps: - uses: actions/checkout@v2 - with: - submodules: true - uses: actions/cache@v1 with: path: ~/.m2/repository diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 6451c2f8bb..45ff3ca29d 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -30,8 +30,6 @@ jobs: steps: - uses: actions/checkout@v2 - with: - submodules: true - uses: actions/cache@v1 with: path: ~/.m2/repository From fbf05de39b5c3e2060f1a145dc7598c739916db6 Mon Sep 17 00:00:00 2001 From: Tboy Date: Tue, 3 Mar 2020 12:27:07 +0800 Subject: [PATCH 05/37] delete duplicate startLocalZkServer (#2062) --- dolphinscheduler-service/src/test/java/queue/ZKServer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-service/src/test/java/queue/ZKServer.java b/dolphinscheduler-service/src/test/java/queue/ZKServer.java index 65fb95c02b..3d503d5691 100644 --- a/dolphinscheduler-service/src/test/java/queue/ZKServer.java +++ b/dolphinscheduler-service/src/test/java/queue/ZKServer.java @@ -80,7 +80,6 @@ public class ZKServer { */ public static void startLocalZkServer(final int port) { startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + File.separator + "test-" + System.currentTimeMillis()); - startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + "test-" + System.currentTimeMillis()); } /** From 3277b90647296ab3c72d74e1b22d2ac22c394eba Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Tue, 3 Mar 2020 14:38:21 +0800 Subject: [PATCH 06/37] Change DOM label (#2064) * Change DOM label * Change name to lowercase --- .../pages/dag/_source/formModel/formModel.vue | 2 +- .../formModel/tasks/_source/nodeStatus.vue | 10 +++++----- .../dag/_source/formModel/tasks/conditions.vue | 15 +++++---------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 2500ce5772..ec228fa7ab 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -280,7 +280,7 @@ import mDependent from './tasks/dependent' import mHttp from './tasks/http' import mDatax from './tasks/datax' - import mConditions from './tasks/CONDITIONS' + import mConditions from './tasks/conditions' import mSqoop from './tasks/sqoop' import mSubProcess from './tasks/sub_process' import mSelectInput from './_source/selectInput' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue index 4afb8b46c5..fa7ee89e98 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue @@ -27,17 +27,17 @@ - + - + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue index 4ac04d91a6..e4f9b3f094 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue @@ -19,16 +19,12 @@
{{$t('Custom Parameters')}}
-
+
@@ -44,14 +40,13 @@ @click="!isDetails && _setRelation($index)"> {{el.relation === 'AND' ? $t('and') : $t('or')}} - -  - + Date: Wed, 4 Mar 2020 09:28:23 +0800 Subject: [PATCH 07/37] 1. add input cache for datax task (#2034) 2. add cache for codemirror component, include python, shell and sql task. --- .../dag/_source/formModel/tasks/datax.vue | 49 ++++++++++++++++++- .../dag/_source/formModel/tasks/python.vue | 34 +++++++++++-- .../dag/_source/formModel/tasks/shell.vue | 29 +++++++++-- .../pages/dag/_source/formModel/tasks/sql.vue | 44 ++++++++++++++++- 4 files changed, 145 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue index ce918f49cf..959610f95a 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue @@ -232,6 +232,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-sql-mirror', { mode: 'sql', @@ -249,9 +251,34 @@ // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', () => { + this._cacheParams() + }) + editor.setValue(this.sql) return editor + }, + _cacheParams () { + this.$emit('on-cache-params', { + dsType: this.dsType, + dataSource: this.rtDatasource, + dtType: this.dtType, + dataTarget: this.rtDatatarget, + sql: editor?editor.getValue():'', + targetTable: this.targetTable, + jobSpeedByte: this.jobSpeedByte * 1024, + jobSpeedRecord: this.jobSpeedRecord, + preStatements: this.preStatements, + postStatements: this.postStatements + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) + } } }, created () { @@ -286,7 +313,27 @@ editor.off($('.code-sql-mirror'), 'keypress', this.keypress) } }, - computed: {}, + watch: { + //Watch the cacheParams + cacheParams (val) { + this._cacheParams(); + } + }, + computed: { + cacheParams () { + return { + dsType: this.dsType, + dataSource: this.rtDatasource, + dtType: this.dtType, + dataTarget: this.rtDatatarget, + targetTable: this.targetTable, + jobSpeedByte: this.jobSpeedByte * 1024, + jobSpeedRecord: this.jobSpeedRecord, + preStatements: this.preStatements, + postStatements: this.postStatements + } + } + }, components: { mListBox, mDatasource, mLocalParams, mStatementList, mSelectInput } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index e565b4a6bd..6f495d22a0 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -129,6 +129,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-python-mirror', { mode: 'python', @@ -143,26 +145,45 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', this.changes) + editor.setValue(this.rawScript) return editor + }, + _cacheParams () { + this.$emit('on-cache-params', { + resourceList: this.cacheResourceList, + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-python-mirror'), 'keypress', this.keypress) + editor.off($('.code-python-mirror'), 'changes', this.changes) + } } }, watch: { //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, computed: { cacheParams () { return { resourceList: this.cacheResourceList, - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' + localParams: this.localParams } } }, @@ -193,8 +214,11 @@ }, 200) }, destroyed () { - editor.toTextArea() // Uninstall - editor.off($('.code-python-mirror'), 'keypress', this.keypress) + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-python-mirror'), 'keypress', this.keypress) + editor.off($('.code-python-mirror'), 'changes', this.changes) + } }, components: { mLocalParams, mListBox, mResources } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index ad40c586b9..a4b20f3310 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -163,6 +163,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-shell-mirror', { mode: 'shell', @@ -177,25 +179,45 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + + editor.on('changes', this.changes) + editor.setValue(this.rawScript) return editor + }, + _cacheParams () { + this.$emit('on-cache-params', { + resourceList: this.cacheResourceList, + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) + } } }, watch: { //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, computed: { cacheParams () { return { resourceList: this.cacheResourceList, - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' + localParams: this.localParams } } }, @@ -229,6 +251,7 @@ if (editor) { editor.toTextArea() // Uninstall editor.off($('.code-shell-mirror'), 'keypress', this.keypress) + editor.off($('.code-shell-mirror'), 'changes', this.changes) } }, components: { mLocalParams, mListBox, mResources, mScriptBox } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue index eb5c40992e..843c1eaf1f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue @@ -318,6 +318,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-sql-mirror', { mode: 'sql', @@ -332,9 +334,15 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', this.changes) + editor.setValue(this.sql) return editor @@ -351,6 +359,38 @@ this.receivers = res.receivers && res.receivers.split(',') || [] this.receiversCc = res.receiversCc && res.receiversCc.split(',') || [] }) + }, + _cacheParams () { + this.$emit('on-cache-params', { + type: this.type, + datasource: this.rtDatasource, + sql: editor ? editor.getValue() : '', + udfs: this.udfs, + sqlType: this.sqlType, + title: this.title, + receivers: this.receivers.join(','), + receiversCc: this.receiversCc.join(','), + showType: (() => { + + let showType = this.showType + if (showType.length === 2 && showType[0] === 'ATTACHMENT') { + return [showType[1], showType[0]].join(',') + } else { + return showType.join(',') + } + })(), + localParams: this.localParams, + connParams: this.connParams, + preStatements: this.preStatements, + postStatements: this.postStatements + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) + } } }, watch: { @@ -373,7 +413,7 @@ }, //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, created () { @@ -418,6 +458,7 @@ if (editor) { editor.toTextArea() // Uninstall editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) } }, computed: { @@ -425,7 +466,6 @@ return { type: this.type, datasource: this.rtDatasource, - sql: editor ? editor.getValue() : '', udfs: this.udfs, sqlType: this.sqlType, title: this.title, From b8014d91cf7c03ab821583f47f84a8af6606b675 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Thu, 5 Mar 2020 11:04:00 +0800 Subject: [PATCH 08/37] add workflow id, name for email content (#2071) * add tenantService UT * fix spell * #464 email content * modify mysql1.2.2 ddl --- .../dolphinscheduler/api/service/TenantServiceTest.java | 8 ++++++++ .../api/utils/ZookeeperMonitorUtilsTest.java | 2 +- .../java/org/apache/dolphinscheduler/dao/AlertDao.java | 6 ++++-- .../server/master/runner/MasterTaskExecThread.java | 4 +++- .../dolphinscheduler/server/utils/AlertManager.java | 4 +++- sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql | 2 +- .../1.2.2_schema/postgresql/dolphinscheduler_ddl.sql | 2 +- 7 files changed, 21 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index 31c8c0222d..d6fb6b219c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -175,6 +175,14 @@ public class TenantServiceTest { logger.info(result.toString()); List tenantList = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList)); + + Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(getList()); + Map successRes = tenantService.queryTenantList("1"); + Assert.assertEquals(Status.SUCCESS,successRes.get(Constants.STATUS)); + + Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(null); + Map tenantNotExistRes = tenantService.queryTenantList("1"); + Assert.assertEquals(Status.TENANT_NOT_EXIST,tenantNotExistRes.get(Constants.STATUS)); } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java index b3626fa8a9..0d89d4b6e3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java @@ -28,7 +28,7 @@ public class ZookeeperMonitorUtilsTest { @Test - public void testGetMasterLsit(){ + public void testGetMasterList(){ ZookeeperMonitor zookeeperMonitor = new ZookeeperMonitor(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 19dbf46a6b..1a8c09e611 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -144,9 +144,11 @@ public class AlertDao extends AbstractBaseDao { * @param taskId taskId * @param taskName taskName */ - public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc,int taskId,String taskName){ + public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc, int processInstanceId, + String processInstanceName, int taskId,String taskName){ Alert alert = new Alert(); - String content = String.format("[{'id':'%d','name':'%s','event':'timeout','warnLevel':'middle'}]",taskId,taskName); + String content = String.format("[{'process instance id':'%d','task name':'%s','task id':'%d','task name':'%s'," + + "'event':'timeout','warnLevel':'middle'}]", processInstanceId, processInstanceName, taskId, taskName); alert.setTitle("Task Timeout Warn"); alert.setShowType(ShowType.TABLE); alert.setContent(content); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 66d1a3f4c2..37262c05e2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -128,7 +128,9 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { // process define ProcessDefinition processDefine = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); // send warn mail - alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(),processDefine.getReceiversCc(),taskInstance.getId(),taskInstance.getName()); + alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(), + processDefine.getReceiversCc(), processInstance.getId(), processInstance.getName(), + taskInstance.getId(),taskInstance.getName()); checkTimeout = false; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java index f3441edd17..050af1eec5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java @@ -133,6 +133,8 @@ public class AlertManager { continue; } LinkedHashMap failedTaskMap = new LinkedHashMap(); + failedTaskMap.put("process instance id", String.valueOf(processInstance.getId())); + failedTaskMap.put("process instance name", processInstance.getName()); failedTaskMap.put("task id", String.valueOf(task.getId())); failedTaskMap.put("task name", task.getName()); failedTaskMap.put("task type", task.getTaskType()); @@ -193,7 +195,7 @@ public class AlertManager { logger.info("add alert to db , alert : {}", alert.toString()); }catch (Exception e){ - logger.error("send alert failed! " + e); + logger.error("send alert failed:{} ", e.getMessage()); } } diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql index 790a9a8ada..e5785d9997 100644 --- a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -36,7 +36,7 @@ delimiter ; CALL uc_dolphin_T_t_ds_process_definition_A_modify_by; DROP PROCEDURE uc_dolphin_T_t_ds_process_definition_A_modify_by; --- uc_dolphin_T_t_ds_process_definition_A_modify_by +-- uc_dolphin_T_t_ds_task_instance_A_executor_id drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_instance_A_executor_id; delimiter d// CREATE PROCEDURE uc_dolphin_T_t_ds_task_instance_A_executor_id() diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql index cbe7c22bbe..70471cb26d 100644 --- a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -32,7 +32,7 @@ delimiter ; SELECT uc_dolphin_T_t_ds_process_definition_A_modify_by(); DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_definition_A_modify_by(); --- uc_dolphin_T_t_ds_process_definition_A_modify_by +-- uc_dolphin_T_t_ds_task_instance_A_executor_id delimiter d// CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_task_instance_A_executor_id() RETURNS void AS $$ BEGIN From 3e5a939913527c6aaafca5f170b2460cae2de51f Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Thu, 5 Mar 2020 11:07:44 +0800 Subject: [PATCH 09/37] delete unused import (#2066) --- .../alert/template/impl/DefaultHTMLTemplate.java | 1 - .../apache/dolphinscheduler/alert/utils/JSONUtilsTest.java | 1 - .../apache/dolphinscheduler/api/service/MonitorService.java | 1 - .../api/controller/LoginControllerTest.java | 1 - .../api/controller/MonitorControllerTest.java | 1 - .../api/controller/ProjectControllerTest.java | 2 -- .../api/controller/ResourcesControllerTest.java | 2 -- .../api/controller/TaskRecordControllerTest.java | 5 ----- .../api/controller/WorkerGroupControllerTest.java | 1 - .../dolphinscheduler/api/utils/FourLetterWordMainTest.java | 1 - .../apache/dolphinscheduler/common/log/WorkerLogFilter.java | 3 --- .../common/log/TaskLogDiscriminatorTest.java | 2 -- .../dolphinscheduler/common/log/TaskLogFilterTest.java | 1 - .../apache/dolphinscheduler/common/utils/HttpUtilsTest.java | 1 - .../dolphinscheduler/common/utils/SchemaUtilsTest.java | 3 --- .../dolphinscheduler/common/utils/StringUtilsTest.java | 2 -- .../common/utils/TaskParametersUtilsTest.java | 3 --- .../dolphinscheduler/dao/entity/ProcessDefinition.java | 1 - .../org/apache/dolphinscheduler/dao/entity/UdfFunc.java | 1 - .../dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java | 1 - .../dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java | 1 - .../dolphinscheduler/dao/mapper/ProjectUserMapperTest.java | 2 -- .../apache/dolphinscheduler/dao/mapper/QueueMapperTest.java | 1 - .../remote/command/ExecuteTaskRequestCommand.java | 2 +- .../remote/command/ExecuteTaskResponseCommand.java | 2 +- .../org/apache/dolphinscheduler/remote/command/Ping.java | 1 - .../remote/command/log/GetLogBytesRequestCommand.java | 1 - .../remote/command/log/RollViewLogRequestCommand.java | 1 - .../remote/command/log/ViewLogRequestCommand.java | 1 - .../server/worker/task/sqoop/SqoopTaskTest.java | 6 ------ .../dolphinscheduler/service/log/LogClientService.java | 1 - 31 files changed, 2 insertions(+), 51 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java index 428fa4cb62..79123a1e80 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.alert.template.impl; import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.alert.utils.JSONUtils; -import org.apache.dolphinscheduler.alert.utils.MailUtils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java index cb63a22d79..277c42b5bd 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.alert.utils; -import org.junit.After; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java index 118c5ce936..3370961fd4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java @@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java index bddc055de3..f5a28d01ae 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java @@ -28,7 +28,6 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java index 8fc055daf1..41674d3e54 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import com.alibaba.fastjson.JSONObject; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java index bab82df59d..42cdd1705a 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java @@ -29,8 +29,6 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import javax.ws.rs.POST; - import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index 46d85f4d8d..40bdd5490d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import com.alibaba.fastjson.JSONObject; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java index 8bddb0f905..943e14607b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java @@ -18,9 +18,6 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.FailureStrategy; -import org.apache.dolphinscheduler.common.enums.Priority; -import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.junit.Assert; import org.junit.Test; @@ -31,9 +28,7 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import static org.junit.Assert.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java index 65ecd3f759..8517ad2d45 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java @@ -29,7 +29,6 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import static org.junit.Assert.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java index e8adc6ca9c..69d1f21c37 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.utils; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java index abcc8bc619..6240ed9a2e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java @@ -20,9 +20,6 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.filter.Filter; import ch.qos.logback.core.spi.FilterReply; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; - -import java.util.Arrays; /** * worker log filter diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java index 8745a4f6b4..ff298000f5 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java @@ -27,8 +27,6 @@ import org.slf4j.Marker; import java.util.Map; -import static org.junit.Assert.*; - public class TaskLogDiscriminatorTest { /** diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java index 52767074da..5cca6403c8 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java @@ -21,7 +21,6 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.IThrowableProxy; import ch.qos.logback.classic.spi.LoggerContextVO; import ch.qos.logback.core.spi.FilterReply; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.junit.Assert; import org.junit.Test; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java index 20994ac99f..41049ec335 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java index 907a09e458..7885806b96 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java @@ -19,15 +19,12 @@ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileNotFoundException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java index 947e7310db..eca22def30 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; import org.junit.Test; -import java.util.ArrayList; - public class StringUtilsTest { @Test public void testIsNotEmpty() { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java index db4a86bc26..b316b17469 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index 6e7ea7d64f..dbb880c025 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -29,7 +29,6 @@ import com.baomidou.mybatisplus.core.toolkit.StringUtils; import java.util.Date; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java index 9093034ae8..3518676337 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.TableField; import org.apache.dolphinscheduler.common.enums.UdfType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java index 5e5277b997..58bd673fc5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java index c0dc905848..5db273642a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java index 102d8cd5db..6e995ef3c7 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.junit.Assert; import org.junit.Test; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java index 62b034a081..30d2be03e0 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.Queue; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java index beec055403..93c536c347 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.List; import java.util.concurrent.atomic.AtomicLong; /** * execute task request command */ public class ExecuteTaskRequestCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * application name */ private String applicationName; /** * group name */ private String groupName; /** * task name */ private String taskName; /** * connector port */ private int connectorPort; /** * description info */ private String description; /** * class name */ private String className; /** * method name */ private String methodName; /** * parameters */ private String params; /** * shard itemds */ private List shardItems; public List getShardItems() { return shardItems; } public void setShardItems(List shardItems) { this.shardItems = shardItems; } public String getParams() { return params; } public void setParams(String params) { this.params = params; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getApplicationName() { return applicationName; } public void setApplicationName(String applicationName) { this.applicationName = applicationName; } public String getGroupName() { return groupName; } public void setGroupName(String groupName) { this.groupName = groupName; } public String getTaskName() { return taskName; } public void setTaskName(String taskName) { this.taskName = taskName; } public int getConnectorPort() { return connectorPort; } public void setConnectorPort(int connectorPort) { this.connectorPort = connectorPort; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getClassName() { return className; } public void setClassName(String className) { this.className = className; } public String getMethodName() { return methodName; } public void setMethodName(String methodName) { this.methodName = methodName; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.List; /** * execute task request command */ public class ExecuteTaskRequestCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * application name */ private String applicationName; /** * group name */ private String groupName; /** * task name */ private String taskName; /** * connector port */ private int connectorPort; /** * description info */ private String description; /** * class name */ private String className; /** * method name */ private String methodName; /** * parameters */ private String params; /** * shard itemds */ private List shardItems; public List getShardItems() { return shardItems; } public void setShardItems(List shardItems) { this.shardItems = shardItems; } public String getParams() { return params; } public void setParams(String params) { this.params = params; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getApplicationName() { return applicationName; } public void setApplicationName(String applicationName) { this.applicationName = applicationName; } public String getGroupName() { return groupName; } public void setGroupName(String groupName) { this.groupName = groupName; } public String getTaskName() { return taskName; } public void setTaskName(String taskName) { this.taskName = taskName; } public int getConnectorPort() { return connectorPort; } public void setConnectorPort(int connectorPort) { this.connectorPort = connectorPort; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getClassName() { return className; } public void setClassName(String className) { this.className = className; } public String getMethodName() { return methodName; } public void setMethodName(String methodName) { this.methodName = methodName; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java index 7e35fa6e75..7543fc3d0e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.concurrent.atomic.AtomicLong; /** * execute task response command */ public class ExecuteTaskResponseCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * return result */ private Object result; /** * received time */ private long receivedTime; /** * execute count */ private int executeCount; /** * execute time */ private long executeTime; public String getAttemptId() { return attemptId; } public void setAttemptId(String attemptId) { this.attemptId = attemptId; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public Object getResult() { return result; } public void setResult(Object result) { this.result = result; } public long getReceivedTime() { return receivedTime; } public void setReceivedTime(long receivedTime) { this.receivedTime = receivedTime; } public int getExecuteCount() { return executeCount; } public void setExecuteCount(int executeCount) { this.executeCount = executeCount; } public long getExecuteTime() { return executeTime; } public void setExecuteTime(long executeTime) { this.executeTime = executeTime; } public Command convert2Command(long opaque){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; /** * execute task response command */ public class ExecuteTaskResponseCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * return result */ private Object result; /** * received time */ private long receivedTime; /** * execute count */ private int executeCount; /** * execute time */ private long executeTime; public String getAttemptId() { return attemptId; } public void setAttemptId(String attemptId) { this.attemptId = attemptId; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public Object getResult() { return result; } public void setResult(Object result) { this.result = result; } public long getReceivedTime() { return receivedTime; } public void setReceivedTime(long receivedTime) { this.receivedTime = receivedTime; } public int getExecuteCount() { return executeCount; } public void setExecuteCount(int executeCount) { this.executeCount = executeCount; } public long getExecuteTime() { return executeTime; } public void setExecuteTime(long executeTime) { this.executeTime = executeTime; } public Command convert2Command(long opaque){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java index c50413e98a..c5e4d075af 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java @@ -21,7 +21,6 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * ping machine diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java index 4cc32ed42a..20cf8d9102 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * get log bytes request command diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java index 621d35a804..433c4a0b1f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * roll view log request command diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java index 8835348ee3..b4773d0c0d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * view log request command diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java index f8688e7c0c..511102e4b5 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java @@ -19,12 +19,6 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop; import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; -import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHdfsParameter; -import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHiveParameter; -import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter; -import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHdfsParameter; -import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHiveParameter; -import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index 5daf535625..01a49910df 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.service.log; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.log.*; From d5c795bd45b490720ead48a5fe68430d4402fd5f Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Thu, 5 Mar 2020 11:10:48 +0800 Subject: [PATCH 10/37] fix ugly logger usage (#2063) * fix ugly logger usage * remove space --- .../api/controller/DataAnalysisController.java | 4 ++-- .../dolphinscheduler/api/controller/ExecutorController.java | 2 +- .../apache/dolphinscheduler/remote/NettyRemotingServer.java | 2 +- .../dolphinscheduler/remote/handler/NettyClientHandler.java | 2 +- .../dolphinscheduler/remote/handler/NettyServerHandler.java | 2 +- .../server/master/runner/MasterExecThread.java | 2 +- .../dolphinscheduler/service/process/ProcessService.java | 6 +++--- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java index 1b1dc65e01..f93e7d6944 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java @@ -103,7 +103,7 @@ public class DataAnalysisController extends BaseController{ @RequestParam(value="endDate", required=false) String endDate, @RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){ try{ - logger.info("count process instance state, user:{}, start date: {}, end date:{}, project id", + logger.info("count process instance state, user:{}, start date: {}, end date:{}, project id:{}", loginUser.getUserName(), startDate, endDate, projectId); Map result = dataAnalysisService.countProcessInstanceStateByProject(loginUser, projectId, startDate, endDate); return returnDataList(result); @@ -129,7 +129,7 @@ public class DataAnalysisController extends BaseController{ public Result countDefinitionByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){ try{ - logger.info("count process definition , user:{}, project id", + logger.info("count process definition , user:{}, project id:{}", loginUser.getUserName(), projectId); Map result = dataAnalysisService.countDefinitionByUser(loginUser, projectId); return returnDataList(result); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index 93b095ddf8..cae4993942 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -173,7 +173,7 @@ public class ExecutorController extends BaseController { @ResponseStatus(HttpStatus.OK) public Result startCheckProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "processDefinitionId") int processDefinitionId) { - logger.info("login user {}, check process definition", loginUser.getUserName(), processDefinitionId); + logger.info("login user {}, check process definition {}", loginUser.getUserName(), processDefinitionId); try { Map result = execService.startCheckByProcessDefinedId(processDefinitionId); return returnDataList(result); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java index 29b2317633..dbeb318f2d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java @@ -145,7 +145,7 @@ public class NettyRemotingServer { try { future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); } catch (Exception e) { - logger.error("NettyRemotingServer bind fail {}, exit", e); + logger.error("NettyRemotingServer bind fail {}, exit",e.getMessage(), e); throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); } if (future.isSuccess()) { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java index d5d0d4df83..80e561d05c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java @@ -107,7 +107,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { */ @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - logger.error("exceptionCaught : {}", cause); + logger.error("exceptionCaught : {}",cause.getMessage(), cause); nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel())); ctx.channel().close(); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java index eabd6560de..22251c35c1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java @@ -140,7 +140,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { */ @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - logger.error("exceptionCaught : {}", cause); + logger.error("exceptionCaught : {}",cause.getMessage(), cause); ctx.channel().close(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 4b22b27bec..0b81a30191 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -565,7 +565,7 @@ public class MasterExecThread implements Runnable { TaskInstance taskInstance = completeTaskList.get(nodeName); if(taskInstance == null){ - logger.error("task instance cannot find, please check it!", nodeName); + logger.error("task instance {} cannot find, please check it!", nodeName); return conditionTaskList; } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index b589cd4295..8bcd64f1fd 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -116,12 +116,12 @@ public class ProcessService { ProcessInstance processInstance = constructProcessInstance(command, host); //cannot construct process instance, return null; if(processInstance == null){ - logger.error("scan command, command parameter is error: %s", command.toString()); + logger.error("scan command, command parameter is error: {}", command); moveToErrorCommand(command, "process instance is null"); return null; } if(!checkThreadNum(command, validThreadNum)){ - logger.info("there is not enough thread for this command: {}",command.toString() ); + logger.info("there is not enough thread for this command: {}", command); return setWaitingThreadProcess(command, processInstance); } processInstance.setCommandType(command.getCommandType()); @@ -991,7 +991,7 @@ public class ProcessService { return insertQueueResult; }catch (Exception e){ logger.error("submit task to queue Exception: ", e); - logger.error("task queue error : %s", JSONUtils.toJson(taskInstance)); + logger.error("task queue error : {}", JSONUtils.toJson(taskInstance)); return false; } } From 18d7ff274b8757b2e37f42790473ca472cf66274 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Thu, 5 Mar 2020 11:33:24 +0800 Subject: [PATCH 11/37] action push delete --- .github/workflows/ci_e2e.yml | 2 +- .github/workflows/ci_ut.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 5282681fb6..7d9c5e3e72 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -15,7 +15,7 @@ # limitations under the License. # -on: ["push", "pull_request"] +on: ["pull_request"] env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 45ff3ca29d..e7e47437d4 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -15,7 +15,7 @@ # limitations under the License. # -on: ["push", "pull_request"] +on: ["pull_request"] env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler From 6ec85540bf6d367fa11ede093325c199afc930b5 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 16:36:20 +0800 Subject: [PATCH 12/37] fix typo (#2093) --- .../apache/dolphinscheduler/server/worker/task/TaskProps.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java index 8e5644ed9c..edec419384 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java @@ -78,7 +78,7 @@ public class TaskProps { private Map definedParams; /** - * task path + * task app id */ private String taskAppId; From 4230d5a737b7889254806969ae153817fd19d181 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 17:32:40 +0800 Subject: [PATCH 13/37] add embedded database h2 and update embedded zookeeper (#2082) add embedded database h2 and update embedded zookeeper --- dolphinscheduler-dao/pom.xml | 19 +++- .../src/main/resources/application.properties | 4 + .../server/zk/StandaloneZKServerForTest.java | 100 ------------------ dolphinscheduler-service/pom.xml | 5 + .../service/zk}/ZKServer.java | 87 ++++++++------- .../src/main/resources/quartz.properties | 4 + .../service/zk/ZKServerTest.java | 41 +++++++ .../test/java/queue/BaseTaskQueueTest.java | 1 + pom.xml | 14 ++- sql/h2.mv.db | Bin 0 -> 110592 bytes 10 files changed, 121 insertions(+), 154 deletions(-) delete mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java rename dolphinscheduler-service/src/{test/java/queue => main/java/org/apache/dolphinscheduler/service/zk}/ZKServer.java (67%) create mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java create mode 100644 sql/h2.mv.db diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 20d19410e2..fd43458f87 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -44,6 +44,12 @@ com.baomidou mybatis-plus-boot-starter ${mybatis-plus.version} + + + org.apache.logging.log4j + log4j-to-slf4j + + org.postgresql @@ -71,6 +77,14 @@ log4j-api org.apache.logging.log4j + + org.springframework.boot + spring-boot-starter-tomcat + + + org.apache.logging.log4j + log4j-to-slf4j + @@ -78,7 +92,10 @@ mysql mysql-connector-java - + + com.h2database + h2 + com.alibaba druid diff --git a/dolphinscheduler-dao/src/main/resources/application.properties b/dolphinscheduler-dao/src/main/resources/application.properties index 34bb9f916b..06b0ee94d5 100644 --- a/dolphinscheduler-dao/src/main/resources/application.properties +++ b/dolphinscheduler-dao/src/main/resources/application.properties @@ -23,6 +23,10 @@ spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +# h2 +#spring.datasource.driver-class-name=org.h2.Driver +#spring.datasource.url=jdbc:h2:file:../sql/h2;AUTO_SERVER=TRUE + spring.datasource.username=test spring.datasource.password=test diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java deleted file mode 100644 index 679862f102..0000000000 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.server.zk; - -import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; -import org.apache.zookeeper.server.ServerConfig; -import org.apache.zookeeper.server.ZooKeeperServerMain; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.junit.Before; -import org.junit.Ignore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.Properties; - - -/** - * just for test - */ -@Ignore -public class StandaloneZKServerForTest { - - private static final Logger logger = LoggerFactory.getLogger(StandaloneZKServerForTest.class); - - private static volatile ZooKeeperServerMain zkServer = null; - - - @Before - public void before() { - logger.info("standalone zookeeper server for test service start "); - - ThreadPoolExecutors.getInstance().execute(new Runnable() { - @Override - public void run() { - - //delete zk data dir ? - File zkFile = new File(System.getProperty("java.io.tmpdir"), "zookeeper"); - - startStandaloneServer("2000", zkFile.getAbsolutePath(), "2181", "10", "5"); - } - }); - - } - - - /** - * start zk server - * @param tickTime zookeeper ticktime - * @param dataDir zookeeper data dir - * @param clientPort zookeeper client port - * @param initLimit zookeeper init limit - * @param syncLimit zookeeper sync limit - */ - private void startStandaloneServer(String tickTime, String dataDir, String clientPort, String initLimit, String syncLimit) { - Properties props = new Properties(); - props.setProperty("tickTime", tickTime); - props.setProperty("dataDir", dataDir); - props.setProperty("clientPort", clientPort); - props.setProperty("initLimit", initLimit); - props.setProperty("syncLimit", syncLimit); - - QuorumPeerConfig quorumConfig = new QuorumPeerConfig(); - try { - quorumConfig.parseProperties(props); - - if(zkServer == null ){ - - synchronized (StandaloneZKServerForTest.class){ - if(zkServer == null ){ - zkServer = new ZooKeeperServerMain(); - final ServerConfig config = new ServerConfig(); - config.readFrom(quorumConfig); - zkServer.runFromConfig(config); - } - } - - } - - } catch (Exception e) { - logger.error("start standalone server fail!", e); - } - } - - -} \ No newline at end of file diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 7d775d5497..03bb94dc29 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -48,6 +48,11 @@ + + org.apache.logging.log4j + log4j-core + + org.quartz-scheduler quartz-jobs diff --git a/dolphinscheduler-service/src/test/java/queue/ZKServer.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java similarity index 67% rename from dolphinscheduler-service/src/test/java/queue/ZKServer.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java index 3d503d5691..96331405d4 100644 --- a/dolphinscheduler-service/src/test/java/queue/ZKServer.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java @@ -14,11 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.zk; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.slf4j.Logger; @@ -26,27 +24,45 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; /** - * just for test + * just speed experience version + * embedded zookeeper service */ public class ZKServer { - private static final Logger logger = LoggerFactory.getLogger(ZKServer.class); private static volatile PublicZooKeeperServerMain zkServer = null; public static final int DEFAULT_ZK_TEST_PORT = 2181; - public static final String DEFAULT_ZK_STR = "localhost:" + DEFAULT_ZK_TEST_PORT; - private static String dataDir = null; private static final AtomicBoolean isStarted = new AtomicBoolean(false); + public static void main(String[] args) { + if(!isStarted()){ + ZKServer.start(); + + /** + * register hooks, which are called before the process exits + */ + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + stop(); + } + })); + }else{ + logger.info("zk server aleady started"); + } + } + + /** + * start service + */ public static void start() { try { startLocalZkServer(DEFAULT_ZK_TEST_PORT); @@ -79,7 +95,8 @@ public class ZKServer { * @param port The port to listen on */ public static void startLocalZkServer(final int port) { - startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + File.separator + "test-" + System.currentTimeMillis()); + + startLocalZkServer(port, System.getProperty("user.dir") +"/zookeeper_data", ZooKeeperServer.DEFAULT_TICK_TIME,"20"); } /** @@ -87,48 +104,28 @@ public class ZKServer { * * @param port The port to listen on * @param dataDirPath The path for the Zk data directory + * @param tickTime zk tick time + * @param maxClientCnxns zk max client connections */ - private static synchronized void startLocalZkServer(final int port, final String dataDirPath) { + private static synchronized void startLocalZkServer(final int port, final String dataDirPath,final int tickTime,String maxClientCnxns) { if (zkServer != null) { throw new RuntimeException("Zookeeper server is already started!"); } - try { - zkServer = new PublicZooKeeperServerMain(); - logger.info("Zookeeper data path : {} ", dataDirPath); - dataDir = dataDirPath; - final String[] args = new String[]{Integer.toString(port), dataDirPath}; - Thread init = new Thread(new Runnable() { - @Override - public void run() { - try { - zkServer.initializeAndRun(args); - } catch (QuorumPeerConfig.ConfigException e) { - logger.warn("Caught exception while starting ZK", e); - } catch (IOException e) { - logger.warn("Caught exception while starting ZK", e); - } - } - }, "init-zk-thread"); - init.start(); - } catch (Exception e) { - logger.warn("Caught exception while starting ZK", e); - throw new RuntimeException(e); - } - - CuratorFramework zkClient = CuratorFrameworkFactory.builder() - .connectString(DEFAULT_ZK_STR) - .retryPolicy(new ExponentialBackoffRetry(10,100)) - .sessionTimeoutMs(1000 * 30) - .connectionTimeoutMs(1000 * 30) - .build(); + zkServer = new PublicZooKeeperServerMain(); + logger.info("Zookeeper data path : {} ", dataDirPath); + dataDir = dataDirPath; + final String[] args = new String[]{Integer.toString(port), dataDirPath, Integer.toString(tickTime), maxClientCnxns}; try { - zkClient.blockUntilConnected(10, TimeUnit.SECONDS); - zkClient.close(); - } catch (InterruptedException ignore) { + logger.info("Zookeeper server started "); + isStarted.compareAndSet(false, true); + + zkServer.initializeAndRun(args); + } catch (QuorumPeerConfig.ConfigException e) { + logger.warn("Caught exception while starting ZK", e); + } catch (IOException e) { + logger.warn("Caught exception while starting ZK", e); } - isStarted.compareAndSet(false, true); - logger.info("zk server started"); } /** diff --git a/dolphinscheduler-service/src/main/resources/quartz.properties b/dolphinscheduler-service/src/main/resources/quartz.properties index 9c8930b647..b01be394c6 100644 --- a/dolphinscheduler-service/src/main/resources/quartz.properties +++ b/dolphinscheduler-service/src/main/resources/quartz.properties @@ -26,6 +26,10 @@ org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinschedul # mysql #org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver #org.quartz.dataSource.myDs.URL = jdbc:mysql://localhost:3306/dolphinscheduler?characterEncoding=utf8 +#h2 +#org.quartz.dataSource.myDs.driver=org.h2.Driver +#org.quartz.dataSource.myDs.URL=jdbc:h2:file:/Users/stone/work/myworkspace/incubator-dolphinscheduler/h2;AUTO_SERVER=TRUE + org.quartz.dataSource.myDs.user = test org.quartz.dataSource.myDs.password = test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java new file mode 100644 index 0000000000..48cde32287 --- /dev/null +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.service.zk; + +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.*; + +@Ignore +public class ZKServerTest { + + @Test + public void start() { + //ZKServer is a process, can't unit test + } + + @Test + public void isStarted() { + + } + + @Test + public void stop() { + ZKServer.stop(); + } +} \ No newline at end of file diff --git a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java b/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java index a0cc457e22..97ab9969a3 100644 --- a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java +++ b/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java @@ -18,6 +18,7 @@ package queue; import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; +import org.apache.dolphinscheduler.service.zk.ZKServer; import org.junit.*; /** diff --git a/pom.xml b/pom.xml index 8a1e9b98f2..57063a86c2 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ 5.0.5 1.2.61 1.1.14 - 1.3.163 + 1.4.200 1.6 1.1.1 4.4.1 @@ -200,13 +200,6 @@ ${spring.version} test - - com.h2database - h2 - ${h2.version} - test - - org.apache.dolphinscheduler dolphinscheduler-server @@ -349,6 +342,11 @@ ${mysql.connector.version} test + + com.h2database + h2 + ${h2.version} + org.slf4j slf4j-api diff --git a/sql/h2.mv.db b/sql/h2.mv.db new file mode 100644 index 0000000000000000000000000000000000000000..42ea540bd37b54496241f1669d3455f3b9806cf4 GIT binary patch literal 110592 zcmeIbdz2hkeJ9>snrr-$(ZmKDzvyXO9?2e!x*q)+4E8*Fq?YETo*7xPWZFI5)zaY6 zj64rpLMX^@gTcl;Y-|+A7#r9uXE~gcO*Y|`B{^9qyGb}Wu+V7&#_TVTBf)>~k`1=d?&y#>}=V7&#_TVTBf)?45g)&kBX zza^Fd7<~&io$5bQJ35dr;BjmqSI_h<+Q(M0MBr%cki7zz!Sd45>VQ=*Vx0i~nVHO7 z-|GAkdmvXZjYdB2T?=6K9i3ksURqmR9mu54^*_3-;E_w~iSv?)u0O&1o^K|)UOtuR zUHd3|ZrQV!J-0shb@sgRiVAa)$NH@z|5~lhE!h2f;Sv_Mw%Ax2TRPNlBobYBy!jo8%M%x}2JOch2*Z6fS$;ap zT+i@__%isH$sfFYJJ>GS1KSEzI9zkAv;?92SAlk15Xdx)&4mIzS=NU`sd@3IKzVZ8TLl=zVL8Yb$nR zaCNV{M8xo7#PDF`1(!0hBxOLa2~bMXymV!}WKpR4Fj}YwVl*d+av&Q(qv@p);CFER zCd&PRrK9%Z%yMmUrB+|X!k3k6*slU1STw^H%g8F5Jd~qJ&jp7{bOncsw8h|1p4O0y zbOOsYjL5WBa40XAE&~dxktjDG94gWlfkZ77q$$GJ+Q6xy@{aPvOlo2h|DPQj>${Nl zQ|Z#gNOh(>UP_B_@$>ND7(xas)rmnGHoSANGB`X_s-*S~R`!=Cc5E>+nXMx3czJrX zTq#w{6Vo%5bZUBhaBK`EiO^k>L)DSNnL!k=f2K5e;nwc1-c--Vdxk5e!I@HOs!|?D zmZ{yP{i*cS?9fU)l8=*Cp=+$Dl@lNXDa0#J4%)5^p^DW@Xpc*r?W4W z76C2aJ1Ud2Q)v{wCAE3uOUc18TA7)_p|MhE)~Ugfk<{=ct<@lHIhQ>5M@pTi3LR3D zDo>1*_IWZ^>)qY8HI>@Cg+S2yDk0EHcX#yxfuE0qz~Rc|gyaFl@->e?MsI1~RHZaM zU7nl>(s6I(n?Gm+R;a zvtcx?%-p%Ppt+68m+Z@J0M!1U^BPsm)?*F;eNI|ej?GuJ0&Tvc*L2EyOs8{9KWzT; zrJvhCsQrIyA~Ez%_9P3Z+4F)6zQvv!f5e}c&hzKx-~aX%^xt;!7NV0fsgdJ}#AW~H zyoWYik$FxX$+{!qpS-r?JG~uOsCS*81+qKsFmui)fz_8 zwzEYuZ{>>lMsI58;PlQ^ZzfYKwMVF(@P z4S@ekBnJFl!|63YLUY<8pmPf{LtMtiOb}nlCQAEes?5j@u$3`{+#3;Co~)E-0Cse- zC77(DGnIDM^b7HwGE>0~%B({uzl$E^RAv;R|R$3Xzk zL4boGm8Ld`<3bPyi5MxlLDcQQZ42sR>)HgH;JvYJ0u%1K)_DY~aaBK(&oYp#mezs% zb}WRhZkPz4k1`Qj%7{0x0KE~0f}|sau1JFcO@Ze7HJA+Fm)3jK+``ZpEFw|fS=h`9 z#jmDjXY~4}oVqUU$RRrv=#cd_M8`>B--VC-Mj;$kxMuT*OEC3brv0R35dZ|vj94!8LlroS> zrql7*3yLDxD{~VBL`pp>2Li7EJD-ok&Z$9o^e{KfCtBeVnK}!`y1CF<*mlBy;nqar zlC|@{3r8hlWuhxk&|H^jGqM&SJ!2m=?puk5-Bnt8r1XlqB|257u3w3!o7&r_5 z!+luUA2MTB%(l2KWQAV!F&SO5FgP(>>PX>;M)53~aCD9svhZnludB!(1Kd>ILjDMy z_=Vaz$^=zy{bM2?e8a(xUlAVe$Ym<`YKY&V*eW+ry{rWYE0i=3sL)V9DU0$GdotQR5|OrP* zO%Crq3yn*YhiWy;GaE2tFF7qdixd=gMwD*g)&JN#Zyu&+=QYYmI^4Mwt zOOvpWrSpnFH+M*R(2qjp0@$cR zM`uv!ju0$coHsC9JfQULDE&l?CP(RFX>Ug)sLO$aNyvbNC*pvRwZN-FMk^Wo76~YbZ{1{*QgG~^7AddtJTIzwN|g&D=XF2rMv9K>U^WRV&A=HFV^ke7B>c6 z633h#N@nif3tlMQQbAX5Zpm3&-6GvsSffmLS8$_@{)KKIp9+e}5gaF`Sg#e8C$kHR z;95lphG(?xSu!DhWji(WtWQJ(S~!iyrFa;CDR6&?DbSiMo<)G-Jw$F{MMaPPvvm0X zHeluDHcJONNwCgmVuEDBUS5qx$4q&KuD)x-{pp>Pw^jLqpAZybh29KXo;L%S3KtD_ z@6dHWY@JX@G9HDF6K3Iy_D1T>^onf8CLxXfY?+Wn!5ZuPN@XnYL$;o@&ac&B49YzL z=N`2;1GT$iqqZ0bTFz>T*0%$#$Ak9_d5{qcr6{g5kQOOM-vNyNaV&jC3FmrHc#zs? zGYZs0%S&rVqtVK_&Kj zXzhbZch>I$J>`iqUFIC6QDl}=*o8Q*D36)J>D_R;OtEE*QR;3K3%+O3jnd70Scl2i zV>;k4##_p(vO5TtP|wlrc-fno=! zFqV=y3`sl@p|K_IV@V;3>`Vv;sSJ)l27eSsXVbL`)049ma4VM1Zq!z5D@$w3bvs%T zRc%d&m%h!v_*4LbCC35k+*Rbhq8D1k-3cweCq23sAJ^&JcwZfYc5^>gR1HZkarkDivr5334B*e61_BtIOvA z5X8|sQa`LKb-BAfwo;FQaG`{~s@*e+mTi9yb+afC2IutP4gi*x#>8wV#4Ns5SO}6b zA3m9&B@%B=7|j1xCcDUPGDgP91eqjLWDlv3X);4*$zF0B*+=%1+sOfPklaDuNvh;d z>VGRkMDL3ld8sAFl2TCKnUEH!jP8Pr{xpu-AIF?4o$kdvYN--W^B<|LtlGa2z;H?C6yl}8Hfp%+Mkn2fMmD%R z2~qh0I)_b4?<`?lG(?XIXzQ)m-4QPR>p_{`^zlS?0AuaYrKXF8F8}&Y>KKXiRk6=FERgC!*0$m&eJo0qH(4UGQtL#)E&U~GI|{i9;{VL*n&7s zhv#fNR7fn;X&3MqWhy9Rk^oG#HcNHaG~NF1(WQ!U>iK;K7Qzl84YYm8x*l!#nps`13?q}rVDy#GWtL~Le;YjX)cDF&@QM615<)uq!sSV4-xG}cy5 z&koU@F4X;{;=^t17Guk2v31a+?8He77{67A$1&dU^kuFYQB}NDYJb$y5>R+&>G2T_ zaSl2pW{XsyP4~v`J_ZAwNB&y6oyRmfkB8%Q9ug|?R`pXdI_o-+0$s-pDl1Px2+*=`gXr9XHxhgv)0G!Cw<=}fQ?s9?3@`14wlG3mJtd>xN1 zJ-M*K7iWkZZtR25oQ!b2N)E;vWzjM_6BVR@sO(40c84HVa3>P3qDqpHbR>B@Ao^q+ zh~k`+*_f_eEFD;DGhkx5i3KoTnKZ?U|7_ZF0W9Q#+M}Cz8C!UU4 zwijy5%=WnTaguNw_l^W68)5o}2d+0dYS27Ys$ee7J3w{EFy|g3?*v5dh^=V?5fa|a z5;RqS<^xgPb!Zl`bWz7RVlcWBF#1$SFlrr-@UwdS=DE5q$-$BI;KZ}XU^rm)h#ZkZ zC!%AfJEB?&bEh>$5;pf5E54ZQjf_h-{dNCja+$8btdTiVCkzkau-=3 zN5~>sB1g&HWSOjxRkB9zA@`DF)c;!pzT)p-7!92IanRAq0Ew_UAnb`)2y4kB7Q`rF zo#`s1K=tbY*%vzn8S@mJfuaJy*r@94w3HYIL;~#>s-hZz(1F-0s_5Uj#yluq2zi=U z!p8=D9*YYfH+?MVP=Y#x4pIO*O;kI~9~D>~a4dZ26qo`QhU~BCK4xNfmgKBbF0Tyg2e26Y|Q#{h=tmY z^&L}p?*f9p&=G=WM@G*GJZp{S+2RI8zjl<|T|i8HcYDddHB(h`o<@~uSpZrh-R+Ko za1+M^ihm;}b#7z$8h(H}h=;N{0;qhhGpNvI+iZEPSl!pr=!@x?TFh0IZYfa+k-AeE ztb5F~b3a;N1WMi+yDKaO`UPA@``c4uWeHdb+YagjnlFoZi0RmMfQkSyM}e3Z;;aXE z^j%y-PxeERy9Nfw%7gI2wlVcb%Q)^|^<0HG1%e${A(Qj%pcc7CXu5nm)Va84BETzl z?s0lIa5@nir~b{L0%Kxm;d*_Sfy*cdmdp28I7G3YzBz{bVVI(D1t|PhoTXmUeeY}* zZyB8^+gDcVhwaANg1r(wO()G4clm&$p?uy3k0UqbPK)^>1$CCU9s@<5V2m?RG+ z$-_zVNRm97B=1X-_b16?N%Dat`CyVfo+M8s$%m5U$t3x3l6)jdKAI#SOOmIOP0#Zy!_!C zgg8XuMe%*b>;AE3`>tv{pPbsShpnk%H1G~tf-PhSAQe%7GWUW_pj*av)K&!284WE9 zsL&t~vPUgq0(`t2y_`=_65O=}kQ~VoFq?1E3em$1xhW~VF0VKaRY;g~V` zhP&38KTpe4N^RF+OhqtbaB8ZGo40m*s1Dho)k=ly`{zV}cjA1X5H8YHBcb$q>ZC{t z+|1{w2W(j);bk||n(v0z#3c$bZy6iP1odnR@$LZe1a#*EQEognT`Uz!3Fu6PA_Y{| zd!VxZHjaJE*Z9TMRrX?SvCXu(STi_0IRa5{qku$QRj-98tB&jyJ~Q9KEwM8oAq9Z+UI6Kl zI0grG+{A+sbDl*SyHP{+(b2b;>T_Lt}a`a8bw#fSxocoiU}4LML1mprBfY;N6#D9LwNT5qV4XDCMsp#1(MVE$j?cy}l?W5PV%Y`x075behVVhY#!VCy7S z>`CNA#dd)m0IkwEHomsaXKE*Iv~y$Wdf*Z$p~Zhu+E(c0Hj7%APJV!5RWA4Oz#?QK zWW)>xE91BX>w^@mQA(KsBqmtT0IdwJ4^g;&ERF>$cUX8VSY_m0lD#W>DE)kR@jwaV0<@NOdu zix|@D1n8rXp2r)LA?y9L_2UoUi=%KZdea%HD|io!&_mUhU84E`fNv8 z$8JR`WuKr~%gN2QRLka!B$;M=nr17{K5Cn-v^@FRnMiE@2F>R)0k1`5eu2 zPa6QZdD3YaWl2Q{=g^Pj%uG5 zXm;%swPV$VH`b?UK5uW+Sd61l?erqerJXcK~t|XREUvhreLh@%DF6uf?ezxIO1l&CTgRTt;I$4IPvFDQ_yqkx! zua`Gm+O>zgyx~^yby?Rq{X*I+4|mZ_zavv!#wDaGtJO5)LFo7RZtf8Y*$FH=&= zi#Uzsp4visK*uj@i=zv*Lx?ajSs5Q3LnK;(ve*?qwLHJHJimGz;p3%|^6a<>r`1_n zTV*8J$Syr_B8}OHxdl{BwBEc^Oy(pq(l%e7Ve&~aArgK0|o>hYtr znWlG^#wbzTB-%lbJS?M?x9KO>xews{3UmKu$o`>)r8y}GTuhnt;q`Ge>Ofh4ck$nn=KdydHTa&IQ9O1@@Oy8n$LdjDy+@i5G&#NNRUWO zJp1s~T0%Dx-C9ekDYiCIT>B6ZUXgc6aBnzrn|fyA8ZC$bB`ZCELSJ46BaZAl{ZHlv zpw6_iId+NZjzssxJqJc>Fga7JOV~L?57A29TU%b7Up$oBbmxH+z0`#19q1ir$3JeH zX2(DF_VpfKT3N-628K=(``f<(0pA`mOFu_r>X0RNcmA9bY}F{`}v4oKwYKx-Y&ioT~k9TX9#* ztI$ZF{8K>=d+EOT`fzf7fvo$7pFhVNiM@1Ry!kaj)o=anS2$JdrTgL=Vo~*K1L_Y? zf0a|kUb-*tibvL~Dt+Jk&@B&r>&sUsE|M=GtM`}RTA=3CKM1ne3&`5>njq_okN-)E zwHAAUYOS7r#!4piME7|;a}D0f$h}U>jy7dWBT{X`FZ%iWZ}}fT{dRZ4c|rEiJn%0? zmi?tYJE_Ml)zj0n>F2M!^2!^>pNG}C3Es<16Yv#ye0K0;sg8lw+VUYAf8gYzX8a<2 zii>MU=BU$)+1bb7NVD>NX*kL9xsxY1X$S7ul-R^H^3Q+khu0=jLrJQp*?YOxI5NKo z!Ztnd{|wL$vZqI<9Zd2|_bu;FZmY}WqlRg?0}PhaiAi- zlp&EGN}>t6Q)zF#3`4gtbSSWT0k7Zq$rrC>!1#;jzjSRxy%6;W|Mbe0PQ6ZRzvOy> zd2hb&qc_vB26%}-X+B~|Tz}t}-i}{2hif|CWlcq#E%*KXI~Y9Qz~MR4(?g?w@7v!; zW5^ej@WxZu-2${|94v{K=W0eXJ6GIhnuY8(tC`JiD>RDDZ8a-znN~BGohukUzZycx>O>nkPocC2D5EndF%k9bjJVB?M3SC1Us?pGz-?PGR*yEtQKyUzc^RS#W{y`+smVt~<}?#t6m zNBByA0<#RP_`i{&_xHaoWbjM>2`^__ys>e|_F0VfR<^IKE-l*IsRwQQk>m8Sar zx6d!u*XC-gOUv6@YO}C(Xoc0m=*KIn@%2w@z>cWKb>3>ExdJ%;9aatS>ZMm8K9p(y z_kE|XVlNLrjh8d4hOyl|`)VK)QH?MBm4_SG2UX)qjdBCj|Nc*f!Z>{j3WNUg;*)qe z(`s1fq8gT{#y|hi;Rb}F|Dit{7eZcUtmc15n5aMUzrV{d&0Y}u&WFA*$zosF!#j(O zseG+rSbAp8Xj=75-m>z=TA^s>Yeu1F7;}YYYU0?jW4GRYcRz0W89jEa=c0dp^p>yx z^@qV+$jX2I(f6)RNbP|9FMH6QV)<)5Wxe0%&#>S2@1bJLe_$O_;*kfxl{k;j{oRoG zCY$@~-FX>%Zn=(sZ@qhpeQ*7{zhuuF@A*1^{`Fs8amD7VQdeAc-jUkTe#2r9bB@rj z=0@7P_RnK3?O4B6Q=9bx3fj^4;QR9y}{;2augFkHkXtIaF%QJX+ z1~1PrKb2%O4^W!><}iV9*Fdvx!CoAg>sy$=#~!a8!(VtCnPS~$5scBA#YAeX^|1yLa8Jzc57N5O5a4$JFtYAnHhKkR%&&6 zK;X(X>{o&CY+rrZ##E$v0H0ZhDP$?b3p(1TDIFzR~^x#mQ761*TN~E=dLq*z< z8i};|$h3vvP@dMz1c&l+&5#<2v}R=5$Qqd;ToC0(*2oO0kw_cDBOYo+*4JvOZz8c_ z>4Wcra^zZwYc_*xHiK(6gKIW}Yc_*xHiK(6gKIW}Yc_*xHiK(6qsjCs*O?~QZzk7o zCf9GK2otKEmum7-Oc_Dj-55 z%gu!}I!}wvPgPu^X+&s9eMM*_vkNT@!&oHuJEs$yxwx5JdQ2`oCYK(QOOMH=hl_^A zrN`pZV{z%Rxb#>e3YQ*>OOM5+$Kujsap|$R%v!uuiP|vi$od@&8dZwjhHMpf{H8`Yf zk+~jSw%*e6O6Gctu<>9~F+IBMmNABn8sln@{`ObJqD#28UV*BYCZtXc@qkQVZGL&3X0?nM) zh}6|DC81$>V;WiItKj`Ib>+)azLXNoyb`A7Qe@q)G;h5Tt&H-e6lW=9TBZ_HA;nUv zU@3%I3Sm}(t2EhyKJhb?4b8GNYcAQ;vbv_@f2O8qH9fECMNOa6^oFL}n%>lmjArP` zqGlMHVQPk@8ClK9X+~Z%3Yvl6Yl(!G`JJyMQVE4_g%Me$_F1L&S*7+_rS@5+_Svkr zd6nuGlmZG$vVziR1;HaBvrLh|aYH*0=N(FgS#$kfeh2uq(r&QVy2;pIdV5j)Xd$}Os!h=kK0`aAUWreWtu&h7<|5Ac-0*@j{fJb~O zROS^b^9q%D0Z*Q;pu{ZXyfsjqU`3?{MWqHsh0LOWl+)d4x}oWorsp)hkSu6=P1C_9 zLo>}JIA&>PRx@*&nb*vMW)?LQoV1hRP)4(K%`!C0)be@A^7QW~p|LrRWah>js2i$Z zuZicJc-F}n~zaUGu<_g}gx&dhwVY*$bqhD}h)^iz8 zx`v^A)e0`1mR+~px?9a6G_lMjJEs@CUyY2DPPc7T-;LSGW*l1JpscxHdNX4*>WImm zmzJQKeK}OiFiy4j5>(4CPPOh&@L`_kGOjKCcCbyj#?6%TE;ueo5aVma9IfMMCWW;WAcTXoLap9>vFICJ3Y zbJBzBIK=ZzezOg`E*#?7LLpbrngQNrz2WgLNU%-a_8Fqm*uxCpF87) zav_W0&Y8>6GuYBph-9WI0^5s*B#Lzq0lPsKO&kHX27JF{|5r1Af&!3ln|Jzs4 zf7{7hh)&A5RP#92Z2p_`?#B(AeD_(NoKG$w7vdtD9XmS#F;!FfYnV6gy z#{Xt3^orQ1h~iXFdURT7vk2e7RFA>o|As3f0dd2qEv%tRGAlnK4Wtxs;z+p;EnRyk3jNY_d9W7T% zRen7HKZRFF;8dlwHzLq;;YDdjsUi}|7Gc|w(*ZTqb1hm&rr`&Ew+ecpB?aG2<)zs# zj3r&-g`vmq-`_&ECsw~)+u^pnsFQkZ)o%=g3=X8%#$ zJ{bE1NVX9}C!|?^I}*JG68%z?M2i&D=~&|IKiFH0hngUW1@vqR#Q|x8dN-M-Bk2}f`-UfHS=Um=O;|I2aEdqx5qHsu z)31HKF}tio*1d!ya(&Tg2XZ4YcQQ8S{MVcXw{fS?y9vvL3vM4mnSCzRdA4e;9{8S?YJ!gr#A+rDP*=% zxNQU6-V+~g{%dv_-h3_EDcbtmp-s&{9@hMq@Sa1k(c5~r18pz1xr4@$#meN=>j`k^ z9C7^>XvVd)t$lY^H{^)kiD<-)h)yUnM<`IIXt!r}enmO%uVa%FxY10|sf-ES5F}7S zMoPm@l8M>zQl&h+MK}9WmaLK!0Vx9`P#eU_LKq}s0O1DFV!5rtt_au!8GCPR88acS ztwsUWxGIi16|=5x$l2%HX?3NHSfWPhjW85spL-^H9a-bR+FygLeP3GdQ6|IC7_8UV z(a3_)>l<=*UD}aC9tu>*zJ}-+DVMd=$Z7$xU_vQRBePkQyC)-H|{>rYmfamdjMgIHw z*zh8-@;>IPb)w9_63mZu76oM+OUnIM`UMZWgpC`C0(RBkRRF<%jWf)mcD;jvv*176 zhn4*yBjzG*eeoM~mGTa_N;*;lDmwzkLqV3SbDah2Mq?%tjl%AU^jVr>$imqZ<#vHg zyElPTw~)UHI`Ip&bCh{JF_8i&;=wl@?D!So;f`FUa<7K?9g3}T12wK6c0S2-&_spq z8VKD}Q5x5mSuA;r*1B%_k)p=*?X?iKAIH(R3^TL1G^fg0f5sl1Gr}~xIca!jX?VAD zA$c&gWgHj`!9E#=fwuw!M`B|jaIn?V9zGDL)uc#B0STw0kl@cU78Ih#S3W8t@xX>t zI8tll2jk#jY;t(_S!i6EJXC*Lo*6C1FC6u?u^O9SFko9PDM7D~OD-9dL)U>rk4JHc zXA+AqAR=O5D}C_IT#5T3djsVP|I1*AA^ZleWiZ zH2a;R*&_!SYsgYBw?;&yag zo*3bvN?SaLji{3URzSwe&?@!xNQsP+ z9b_jpI(5|O#g1!4&FXX7xSm=?fZhPHEA0?~rl)Aa6}QtfVkFUudf(vM?;!CFkI+sg zivB>`c0$>#SlY+1Zg+SuT`zK$<)d+e`(X=29gWc*yQ+bTzD*;9ca&L_+g9zPErNT> z7{$8%We!EP)w7i4XH-S0c7hc@Re5=oFRj8*l}F1zlNOf6yVWgVnEjPD)0Qolp2oXU zx5n-RtFA8HWiLiwpe^pQW3wwM?+wSoke->`UBbj;n>*eyN9is+VN=~RgX2>{`Edk` zB`T3Il4v{7M9oJ4!{(+Kg1MhO%~50K=B?TV_ftNf&tC9 z0V^-JSvtr`qF36+1PPAuUyVk`OnHW`zH7t%>7A3eRr!J+nB0L}pM)t+*HFx0j3SW5 zF2RgW#-q@I*era}-n@HpiNJ+)1!{V164L0;mI)d2<#`iL%#MvY{-NPgnO?xCq{XT~ z{%ds@gK|%%=N`2;1GT$^I`;q~1hs*8(zDiqzaNwQRC~Q0Xg!XFtnky#&_$D;tUSnw zh0>01qi2`VzXKTk<5>EPVwCGa;X$J76ckP=SH{;UP!G||@S@Smxz63B60$}%_`+@; zHxjhU`qp;&oqL4b0))sraN8qf)IUey@?5mE#L2C|$*D-K=CUUiT4>Q80%PbjqyS(C zftMHCtOsSa5LaTy(`q;s_(*NB5segi$#x|F#R2c|mw4`|_TkjkC^E|_tj!)*ln1*D zcLb-zPhgs2{}lA80(CcwQ7rhL#rjPwt{9vt?Z9Bt8_`vHw+&Y4QS$+rwZ`E%Z;>yz zQ0Z`t(Hocd5ABP}~W3lx$R+uYxfC^(NiNlb@6A>C);y#uXqR7sKaFEL22xRa_ zadftO<9IwVY~Xyjm8G@ix*aWvsu zsh_u*xMlKCs#KsMB*=Xu0blD%?HZ{+<7geJAJ&z+916!)>M;;5l(1K|dq&Z+?a!fZ z76p!Df$ZQ80G5`<#B3+TEWTA(2$C`%KAE85`jdpg{BLEli|i(2WSmTpNis$DkP4Y5 zGh~+RCAX1%WIwr`93ThD9ps&)O75iow=zWZzNnFxT4F3I1?8OyX@SbRwp0hnrS zmg=r)y8PdxOBLhP^ZO1ggdM~(Wom_}&W+KrL3m0+>V~=KK{^mplZXM5`lI6{QRs1g zJ#*Iym7U31pRBtU4ID^=SR?K?WqL6}_m9}(_f+GPBjr(S#`Gu;)1wl4`QU#7Lc1^a z;NSAEJz!GX%2d~NGYQdsG^%3=NE}OkDJh+G4@iOjVG0#~m#A>7xK9qFyH(6$su6;^ z&;Yu(r17Ja5X^Q^m4XuYKxcg80#RhhN^hkd%QeLrRR*Ima58%SdP88W|tlhnc?pK_t5x;x+F- zkqr@>nc>=8f=`M;YY)+??|XIWG!IsgVla)hmD96Bbf*iIa}^(MW49PvK8vk`9%Uy^ zV!-&VIy{c?hNmxc&4{YvrBeH&mX?6RJ4=s`Xoz#rA+a%E1=@6P?CxVQ(0SyqrQ3N- zqw{z;PUj(^5^vKmC8M*h11X@$XHZ$WvOTQpkd$CF-lpS_F;QLF0`6dus#~`QnMJ|3 zhxA#lz;#rln~e2Qd(lVzIF8i>QfUttyK8!vU2iiHr;cFt3>HTQ3>=j~p)`irn(~0i z?M9(d`hz!fsI?A4Mj9gi(Nxv;?(XNVkb?1RvpjBve5 z4#pZ~(K0&|6{LWu>_^RZhagsPClaorN|KRu7QsBgc zz_PtiV`jF;t&eLMm!pT#5}0g+=^Gw+$naZq9_CJ!Dws?24p7}O%(;iiI{}e9Vr!Z} zgoHP<1Wgs7`9M^69hyZfUDPp-7>w=&j6T&7jAU=@cF=q|Mhi2QnS2LF(t{Jv9)sb4 z)gy953Z00K8S{ow=9Wmp=3ZmP=cC=|yy>RD?w?Gq(e;-#GDqs9L2S|_hsa?vPu@lD zA`9dQStLv3D7l*~lNGW`*2q2NUUH23e`~;3{2dIVfm1&YI$9YZ5jF>eJrN6GEqTO( z7$vMTU4;~=ejOnDVy7Tuo`N$_Q~($oRh^xd62pKa#g%6zqQ^3NI{S_TZ;hOP{|FI*m zVs>G2wgQv+O#0Pjn@^UnRL>5uE)gw?*j@7Ri|thpgLiVLjfeN1agg|xO&~-oJ-2#X zB}mN9$HuHLhghiXSl=;q_bwpl3mqY7c4YL7z_Zq9o-J-r^lL}S-37$Ncej`PTQgN9 z=V?@lmIa_C(%tST2sd#&p!hdpLI|Vh%RoGo%@IK5bDcp2L-#6M04i4Zbu{{7IwtO_ zO1G3Kgh<_~4Awm(`Ex&7Uj$0t8M`Yi1^NYCM*G`SVr2O+ zQs5kyQ$fk0V{Me2Oc0Adw9L-6>_ z@)k_VIIp(J@YNghd(N0a1zN%H6;=oKBM8NRsE04pGuM!ljNl& z`E-(eCP}?W$AOnWe1i~&D7+}XuXx=*)@6Hk#8xS7vU57@Fq!pm-^HQx=bIUT!a8_ERrYzp!20PzHL=L1o0JT+Y` z6-o){Oobu^RMvZ-vi>%XeaqMQ#ne^yVr{X_w7FO_I6OImL4O+sB;u-iEks#$WUuha z(FmCc87X;gVCW(m5dJVn?2wvQ|gN@q;zqtFRg1t_SNA$a#?7{@X=l?2e^ zZ=)VDgA^#oiiG)nN#Ok55jbbjW1*ZD*BQ!@0w}*f37G$vINlu!&6qHcH(ReVFGTw> zftVVcp2pTmtk{!)j*9I9J3ywaiYB&OV&N0)uyTim$AVQxZi*8~ z++P;$0+Km8AXH9JW$pejZTI-5X#s~Hw66S-CNC&XN4_^HKWmpi|2eWp)HZ zScPfJJw@{@w;@sH>G25$SKCC6PtzRR@d7!r1x`_Lh(iVL_PFBu@ zS7d!ZNwaFFE)?ftXm(Wlyg;*Sr>GsPF1)cmMe})ko5o@sjcTVCX)f)gIU2uInqQ*X zw9_um_&lT=qZxggX4I}z^o|5W+nS=_Gc>n$tW&!D@a%Bmb9VmIT^sm?&)5FmDR#m0 zwO{=s{`~Vl;Lrayl-!Wiup;^kN%E5<61`>que*|1K7GmgT?@&dZMdlGIQiLzTM=;c z{CB!GBKZ54n+_lB z(h}rJjhxm*me(m+Zsu7clN$~{NRpyDKf0fj|IxkSWOS4PaI6Itg-VxCzcxa!Suj^gXx3m zJNgc$o3;4`yP^DgGTrB9u`+|{zVtw|wyP^N)TVOrstPOZD2q6iZ60N2?~+%TkKj zZZ+TtSyc%)D9uS#AF3`z#ml>)Z!$eucUJGGzbI!qS`+ z1TLmb`tW+f51;_q43@+biYW11J@UqXa6}al<$~%J({vh$Z`dnLm~j>hA_2kyqSm6j zPIxsJib_Lmd9)V)g=K!!(MBim?Nc-93@uL;_Owo=N{S&05;Dh$bES20ZDFBL6=oFB zPARm=r&swLxZ@m;4mKODF!8D9QE}JbF26_6!UpV#te)4E9 z(wfiy+pDlPb3?3z=OaNPG4bquS8EB~NOWs0sixT4#47i72ZUGTT@pM7a+`Xle2o@F zfRdFSK%p-$gAqseo&KqL0jM*rY>r)Gx+BqjanFI#8cfdA>JrXlr-x{z?yW5^&MzKH zZMyToiC${L^bYinv*RDPO~dH2mwWqq4==5(;%B{R^y|4of4)D5qx*a9d+bH}GoSQH zHJ0vO#IG{H1q3hf(D|jcmEMy$<{;6NNL=*Y58d+0-#q^3{VlUknUl&f*ez&c-tL0T_q)+~-AcwtlUwnNyIln;G{lm|ns6J$?|tZ&hradYs}mQ=7m(Ha%Wo}E zbLt-iS?mR5?RZU)^~J~kB*j{by+E~APd{fR6MCZiyq>uR?_}g&r)5W*vZWEJHsKfj z{QbB5kDq?KJK?+_`)3~be~T>pOM7-wk6Wszr)Sg8UwP$~H;z9Kt8){)mzyTwEAaU2 z;K@=Q1FN;=LpJ`v$wkfhMfemK*N)6lrx&xckHL{<<@?fblI3$JPj1o<+_5RKiD~4Y z|JDz$O{9jBR86z@a;ei4Lidf-O`w1e#F(P;;h{L=kvZ{xjRs%IaibX#T{?Ot`u z%;-iopY7>Q(Oe$;%nMu!{^gr^Nlz)g$R-Vr$y5)z1{(G6{sM(3cy0GQh1&KcQm;;J zujWhs#Y9q>TQ9@VEestBtX{zDH-7TPYZ);9;`uLK z8&NMr{lP!Ia-~zR)7me&USQsv@B7HjbgThh;!m277!udt_rzy(ZBcY@1rr~3rcw7sq1e6S~L!pMCR0mx3XlUXE#p#rDG8Jq~QPT z-~W(H0eeC6^r@XYM0^ww6l2G!H||O%F!t)&c)gvg8O`imahqutvfHd?HoL9RC^om% zth{Af&0Kb_VDyaZM!r9T|8!cX-+k_>Yfw(YI`x^|l+JjWutw0nbN>xvU2J5P=)Rmc zLz#JfrKBFO{cJ-oZH@o+mjE<<`SG{K%h&!9FNzFoym9;Lk)zxFs${!;%&u=2XY6d( z`G2_Tq3f}iv=K-QFxu07d79}6U+GU^mVp)jH&XQe{ssUcT^a{j>GVTAq@6=W7<>9CCa%R;qwwq^P4P+v!@rA$gaO3)*YCNe? zZh-pV|EW+Ir%yp)&|hAB5-(?34eMM~!xGi_=N~%UfKc>5^rz!O$jgk?{O<@8^+*2q zcR8ln3u52-(C$i&hYn9F%_3MR8SlHTPV`*&ZP(Sno z`Nre_n0QO#!usL0#k=}-Q@7pMrt@lK+*h5IqO&6P41b6(gMXR)!Ku=-{42*FdHyKy zN0C2j{4vKLb^d7Zhs_^N_Aq#P1~1RxDXXU5ZUgXNu2YPC>mQO37WDoKmWc%Zo>fou@^|tg1E9G$J&lz9KY|*%&GKa>fWM_q%%%o4L4|TzX6{ zJtmhPlS_}urH6}##ihsM(qnPyvAFbDA_|uti%XBirN`pZV{z%RxXfC-REw8t@lq{b zs>Ms?d1ZO2Szct1Z3;aWxmr%=zdz?}#4 z#CoQsWHq>@Xf-&bY>~MhUAErR@=E4|sEVR+nE!V6`;V7=Cuq7*~7rx4$YT z59C$xH>e2YQ1HV@N$IbnlG4v*b%^ShDiJF1=dX_mf9wx_`>Tno5=@0C7OS4&HmV~w z%BL>uS;;je(9C&FUtRrD5*mg#rjb>?3f?bMSH3LeODVz3D`9FbMb`aFqud+O$|zq- zah5`+WhyZhQY@tkmO_}N5M~v)N|P<<>(I<(L$fT+noBmdtgh+!pQ-6tP0wq3QPbx% zy`ky0rZ+VsqZxX#s2PT4n3`c}MpiR&nvvIxf@a|NS|Xukes3j_N+@(IjL0gr&nmUg zDz(olwa+TG&t|>Nt5mn36i`r-6_iFR2p$QUWr_suCfb2G?@%htn(OylbG&>GcL895 z{fx;mDnE1TXHogo4fQjtehN%UsZs(BUH!Dw&%E+!sx+D^jiwq|B2*L$F3)L3T{9Y* zVQWTHGc%g0C+9S?rkQokY-naPnM+#9Y*J5}nw8b8oMz=UtDsp$&8j7B&8lk_V1^ff zETd%&EenvrQIi@@E}v5Kzu1-Ss^SuEGtmJ zzm%Yyz@rEf;1OR6m3f8Ayh3GOz>}vdC@~8;Zw(aJT~Vn)QK>;uA+snT<#aciZfLrt z={Zd=Bnz5e({!-O&`dK4j#-+S)y$k`<~6gRnMKV6C+#FSl+i3*vkc8LwR|43JpJW^ z(AXSDGIQe%)MM7K*Ti#9JnQ1wIFE&!{L_?AOMDl_QzSP#9Yij7VNDW0^0loxxZ}`mEt+<38)A*Vk$EH5cqvEC*X2FwAVN zX_yVVQ|H|M;~stLvNlEedQCV`J>GHZ)b)FFso%N9N1D%hsfP>LaeSXUl9ik;#JtKQ zmp1qu${!>WdwzLyC||k!f5wSBGm1QUsh{cN-t=3W!d@VjS}v@ISI#cp4pE|KT5`%Z#$ zPfq103yDLwhOv!)Vq~x)&fgo}i94@{aUv)^3u-@>Vr`*kL5aBI<>^t}6VI-9s^G8? z?ES-Xjf@4`;3OV~n%{tbJEdGPNI-&;0sfS

9+dVANTZwqCRi-f@HUQ_ zNG$Ceo}8V)wdCShD<3f;4!|VNq79D54(pV4FX4zBUo?uj8-cl#u`%aA<~+EKJB8j& zfZpe#py%geECl+GM)&aCgZ~hDh(sVgLcI?N{Cpe)(&PIj51ivcJ+uVC;+Kj)MsI1~ z6izsqW`}gfl#lXwsd_t3OT_67>3}*FGFvI!wgGPMi4QmbH9HJ%z838iZT;=grsf|H zYyL}kJPbZtu+iIkwgYW1wmE~=k;Tg7)awawxTBQbvN%=6wKT0*%j$;gPo0RkZ!%s) zbV894p+KFY-JaR`73H|Uj!jPBL~}u>GA3|CkU$9;DGfVGCT7P=mGbZw-Rw(Qyh@J! d@3Ft#wPUc#9S1<(Y#KSEIj7jaXZvyC{|~3$rtbg% literal 0 HcmV?d00001 From 7ed28af4fe3a205e83f38834c3db0b749a214120 Mon Sep 17 00:00:00 2001 From: Tboy Date: Thu, 5 Mar 2020 19:00:06 +0800 Subject: [PATCH 14/37] Upgrade curator version (#2083) * upgrade curator version issue: https://github.com/apache/incubator-dolphinscheduler/issues/2020 curator issue: https://github.com/apache/curator/pull/297 * add DefaultEnsembleProvider override * add some unit test * add License --- dolphinscheduler-api/pom.xml | 6 ++ dolphinscheduler-server/pom.xml | 6 ++ .../service/zk/DefaultEnsembleProvider.java | 10 +++ .../zk/DefaultEnsembleProviderTest.java | 65 +++++++++++++++++++ pom.xml | 9 ++- 5 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 41971734ec..b7c3f3da69 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -140,6 +140,12 @@ org.apache.curator curator-recipes + + + org.apache.zookeeper + zookeeper + + diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 080b87ebaa..86490197b6 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -86,6 +86,12 @@ org.apache.curator curator-recipes + + + org.apache.zookeeper + zookeeper + + org.apache.zookeeper diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java index 9eedf7a4ca..dbe8bd6395 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java @@ -45,4 +45,14 @@ public class DefaultEnsembleProvider implements EnsembleProvider { public void close() throws IOException { //NOP } + + @Override + public void setConnectionString(String connectionString) { + //NOP + } + + @Override + public boolean updateServerListEnabled() { + return false; + } } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java new file mode 100644 index 0000000000..cdec9d0547 --- /dev/null +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.service.zk; + +import org.apache.curator.ensemble.EnsembleProvider; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.*; + +public class DefaultEnsembleProviderTest { + private static final String DEFAULT_SERVER_LIST = "localhost:2181"; + + @Test + public void startAndClose() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + try { + ensembleProvider.start(); + } catch (Exception e) { + Assert.fail("EnsembleProvider start error: " + e.getMessage()); + } + try { + ensembleProvider.close(); + } catch (IOException e) { + Assert.fail("EnsembleProvider close error: " + e.getMessage()); + } + } + + @Test + public void getConnectionString() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + Assert.assertEquals(DEFAULT_SERVER_LIST, ensembleProvider.getConnectionString()); + } + + @Test + public void setConnectionString() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + ensembleProvider.setConnectionString("otherHost:2181"); + Assert.assertEquals(DEFAULT_SERVER_LIST, ensembleProvider.getConnectionString()); + } + + @Test + public void updateServerListEnabled() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + Assert.assertFalse(ensembleProvider.updateServerListEnabled()); + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 57063a86c2..dee1dce8b2 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,7 @@ UTF-8 UTF-8 - 2.12.0 + 4.3.0 5.1.5.RELEASE 2.1.3.RELEASE 1.8 @@ -244,6 +244,12 @@ org.apache.curator curator-recipes ${curator.version} + + + org.apache.zookeeper + zookeeper + + org.apache.zookeeper @@ -728,6 +734,7 @@ **/server/worker/task/datax/DataxTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/utils/DataxUtilsTest.java + **/service/zk/DefaultEnsembleProviderTest.java From 1f92b4c4db3c2b96d767476bf77002d894de49c2 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 23:21:10 +0800 Subject: [PATCH 15/37] Optimize ParameterUtils.curingGlobalParams() execution efficiency (#2090) * Optimize ParameterUtils.curingGlobalParams() execution efficiency * Remove excess null check --- .../common/utils/ParameterUtils.java | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index 7f2888384e..9492b49cb1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -119,10 +119,15 @@ public class ParameterUtils { */ public static String curingGlobalParams(Map globalParamMap, List globalParamList, CommandType commandType, Date scheduleTime){ - Map globalMap = new HashMap<>(); - if(globalParamMap!= null){ - globalMap.putAll(globalParamMap); - } + + if (globalParamList == null || globalParamList.isEmpty()) { + return null; + } + + Map globalMap = new HashMap<>(); + if (globalParamMap!= null){ + globalMap.putAll(globalParamMap); + } Map allParamMap = new HashMap<>(); //If it is a complement, a complement time needs to be passed in, according to the task type Map timeParams = BusinessTimeUtils @@ -132,9 +137,7 @@ public class ParameterUtils { allParamMap.putAll(timeParams); } - if (globalMap != null) { - allParamMap.putAll(globalMap); - } + allParamMap.putAll(globalMap); Set> entries = allParamMap.entrySet(); @@ -146,22 +149,15 @@ public class ParameterUtils { resolveMap.put(entry.getKey(),str); } } + globalMap.putAll(resolveMap); - if (globalMap != null){ - globalMap.putAll(resolveMap); - } - - if (globalParamList != null && globalParamList.size() > 0){ - - for (Property property : globalParamList){ - String val = globalMap.get(property.getProp()); - if (val != null){ - property.setValue(val); - } + for (Property property : globalParamList){ + String val = globalMap.get(property.getProp()); + if (val != null){ + property.setValue(val); } - return JSONObject.toJSONString(globalParamList); } - return null; + return JSONObject.toJSONString(globalParamList); } From 6070738acbb89e0a7ed49b487fc1225888e29019 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 23:34:37 +0800 Subject: [PATCH 16/37] Support worker server to run bat script (#2023) * Support worker server to run bat script 1. Reimplement ProcessImpl.java, ProcessEnvironment.java and ProcessBuilder.java for Windows 2. Modify shell task code for windows 3. Add ASF License * Add Unit Test --- .../dolphinscheduler/common/Constants.java | 2 +- .../utils/process/ProcessBuilderForWin32.java | 1065 +++++++++++++++++ .../process/ProcessEnvironmentForWin32.java | 286 +++++ .../utils/process/ProcessImplForWin32.java | 752 ++++++++++++ .../common/ConstantsTest.java | 40 + .../common/utils/OSUtilsTest.java | 24 +- .../process/ProcessBuilderForWin32Test.java | 210 ++++ .../ProcessEnvironmentForWin32Test.java | 124 ++ .../process/ProcessImplForWin32Test.java | 70 ++ .../worker/task/AbstractCommandExecutor.java | 92 +- .../worker/task/ShellCommandExecutor.java | 39 +- .../server/worker/task/datax/DataxTask.java | 11 +- .../server/worker/task/shell/ShellTask.java | 9 +- .../server/utils/ProcessUtilsTest.java | 17 + .../worker/task/shell/ShellTaskTest.java | 198 +++ pom.xml | 4 + 16 files changed, 2887 insertions(+), 56 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 73125f4926..6af0e6445f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -746,7 +746,7 @@ public final class Constants { * application regex */ public static final String APPLICATION_REGEX = "application_\\d+_\\d+"; - public static final String PID = "pid"; + public static final String PID = OSUtils.isWindows() ? "handle" : "pid"; /** * month_begin */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java new file mode 100644 index 0000000000..4fb5f94616 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java @@ -0,0 +1,1065 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * This class is used to create operating system processes. + * + *

Each {@code ProcessBuilderForWindows} instance manages a collection + * of process attributes. The {@link #start()} method creates a new + * {@link Process} instance with those attributes. The {@link + * #start()} method can be invoked repeatedly from the same instance + * to create new subprocesses with identical or related attributes. + * + *

Each process builder manages these process attributes: + * + *

    + * + *
  • a command, a list of strings which signifies the + * external program file to be invoked and its arguments, if any. + * Which string lists represent a valid operating system command is + * system-dependent. For example, it is common for each conceptual + * argument to be an element in this list, but there are operating + * systems where programs are expected to tokenize command line + * strings themselves - on such a system a Java implementation might + * require commands to contain exactly two elements. + * + *
  • an environment, which is a system-dependent mapping from + * variables to values. The initial value is a copy of + * the environment of the current process (see {@link System#getenv()}). + * + *
  • a working directory. The default value is the current + * working directory of the current process, usually the directory + * named by the system property {@code user.dir}. + * + *
  • a source of standard input. + * By default, the subprocess reads input from a pipe. Java code + * can access this pipe via the output stream returned by + * {@link Process#getOutputStream()}. However, standard input may + * be redirected to another source using + * {@link #redirectInput(ProcessBuilderForWin32.Redirect) redirectInput}. + * In this case, {@link Process#getOutputStream()} will return a + * null output stream, for which: + * + *
      + *
    • the {@link OutputStream#write(int) write} methods always + * throw {@code IOException} + *
    • the {@link OutputStream#close() close} method does nothing + *
    + * + *
  • a destination for standard output + * and standard error. By default, the subprocess writes standard + * output and standard error to pipes. Java code can access these pipes + * via the input streams returned by {@link Process#getInputStream()} and + * {@link Process#getErrorStream()}. However, standard output and + * standard error may be redirected to other destinations using + * {@link #redirectOutput(ProcessBuilderForWin32.Redirect) redirectOutput} and + * {@link #redirectError(ProcessBuilderForWin32.Redirect) redirectError}. + * In this case, {@link Process#getInputStream()} and/or + * {@link Process#getErrorStream()} will return a null input + * stream, for which: + * + *
      + *
    • the {@link InputStream#read() read} methods always return + * {@code -1} + *
    • the {@link InputStream#available() available} method always returns + * {@code 0} + *
    • the {@link InputStream#close() close} method does nothing + *
    + * + *
  • a redirectErrorStream property. Initially, this property + * is {@code false}, meaning that the standard output and error + * output of a subprocess are sent to two separate streams, which can + * be accessed using the {@link Process#getInputStream()} and {@link + * Process#getErrorStream()} methods. + * + *

    If the value is set to {@code true}, then: + * + *

      + *
    • standard error is merged with the standard output and always sent + * to the same destination (this makes it easier to correlate error + * messages with the corresponding output) + *
    • the common destination of standard error and standard output can be + * redirected using + * {@link #redirectOutput(ProcessBuilderForWin32.Redirect) redirectOutput} + *
    • any redirection set by the + * {@link #redirectError(ProcessBuilderForWin32.Redirect) redirectError} + * method is ignored when creating a subprocess + *
    • the stream returned from {@link Process#getErrorStream()} will + * always be a null input stream + *
    + * + *
+ * + *

Modifying a process builder's attributes will affect processes + * subsequently started by that object's {@link #start()} method, but + * will never affect previously started processes or the Java process + * itself. + * + *

Most error checking is performed by the {@link #start()} method. + * It is possible to modify the state of an object so that {@link + * #start()} will fail. For example, setting the command attribute to + * an empty list will not throw an exception unless {@link #start()} + * is invoked. + * + *

Note that this class is not synchronized. + * If multiple threads access a {@code ProcessBuilderForWindows} instance + * concurrently, and at least one of the threads modifies one of the + * attributes structurally, it must be synchronized externally. + * + *

Starting a new process which uses the default working directory + * and environment is easy: + * + *

 {@code
+ * Process p = new ProcessBuilderForWindows("myCommand", "myArg").start();
+ * }
+ * + *

Here is an example that starts a process with a modified working + * directory and environment, and redirects standard output and error + * to be appended to a log file: + * + *

 {@code
+ * ProcessBuilderForWindows pb =
+ *   new ProcessBuilderForWindows("myCommand", "myArg1", "myArg2");
+ * Map env = pb.environment();
+ * env.put("VAR1", "myValue");
+ * env.remove("OTHERVAR");
+ * env.put("VAR2", env.get("VAR1") + "suffix");
+ * pb.directory(new File("myDir"));
+ * File log = new File("log");
+ * pb.redirectErrorStream(true);
+ * pb.redirectOutput(Redirect.appendTo(log));
+ * Process p = pb.start();
+ * assert pb.redirectInput() == Redirect.PIPE;
+ * assert pb.redirectOutput().file() == log;
+ * assert p.getInputStream().read() == -1;
+ * }
+ * + *

To start a process with an explicit set of environment + * variables, first call {@link Map#clear() Map.clear()} + * before adding environment variables. + * + * @author Martin Buchholz + * @since 1.5 + */ + +public class ProcessBuilderForWin32 { + + private String username; + private String password; + private List command; + private File directory; + private Map environment; + private boolean redirectErrorStream; + private ProcessBuilderForWin32.Redirect[] redirects; + + /** + * Constructs a process builder with the specified operating + * system program and arguments. This constructor does not + * make a copy of the {@code command} list. Subsequent + * updates to the list will be reflected in the state of the + * process builder. It is not checked whether + * {@code command} corresponds to a valid operating system + * command. + * + * @param command the list containing the program and its arguments + * @throws NullPointerException if the argument is null + */ + public ProcessBuilderForWin32(List command) { + if (command == null) + throw new NullPointerException(); + this.command = command; + } + + /** + * Constructs a process builder with the specified operating + * system program and arguments. This is a convenience + * constructor that sets the process builder's command to a string + * list containing the same strings as the {@code command} + * array, in the same order. It is not checked whether + * {@code command} corresponds to a valid operating system + * command. + * + * @param command a string array containing the program and its arguments + */ + public ProcessBuilderForWin32(String... command) { + this.command = new ArrayList<>(command.length); + for (String arg : command) + this.command.add(arg); + } + + /** + * set username and password for process + * + * @param username username + * @param password password + * @return this process builder + */ + public ProcessBuilderForWin32 user(String username, String password) { + this.username = username; + this.password = password; + return this; + } + + /** + * Sets this process builder's operating system program and + * arguments. This method does not make a copy of the + * {@code command} list. Subsequent updates to the list will + * be reflected in the state of the process builder. It is not + * checked whether {@code command} corresponds to a valid + * operating system command. + * + * @param command the list containing the program and its arguments + * @return this process builder + * + * @throws NullPointerException if the argument is null + */ + public ProcessBuilderForWin32 command(List command) { + if (command == null) + throw new NullPointerException(); + this.command = command; + return this; + } + + /** + * Sets this process builder's operating system program and + * arguments. This is a convenience method that sets the command + * to a string list containing the same strings as the + * {@code command} array, in the same order. It is not + * checked whether {@code command} corresponds to a valid + * operating system command. + * + * @param command a string array containing the program and its arguments + * @return this process builder + */ + public ProcessBuilderForWin32 command(String... command) { + this.command = new ArrayList<>(command.length); + for (String arg : command) + this.command.add(arg); + return this; + } + + /** + * Returns this process builder's operating system program and + * arguments. The returned list is not a copy. Subsequent + * updates to the list will be reflected in the state of this + * process builder. + * + * @return this process builder's program and its arguments + */ + public List command() { + return command; + } + + /** + * Returns a string map view of this process builder's environment. + * + * Whenever a process builder is created, the environment is + * initialized to a copy of the current process environment (see + * {@link System#getenv()}). Subprocesses subsequently started by + * this object's {@link #start()} method will use this map as + * their environment. + * + *

The returned object may be modified using ordinary {@link + * Map Map} operations. These modifications will be + * visible to subprocesses started via the {@link #start()} + * method. Two {@code ProcessBuilderForWindows} instances always + * contain independent process environments, so changes to the + * returned map will never be reflected in any other + * {@code ProcessBuilderForWindows} instance or the values returned by + * {@link System#getenv System.getenv}. + * + *

If the system does not support environment variables, an + * empty map is returned. + * + *

The returned map does not permit null keys or values. + * Attempting to insert or query the presence of a null key or + * value will throw a {@link NullPointerException}. + * Attempting to query the presence of a key or value which is not + * of type {@link String} will throw a {@link ClassCastException}. + * + *

The behavior of the returned map is system-dependent. A + * system may not allow modifications to environment variables or + * may forbid certain variable names or values. For this reason, + * attempts to modify the map may fail with + * {@link UnsupportedOperationException} or + * {@link IllegalArgumentException} + * if the modification is not permitted by the operating system. + * + *

Since the external format of environment variable names and + * values is system-dependent, there may not be a one-to-one + * mapping between them and Java's Unicode strings. Nevertheless, + * the map is implemented in such a way that environment variables + * which are not modified by Java code will have an unmodified + * native representation in the subprocess. + * + *

The returned map and its collection views may not obey the + * general contract of the {@link Object#equals} and + * {@link Object#hashCode} methods. + * + *

The returned map is typically case-sensitive on all platforms. + * + *

If a security manager exists, its + * {@link SecurityManager#checkPermission checkPermission} method + * is called with a + * {@link RuntimePermission}{@code ("getenv.*")} permission. + * This may result in a {@link SecurityException} being thrown. + * + *

When passing information to a Java subprocess, + * system properties + * are generally preferred over environment variables. + * + * @return this process builder's environment + * + * @throws SecurityException + * if a security manager exists and its + * {@link SecurityManager#checkPermission checkPermission} + * method doesn't allow access to the process environment + * + * @see Runtime#exec(String[],String[], File) + * @see System#getenv() + */ + public Map environment() { + SecurityManager security = System.getSecurityManager(); + if (security != null) + security.checkPermission(new RuntimePermission("getenv.*")); + + if (environment == null) + environment = ProcessEnvironmentForWin32.environment(); + + assert environment != null; + + return environment; + } + + // Only for use by Runtime.exec(...envp...) + ProcessBuilderForWin32 environment(String[] envp) { + assert environment == null; + if (envp != null) { + environment = ProcessEnvironmentForWin32.emptyEnvironment(envp.length); + assert environment != null; + + for (String envstring : envp) { + // Before 1.5, we blindly passed invalid envstrings + // to the child process. + // We would like to throw an exception, but do not, + // for compatibility with old broken code. + + // Silently discard any trailing junk. + if (envstring.indexOf((int) '\u0000') != -1) + envstring = envstring.replaceFirst("\u0000.*", ""); + + int eqlsign = + envstring.indexOf('=', ProcessEnvironmentForWin32.MIN_NAME_LENGTH); + // Silently ignore envstrings lacking the required `='. + if (eqlsign != -1) + environment.put(envstring.substring(0,eqlsign), + envstring.substring(eqlsign+1)); + } + } + return this; + } + + /** + * Returns this process builder's working directory. + * + * Subprocesses subsequently started by this object's {@link + * #start()} method will use this as their working directory. + * The returned value may be {@code null} -- this means to use + * the working directory of the current Java process, usually the + * directory named by the system property {@code user.dir}, + * as the working directory of the child process. + * + * @return this process builder's working directory + */ + public File directory() { + return directory; + } + + /** + * Sets this process builder's working directory. + * + * Subprocesses subsequently started by this object's {@link + * #start()} method will use this as their working directory. + * The argument may be {@code null} -- this means to use the + * working directory of the current Java process, usually the + * directory named by the system property {@code user.dir}, + * as the working directory of the child process. + * + * @param directory the new working directory + * @return this process builder + */ + public ProcessBuilderForWin32 directory(File directory) { + this.directory = directory; + return this; + } + + // ---------------- I/O Redirection ---------------- + + /** + * Implements a null input stream. + */ + static class NullInputStream extends InputStream { + static final ProcessBuilderForWin32.NullInputStream INSTANCE = new ProcessBuilderForWin32.NullInputStream(); + private NullInputStream() {} + public int read() { return -1; } + public int available() { return 0; } + } + + /** + * Implements a null output stream. + */ + static class NullOutputStream extends OutputStream { + static final ProcessBuilderForWin32.NullOutputStream INSTANCE = new ProcessBuilderForWin32.NullOutputStream(); + private NullOutputStream() {} + public void write(int b) throws IOException { + throw new IOException("Stream closed"); + } + } + + /** + * Represents a source of subprocess input or a destination of + * subprocess output. + * + * Each {@code Redirect} instance is one of the following: + * + *

    + *
  • the special value {@link #PIPE Redirect.PIPE} + *
  • the special value {@link #INHERIT Redirect.INHERIT} + *
  • a redirection to read from a file, created by an invocation of + * {@link ProcessBuilderForWin32.Redirect#from Redirect.from(File)} + *
  • a redirection to write to a file, created by an invocation of + * {@link ProcessBuilderForWin32.Redirect#to Redirect.to(File)} + *
  • a redirection to append to a file, created by an invocation of + * {@link ProcessBuilderForWin32.Redirect#appendTo Redirect.appendTo(File)} + *
+ * + *

Each of the above categories has an associated unique + * {@link ProcessBuilderForWin32.Redirect.Type Type}. + * + * @since 1.7 + */ + public static abstract class Redirect { + /** + * The type of a {@link ProcessBuilderForWin32.Redirect}. + */ + public enum Type { + /** + * The type of {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + */ + PIPE, + + /** + * The type of {@link ProcessBuilderForWin32.Redirect#INHERIT Redirect.INHERIT}. + */ + INHERIT, + + /** + * The type of redirects returned from + * {@link ProcessBuilderForWin32.Redirect#from Redirect.from(File)}. + */ + READ, + + /** + * The type of redirects returned from + * {@link ProcessBuilderForWin32.Redirect#to Redirect.to(File)}. + */ + WRITE, + + /** + * The type of redirects returned from + * {@link ProcessBuilderForWin32.Redirect#appendTo Redirect.appendTo(File)}. + */ + APPEND + }; + + /** + * Returns the type of this {@code Redirect}. + * @return the type of this {@code Redirect} + */ + public abstract ProcessBuilderForWin32.Redirect.Type type(); + + /** + * Indicates that subprocess I/O will be connected to the + * current Java process over a pipe. + * + * This is the default handling of subprocess standard I/O. + * + *

It will always be true that + *

 {@code
+         * Redirect.PIPE.file() == null &&
+         * Redirect.PIPE.type() == Redirect.Type.PIPE
+         * }
+ */ + public static final ProcessBuilderForWin32.Redirect PIPE = new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.PIPE; } + public String toString() { return type().toString(); }}; + + /** + * Indicates that subprocess I/O source or destination will be the + * same as those of the current process. This is the normal + * behavior of most operating system command interpreters (shells). + * + *

It will always be true that + *

 {@code
+         * Redirect.INHERIT.file() == null &&
+         * Redirect.INHERIT.type() == Redirect.Type.INHERIT
+         * }
+ */ + public static final ProcessBuilderForWin32.Redirect INHERIT = new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.INHERIT; } + public String toString() { return type().toString(); }}; + + /** + * Returns the {@link File} source or destination associated + * with this redirect, or {@code null} if there is no such file. + * + * @return the file associated with this redirect, + * or {@code null} if there is no such file + */ + public File file() { return null; } + + /** + * When redirected to a destination file, indicates if the output + * is to be written to the end of the file. + */ + boolean append() { + throw new UnsupportedOperationException(); + } + + /** + * Returns a redirect to read from the specified file. + * + *

It will always be true that + *

 {@code
+         * Redirect.from(file).file() == file &&
+         * Redirect.from(file).type() == Redirect.Type.READ
+         * }
+ * + * @param file The {@code File} for the {@code Redirect}. + * @throws NullPointerException if the specified file is null + * @return a redirect to read from the specified file + */ + public static ProcessBuilderForWin32.Redirect from(final File file) { + if (file == null) + throw new NullPointerException(); + return new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.READ; } + public File file() { return file; } + public String toString() { + return "redirect to read from file \"" + file + "\""; + } + }; + } + + /** + * Returns a redirect to write to the specified file. + * If the specified file exists when the subprocess is started, + * its previous contents will be discarded. + * + *

It will always be true that + *

 {@code
+         * Redirect.to(file).file() == file &&
+         * Redirect.to(file).type() == Redirect.Type.WRITE
+         * }
+ * + * @param file The {@code File} for the {@code Redirect}. + * @throws NullPointerException if the specified file is null + * @return a redirect to write to the specified file + */ + public static ProcessBuilderForWin32.Redirect to(final File file) { + if (file == null) + throw new NullPointerException(); + return new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.WRITE; } + public File file() { return file; } + public String toString() { + return "redirect to write to file \"" + file + "\""; + } + boolean append() { return false; } + }; + } + + /** + * Returns a redirect to append to the specified file. + * Each write operation first advances the position to the + * end of the file and then writes the requested data. + * Whether the advancement of the position and the writing + * of the data are done in a single atomic operation is + * system-dependent and therefore unspecified. + * + *

It will always be true that + *

 {@code
+         * Redirect.appendTo(file).file() == file &&
+         * Redirect.appendTo(file).type() == Redirect.Type.APPEND
+         * }
+ * + * @param file The {@code File} for the {@code Redirect}. + * @throws NullPointerException if the specified file is null + * @return a redirect to append to the specified file + */ + public static ProcessBuilderForWin32.Redirect appendTo(final File file) { + if (file == null) + throw new NullPointerException(); + return new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.APPEND; } + public File file() { return file; } + public String toString() { + return "redirect to append to file \"" + file + "\""; + } + boolean append() { return true; } + }; + } + + /** + * Compares the specified object with this {@code Redirect} for + * equality. Returns {@code true} if and only if the two + * objects are identical or both objects are {@code Redirect} + * instances of the same type associated with non-null equal + * {@code File} instances. + */ + public boolean equals(Object obj) { + if (obj == this) + return true; + if (! (obj instanceof ProcessBuilderForWin32.Redirect)) + return false; + ProcessBuilderForWin32.Redirect r = (ProcessBuilderForWin32.Redirect) obj; + if (r.type() != this.type()) + return false; + assert this.file() != null; + return this.file().equals(r.file()); + } + + /** + * Returns a hash code value for this {@code Redirect}. + * @return a hash code value for this {@code Redirect} + */ + public int hashCode() { + File file = file(); + if (file == null) + return super.hashCode(); + else + return file.hashCode(); + } + + /** + * No public constructors. Clients must use predefined + * static {@code Redirect} instances or factory methods. + */ + private Redirect() {} + } + + private ProcessBuilderForWin32.Redirect[] redirects() { + if (redirects == null) + redirects = new ProcessBuilderForWin32.Redirect[] { + ProcessBuilderForWin32.Redirect.PIPE, ProcessBuilderForWin32.Redirect.PIPE, ProcessBuilderForWin32.Redirect.PIPE + }; + return redirects; + } + + /** + * Sets this process builder's standard input source. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method obtain their standard input from this source. + * + *

If the source is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE} + * (the initial value), then the standard input of a + * subprocess can be written to using the output stream + * returned by {@link Process#getOutputStream()}. + * If the source is set to any other value, then + * {@link Process#getOutputStream()} will return a + * null output stream. + * + * @param source the new standard input source + * @return this process builder + * @throws IllegalArgumentException + * if the redirect does not correspond to a valid source + * of data, that is, has type + * {@link ProcessBuilderForWin32.Redirect.Type#WRITE WRITE} or + * {@link ProcessBuilderForWin32.Redirect.Type#APPEND APPEND} + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectInput(ProcessBuilderForWin32.Redirect source) { + if (source.type() == ProcessBuilderForWin32.Redirect.Type.WRITE || + source.type() == ProcessBuilderForWin32.Redirect.Type.APPEND) + throw new IllegalArgumentException( + "Redirect invalid for reading: " + source); + redirects()[0] = source; + return this; + } + + /** + * Sets this process builder's standard output destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method send their standard output to this destination. + * + *

If the destination is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE} + * (the initial value), then the standard output of a subprocess + * can be read using the input stream returned by {@link + * Process#getInputStream()}. + * If the destination is set to any other value, then + * {@link Process#getInputStream()} will return a + * null input stream. + * + * @param destination the new standard output destination + * @return this process builder + * @throws IllegalArgumentException + * if the redirect does not correspond to a valid + * destination of data, that is, has type + * {@link ProcessBuilderForWin32.Redirect.Type#READ READ} + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectOutput(ProcessBuilderForWin32.Redirect destination) { + if (destination.type() == ProcessBuilderForWin32.Redirect.Type.READ) + throw new IllegalArgumentException( + "Redirect invalid for writing: " + destination); + redirects()[1] = destination; + return this; + } + + /** + * Sets this process builder's standard error destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method send their standard error to this destination. + * + *

If the destination is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE} + * (the initial value), then the error output of a subprocess + * can be read using the input stream returned by {@link + * Process#getErrorStream()}. + * If the destination is set to any other value, then + * {@link Process#getErrorStream()} will return a + * null input stream. + * + *

If the {@link #redirectErrorStream redirectErrorStream} + * attribute has been set {@code true}, then the redirection set + * by this method has no effect. + * + * @param destination the new standard error destination + * @return this process builder + * @throws IllegalArgumentException + * if the redirect does not correspond to a valid + * destination of data, that is, has type + * {@link ProcessBuilderForWin32.Redirect.Type#READ READ} + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectError(ProcessBuilderForWin32.Redirect destination) { + if (destination.type() == ProcessBuilderForWin32.Redirect.Type.READ) + throw new IllegalArgumentException( + "Redirect invalid for writing: " + destination); + redirects()[2] = destination; + return this; + } + + /** + * Sets this process builder's standard input source to a file. + * + *

This is a convenience method. An invocation of the form + * {@code redirectInput(file)} + * behaves in exactly the same way as the invocation + * {@link #redirectInput(ProcessBuilderForWin32.Redirect) redirectInput} + * {@code (Redirect.from(file))}. + * + * @param file the new standard input source + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectInput(File file) { + return redirectInput(ProcessBuilderForWin32.Redirect.from(file)); + } + + /** + * Sets this process builder's standard output destination to a file. + * + *

This is a convenience method. An invocation of the form + * {@code redirectOutput(file)} + * behaves in exactly the same way as the invocation + * {@link #redirectOutput(ProcessBuilderForWin32.Redirect) redirectOutput} + * {@code (Redirect.to(file))}. + * + * @param file the new standard output destination + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectOutput(File file) { + return redirectOutput(ProcessBuilderForWin32.Redirect.to(file)); + } + + /** + * Sets this process builder's standard error destination to a file. + * + *

This is a convenience method. An invocation of the form + * {@code redirectError(file)} + * behaves in exactly the same way as the invocation + * {@link #redirectError(ProcessBuilderForWin32.Redirect) redirectError} + * {@code (Redirect.to(file))}. + * + * @param file the new standard error destination + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectError(File file) { + return redirectError(ProcessBuilderForWin32.Redirect.to(file)); + } + + /** + * Returns this process builder's standard input source. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method obtain their standard input from this source. + * The initial value is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + * + * @return this process builder's standard input source + * @since 1.7 + */ + public ProcessBuilderForWin32.Redirect redirectInput() { + return (redirects == null) ? ProcessBuilderForWin32.Redirect.PIPE : redirects[0]; + } + + /** + * Returns this process builder's standard output destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method redirect their standard output to this destination. + * The initial value is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + * + * @return this process builder's standard output destination + * @since 1.7 + */ + public ProcessBuilderForWin32.Redirect redirectOutput() { + return (redirects == null) ? ProcessBuilderForWin32.Redirect.PIPE : redirects[1]; + } + + /** + * Returns this process builder's standard error destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method redirect their standard error to this destination. + * The initial value is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + * + * @return this process builder's standard error destination + * @since 1.7 + */ + public ProcessBuilderForWin32.Redirect redirectError() { + return (redirects == null) ? ProcessBuilderForWin32.Redirect.PIPE : redirects[2]; + } + + /** + * Sets the source and destination for subprocess standard I/O + * to be the same as those of the current Java process. + * + *

This is a convenience method. An invocation of the form + *

 {@code
+     * pb.inheritIO()
+     * }
+ * behaves in exactly the same way as the invocation + *
 {@code
+     * pb.redirectInput(Redirect.INHERIT)
+     *   .redirectOutput(Redirect.INHERIT)
+     *   .redirectError(Redirect.INHERIT)
+     * }
+ * + * This gives behavior equivalent to most operating system + * command interpreters, or the standard C library function + * {@code system()}. + * + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 inheritIO() { + Arrays.fill(redirects(), ProcessBuilderForWin32.Redirect.INHERIT); + return this; + } + + /** + * Tells whether this process builder merges standard error and + * standard output. + * + *

If this property is {@code true}, then any error output + * generated by subprocesses subsequently started by this object's + * {@link #start()} method will be merged with the standard + * output, so that both can be read using the + * {@link Process#getInputStream()} method. This makes it easier + * to correlate error messages with the corresponding output. + * The initial value is {@code false}. + * + * @return this process builder's {@code redirectErrorStream} property + */ + public boolean redirectErrorStream() { + return redirectErrorStream; + } + + /** + * Sets this process builder's {@code redirectErrorStream} property. + * + *

If this property is {@code true}, then any error output + * generated by subprocesses subsequently started by this object's + * {@link #start()} method will be merged with the standard + * output, so that both can be read using the + * {@link Process#getInputStream()} method. This makes it easier + * to correlate error messages with the corresponding output. + * The initial value is {@code false}. + * + * @param redirectErrorStream the new property value + * @return this process builder + */ + public ProcessBuilderForWin32 redirectErrorStream(boolean redirectErrorStream) { + this.redirectErrorStream = redirectErrorStream; + return this; + } + + /** + * Starts a new process using the attributes of this process builder. + * + *

The new process will + * invoke the command and arguments given by {@link #command()}, + * in a working directory as given by {@link #directory()}, + * with a process environment as given by {@link #environment()}. + * + *

This method checks that the command is a valid operating + * system command. Which commands are valid is system-dependent, + * but at the very least the command must be a non-empty list of + * non-null strings. + * + *

A minimal set of system dependent environment variables may + * be required to start a process on some operating systems. + * As a result, the subprocess may inherit additional environment variable + * settings beyond those in the process builder's {@link #environment()}. + * + *

If there is a security manager, its + * {@link SecurityManager#checkExec checkExec} + * method is called with the first component of this object's + * {@code command} array as its argument. This may result in + * a {@link SecurityException} being thrown. + * + *

Starting an operating system process is highly system-dependent. + * Among the many things that can go wrong are: + *

    + *
  • The operating system program file was not found. + *
  • Access to the program file was denied. + *
  • The working directory does not exist. + *
+ * + *

In such cases an exception will be thrown. The exact nature + * of the exception is system-dependent, but it will always be a + * subclass of {@link IOException}. + * + *

Subsequent modifications to this process builder will not + * affect the returned {@link Process}. + * + * @return a new {@link Process} object for managing the subprocess + * + * @throws NullPointerException + * if an element of the command list is null + * + * @throws IndexOutOfBoundsException + * if the command is an empty list (has size {@code 0}) + * + * @throws SecurityException + * if a security manager exists and + *

    + * + *
  • its + * {@link SecurityManager#checkExec checkExec} + * method doesn't allow creation of the subprocess, or + * + *
  • the standard input to the subprocess was + * {@linkplain #redirectInput redirected from a file} + * and the security manager's + * {@link SecurityManager#checkRead checkRead} method + * denies read access to the file, or + * + *
  • the standard output or standard error of the + * subprocess was + * {@linkplain #redirectOutput redirected to a file} + * and the security manager's + * {@link SecurityManager#checkWrite checkWrite} method + * denies write access to the file + * + *
+ * + * @throws IOException if an I/O error occurs + * + * @see Runtime#exec(String[], String[], File) + */ + public Process start() throws IOException { + // Must convert to array first -- a malicious user-supplied + // list might try to circumvent the security check. + String[] cmdarray = command.toArray(new String[command.size()]); + cmdarray = cmdarray.clone(); + + for (String arg : cmdarray) + if (arg == null) + throw new NullPointerException(); + // Throws IndexOutOfBoundsException if command is empty + String prog = cmdarray[0]; + + SecurityManager security = System.getSecurityManager(); + if (security != null) + security.checkExec(prog); + + String dir = directory == null ? null : directory.toString(); + + for (int i = 1; i < cmdarray.length; i++) { + if (cmdarray[i].indexOf('\u0000') >= 0) { + throw new IOException("invalid null character in command"); + } + } + + try { + return ProcessImplForWin32.start( + username, + password, + cmdarray, + environment, + dir, + redirects, + redirectErrorStream); + } catch (IOException | IllegalArgumentException e) { + String exceptionInfo = ": " + e.getMessage(); + Throwable cause = e; + if ((e instanceof IOException) && security != null) { + // Can not disclose the fail reason for read-protected files. + try { + security.checkRead(prog); + } catch (SecurityException se) { + exceptionInfo = ""; + cause = se; + } + } + // It's much easier for us to create a high-quality error + // message than the low-level C code which found the problem. + throw new IOException( + "Cannot run program \"" + prog + "\"" + + (dir == null ? "" : " (in directory \"" + dir + "\")") + + exceptionInfo, + cause); + } + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java new file mode 100644 index 0000000000..3dbe7cb50f --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import com.sun.jna.platform.win32.Kernel32Util; + +import java.util.*; + +final class ProcessEnvironmentForWin32 extends HashMap { + + private static final long serialVersionUID = -8017839552603542824L; + + private static String validateName(String name) { + // An initial `=' indicates a magic Windows variable name -- OK + if (name.indexOf('=', 1) != -1 || + name.indexOf('\u0000') != -1) + throw new IllegalArgumentException + ("Invalid environment variable name: \"" + name + "\""); + return name; + } + + private static String validateValue(String value) { + if (value.indexOf('\u0000') != -1) + throw new IllegalArgumentException + ("Invalid environment variable value: \"" + value + "\""); + return value; + } + + private static String nonNullString(Object o) { + if (o == null) + throw new NullPointerException(); + return (String) o; + } + + public String put(String key, String value) { + return super.put(validateName(key), validateValue(value)); + } + + public String get(Object key) { + return super.get(nonNullString(key)); + } + + public boolean containsKey(Object key) { + return super.containsKey(nonNullString(key)); + } + + public boolean containsValue(Object value) { + return super.containsValue(nonNullString(value)); + } + + public String remove(Object key) { + return super.remove(nonNullString(key)); + } + + private static class CheckedEntry implements Entry { + private final Entry e; + public CheckedEntry(Entry e) {this.e = e;} + public String getKey() { return e.getKey();} + public String getValue() { return e.getValue();} + public String setValue(String value) { + return e.setValue(validateValue(value)); + } + public String toString() { return getKey() + "=" + getValue();} + public boolean equals(Object o) {return e.equals(o);} + public int hashCode() {return e.hashCode();} + } + + private static class CheckedEntrySet extends AbstractSet> { + private final Set> s; + public CheckedEntrySet(Set> s) {this.s = s;} + public int size() {return s.size();} + public boolean isEmpty() {return s.isEmpty();} + public void clear() { s.clear();} + public Iterator> iterator() { + return new Iterator>() { + Iterator> i = s.iterator(); + public boolean hasNext() { return i.hasNext();} + public Entry next() { + return new CheckedEntry(i.next()); + } + public void remove() { i.remove();} + }; + } + private static Entry checkedEntry(Object o) { + @SuppressWarnings("unchecked") + Entry e = (Entry) o; + nonNullString(e.getKey()); + nonNullString(e.getValue()); + return e; + } + public boolean contains(Object o) {return s.contains(checkedEntry(o));} + public boolean remove(Object o) {return s.remove(checkedEntry(o));} + } + + private static class CheckedValues extends AbstractCollection { + private final Collection c; + public CheckedValues(Collection c) {this.c = c;} + public int size() {return c.size();} + public boolean isEmpty() {return c.isEmpty();} + public void clear() { c.clear();} + public Iterator iterator() {return c.iterator();} + public boolean contains(Object o) {return c.contains(nonNullString(o));} + public boolean remove(Object o) {return c.remove(nonNullString(o));} + } + + private static class CheckedKeySet extends AbstractSet { + private final Set s; + public CheckedKeySet(Set s) {this.s = s;} + public int size() {return s.size();} + public boolean isEmpty() {return s.isEmpty();} + public void clear() { s.clear();} + public Iterator iterator() {return s.iterator();} + public boolean contains(Object o) {return s.contains(nonNullString(o));} + public boolean remove(Object o) {return s.remove(nonNullString(o));} + } + + public Set keySet() { + return new CheckedKeySet(super.keySet()); + } + + public Collection values() { + return new CheckedValues(super.values()); + } + + public Set> entrySet() { + return new CheckedEntrySet(super.entrySet()); + } + + private static final class NameComparator implements Comparator { + public int compare(String s1, String s2) { + // We can't use String.compareToIgnoreCase since it + // canonicalizes to lower case, while Windows + // canonicalizes to upper case! For example, "_" should + // sort *after* "Z", not before. + int n1 = s1.length(); + int n2 = s2.length(); + int min = Math.min(n1, n2); + for (int i = 0; i < min; i++) { + char c1 = s1.charAt(i); + char c2 = s2.charAt(i); + if (c1 != c2) { + c1 = Character.toUpperCase(c1); + c2 = Character.toUpperCase(c2); + if (c1 != c2) + // No overflow because of numeric promotion + return c1 - c2; + } + } + return n1 - n2; + } + } + + private static final class EntryComparator implements Comparator> { + public int compare(Entry e1, + Entry e2) { + return nameComparator.compare(e1.getKey(), e2.getKey()); + } + } + + // Allow `=' as first char in name, e.g. =C:=C:\DIR + static final int MIN_NAME_LENGTH = 1; + + private static final NameComparator nameComparator; + private static final EntryComparator entryComparator; + private static final ProcessEnvironmentForWin32 theEnvironment; + private static final Map theUnmodifiableEnvironment; + private static final Map theCaseInsensitiveEnvironment; + + static { + nameComparator = new NameComparator(); + entryComparator = new EntryComparator(); + theEnvironment = new ProcessEnvironmentForWin32(); + theUnmodifiableEnvironment = Collections.unmodifiableMap(theEnvironment); + + theEnvironment.putAll(environmentBlock()); + + theCaseInsensitiveEnvironment = new TreeMap<>(nameComparator); + theCaseInsensitiveEnvironment.putAll(theEnvironment); + } + + private ProcessEnvironmentForWin32() { + super(); + } + + private ProcessEnvironmentForWin32(int capacity) { + super(capacity); + } + + // Only for use by System.getenv(String) + static String getenv(String name) { + // The original implementation used a native call to _wgetenv, + // but it turns out that _wgetenv is only consistent with + // GetEnvironmentStringsW (for non-ASCII) if `wmain' is used + // instead of `main', even in a process created using + // CREATE_UNICODE_ENVIRONMENT. Instead we perform the + // case-insensitive comparison ourselves. At least this + // guarantees that System.getenv().get(String) will be + // consistent with System.getenv(String). + return theCaseInsensitiveEnvironment.get(name); + } + + // Only for use by System.getenv() + static Map getenv() { + return theUnmodifiableEnvironment; + } + + // Only for use by ProcessBuilder.environment() + @SuppressWarnings("unchecked") + static Map environment() { + return (Map) theEnvironment.clone(); + } + + // Only for use by ProcessBuilder.environment(String[] envp) + static Map emptyEnvironment(int capacity) { + return new ProcessEnvironmentForWin32(capacity); + } + + private static Map environmentBlock() { + return Kernel32Util.getEnvironmentVariables(); + } + + // Only for use by ProcessImpl.start() + String toEnvironmentBlock() { + // Sort Unicode-case-insensitively by name + List> list = new ArrayList<>(entrySet()); + Collections.sort(list, entryComparator); + + StringBuilder sb = new StringBuilder(size()*30); + int cmp = -1; + + // Some versions of MSVCRT.DLL require SystemRoot to be set. + // So, we make sure that it is always set, even if not provided + // by the caller. + final String SYSTEMROOT = "SystemRoot"; + + for (Entry e : list) { + String key = e.getKey(); + String value = e.getValue(); + if (cmp < 0 && (cmp = nameComparator.compare(key, SYSTEMROOT)) > 0) { + // Not set, so add it here + addToEnvIfSet(sb, SYSTEMROOT); + } + addToEnv(sb, key, value); + } + if (cmp < 0) { + // Got to end of list and still not found + addToEnvIfSet(sb, SYSTEMROOT); + } + if (sb.length() == 0) { + // Environment was empty and SystemRoot not set in parent + sb.append('\u0000'); + } + // Block is double NUL terminated + sb.append('\u0000'); + return sb.toString(); + } + + // add the environment variable to the child, if it exists in parent + private static void addToEnvIfSet(StringBuilder sb, String name) { + String s = getenv(name); + if (s != null) + addToEnv(sb, name, s); + } + + private static void addToEnv(StringBuilder sb, String name, String val) { + sb.append(name).append('=').append(val).append('\u0000'); + } + + static String toEnvironmentBlock(Map map) { + return map == null ? null : ((ProcessEnvironmentForWin32)map).toEnvironmentBlock(); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java new file mode 100644 index 0000000000..4583be8aff --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java @@ -0,0 +1,752 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import com.sun.jna.Pointer; +import com.sun.jna.platform.win32.*; +import com.sun.jna.ptr.IntByReference; +import sun.security.action.GetPropertyAction; + +import java.io.*; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE; + +public class ProcessImplForWin32 extends Process { + private static final sun.misc.JavaIOFileDescriptorAccess fdAccess + = sun.misc.SharedSecrets.getJavaIOFileDescriptorAccess(); + + private static final int PIPE_SIZE = 4096 + 24; + + private static final int HANDLE_STORAGE_SIZE = 6; + + private static final int OFFSET_READ = 0; + + private static final int OFFSET_WRITE = 1; + + private static final WinNT.HANDLE JAVA_INVALID_HANDLE_VALUE = new WinNT.HANDLE(Pointer.createConstant(-1)); + + /** + * Open a file for writing. If {@code append} is {@code true} then the file + * is opened for atomic append directly and a FileOutputStream constructed + * with the resulting handle. This is because a FileOutputStream created + * to append to a file does not open the file in a manner that guarantees + * that writes by the child process will be atomic. + */ + private static FileOutputStream newFileOutputStream(File f, boolean append) + throws IOException + { + if (append) { + String path = f.getPath(); + SecurityManager sm = System.getSecurityManager(); + if (sm != null) + sm.checkWrite(path); + long handle = openForAtomicAppend(path); + final FileDescriptor fd = new FileDescriptor(); + fdAccess.setHandle(fd, handle); + return AccessController.doPrivileged( + new PrivilegedAction() { + public FileOutputStream run() { + return new FileOutputStream(fd); + } + } + ); + } else { + return new FileOutputStream(f); + } + } + + // System-dependent portion of ProcessBuilderForWindows.start() + static Process start(String username, + String password, + String cmdarray[], + java.util.Map environment, + String dir, + ProcessBuilderForWin32.Redirect[] redirects, + boolean redirectErrorStream) + throws IOException + { + String envblock = ProcessEnvironmentForWin32.toEnvironmentBlock(environment); + + FileInputStream f0 = null; + FileOutputStream f1 = null; + FileOutputStream f2 = null; + + try { + long[] stdHandles; + if (redirects == null) { + stdHandles = new long[] { -1L, -1L, -1L }; + } else { + stdHandles = new long[3]; + + if (redirects[0] == ProcessBuilderForWin32.Redirect.PIPE) + stdHandles[0] = -1L; + else if (redirects[0] == ProcessBuilderForWin32.Redirect.INHERIT) + stdHandles[0] = fdAccess.getHandle(FileDescriptor.in); + else { + f0 = new FileInputStream(redirects[0].file()); + stdHandles[0] = fdAccess.getHandle(f0.getFD()); + } + + if (redirects[1] == ProcessBuilderForWin32.Redirect.PIPE) + stdHandles[1] = -1L; + else if (redirects[1] == ProcessBuilderForWin32.Redirect.INHERIT) + stdHandles[1] = fdAccess.getHandle(FileDescriptor.out); + else { + f1 = newFileOutputStream(redirects[1].file(), + redirects[1].append()); + stdHandles[1] = fdAccess.getHandle(f1.getFD()); + } + + if (redirects[2] == ProcessBuilderForWin32.Redirect.PIPE) + stdHandles[2] = -1L; + else if (redirects[2] == ProcessBuilderForWin32.Redirect.INHERIT) + stdHandles[2] = fdAccess.getHandle(FileDescriptor.err); + else { + f2 = newFileOutputStream(redirects[2].file(), + redirects[2].append()); + stdHandles[2] = fdAccess.getHandle(f2.getFD()); + } + } + + return new ProcessImplForWin32(username, password, cmdarray, envblock, dir, stdHandles, redirectErrorStream); + } finally { + // In theory, close() can throw IOException + // (although it is rather unlikely to happen here) + try { if (f0 != null) f0.close(); } + finally { + try { if (f1 != null) f1.close(); } + finally { if (f2 != null) f2.close(); } + } + } + + } + + private static class LazyPattern { + // Escape-support version: + // "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)"; + private static final Pattern PATTERN = + Pattern.compile("[^\\s\"]+|\"[^\"]*\""); + }; + + /* Parses the command string parameter into the executable name and + * program arguments. + * + * The command string is broken into tokens. The token separator is a space + * or quota character. The space inside quotation is not a token separator. + * There are no escape sequences. + */ + private static String[] getTokensFromCommand(String command) { + ArrayList matchList = new ArrayList<>(8); + Matcher regexMatcher = ProcessImplForWin32.LazyPattern.PATTERN.matcher(command); + while (regexMatcher.find()) + matchList.add(regexMatcher.group()); + return matchList.toArray(new String[matchList.size()]); + } + + private static final int VERIFICATION_CMD_BAT = 0; + private static final int VERIFICATION_WIN32 = 1; + private static final int VERIFICATION_WIN32_SAFE = 2; // inside quotes not allowed + private static final int VERIFICATION_LEGACY = 3; + // See Command shell overview for documentation of special characters. + // https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-xp/bb490954(v=technet.10) + private static final char ESCAPE_VERIFICATION[][] = { + // We guarantee the only command file execution for implicit [cmd.exe] run. + // http://technet.microsoft.com/en-us/library/bb490954.aspx + {' ', '\t', '<', '>', '&', '|', '^'}, + {' ', '\t', '<', '>'}, + {' ', '\t', '<', '>'}, + {' ', '\t'} + }; + + private static String createCommandLine(int verificationType, + final String executablePath, + final String cmd[]) + { + StringBuilder cmdbuf = new StringBuilder(80); + + cmdbuf.append(executablePath); + + for (int i = 1; i < cmd.length; ++i) { + cmdbuf.append(' '); + String s = cmd[i]; + if (needsEscaping(verificationType, s)) { + cmdbuf.append('"'); + + if (verificationType == VERIFICATION_WIN32_SAFE) { + // Insert the argument, adding '\' to quote any interior quotes + int length = s.length(); + for (int j = 0; j < length; j++) { + char c = s.charAt(j); + if (c == DOUBLEQUOTE) { + int count = countLeadingBackslash(verificationType, s, j); + while (count-- > 0) { + cmdbuf.append(BACKSLASH); // double the number of backslashes + } + cmdbuf.append(BACKSLASH); // backslash to quote the quote + } + cmdbuf.append(c); + } + } else { + cmdbuf.append(s); + } + // The code protects the [java.exe] and console command line + // parser, that interprets the [\"] combination as an escape + // sequence for the ["] char. + // http://msdn.microsoft.com/en-us/library/17w5ykft.aspx + // + // If the argument is an FS path, doubling of the tail [\] + // char is not a problem for non-console applications. + // + // The [\"] sequence is not an escape sequence for the [cmd.exe] + // command line parser. The case of the [""] tail escape + // sequence could not be realized due to the argument validation + // procedure. + int count = countLeadingBackslash(verificationType, s, s.length()); + while (count-- > 0) { + cmdbuf.append(BACKSLASH); // double the number of backslashes + } + cmdbuf.append('"'); + } else { + cmdbuf.append(s); + } + } + return cmdbuf.toString(); + } + + /** + * Return the argument without quotes (1st and last) if present, else the arg. + * @param str a string + * @return the string without 1st and last quotes + */ + private static String unQuote(String str) { + int len = str.length(); + return (len >= 2 && str.charAt(0) == DOUBLEQUOTE && str.charAt(len - 1) == DOUBLEQUOTE) + ? str.substring(1, len - 1) + : str; + } + + private static boolean needsEscaping(int verificationType, String arg) { + // Switch off MS heuristic for internal ["]. + // Please, use the explicit [cmd.exe] call + // if you need the internal ["]. + // Example: "cmd.exe", "/C", "Extended_MS_Syntax" + + // For [.exe] or [.com] file the unpaired/internal ["] + // in the argument is not a problem. + String unquotedArg = unQuote(arg); + boolean argIsQuoted = !arg.equals(unquotedArg); + boolean embeddedQuote = unquotedArg.indexOf(DOUBLEQUOTE) >= 0; + + switch (verificationType) { + case VERIFICATION_CMD_BAT: + if (embeddedQuote) { + throw new IllegalArgumentException("Argument has embedded quote, " + + "use the explicit CMD.EXE call."); + } + break; // break determine whether to quote + case VERIFICATION_WIN32_SAFE: + if (argIsQuoted && embeddedQuote) { + throw new IllegalArgumentException("Malformed argument has embedded quote: " + + unquotedArg); + } + break; + default: + break; + } + + if (!argIsQuoted) { + char testEscape[] = ESCAPE_VERIFICATION[verificationType]; + for (int i = 0; i < testEscape.length; ++i) { + if (arg.indexOf(testEscape[i]) >= 0) { + return true; + } + } + } + return false; + } + + private static String getExecutablePath(String path) + throws IOException + { + String name = unQuote(path); + if (name.indexOf(DOUBLEQUOTE) >= 0) { + throw new IllegalArgumentException("Executable name has embedded quote, " + + "split the arguments: " + name); + } + // Win32 CreateProcess requires path to be normalized + File fileToRun = new File(name); + + // From the [CreateProcess] function documentation: + // + // "If the file name does not contain an extension, .exe is appended. + // Therefore, if the file name extension is .com, this parameter + // must include the .com extension. If the file name ends in + // a period (.) with no extension, or if the file name contains a path, + // .exe is not appended." + // + // "If the file name !does not contain a directory path!, + // the system searches for the executable file in the following + // sequence:..." + // + // In practice ANY non-existent path is extended by [.exe] extension + // in the [CreateProcess] function with the only exception: + // the path ends by (.) + + return fileToRun.getPath(); + } + + /** + * An executable is any program that is an EXE or does not have an extension + * and the Windows createProcess will be looking for .exe. + * The comparison is case insensitive based on the name. + * @param executablePath the executable file + * @return true if the path ends in .exe or does not have an extension. + */ + private boolean isExe(String executablePath) { + File file = new File(executablePath); + String upName = file.getName().toUpperCase(Locale.ROOT); + return (upName.endsWith(".EXE") || upName.indexOf('.') < 0); + } + + // Old version that can be bypassed + private boolean isShellFile(String executablePath) { + String upPath = executablePath.toUpperCase(); + return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT")); + } + + private String quoteString(String arg) { + StringBuilder argbuf = new StringBuilder(arg.length() + 2); + return argbuf.append('"').append(arg).append('"').toString(); + } + + // Count backslashes before start index of string. + // .bat files don't include backslashes as part of the quote + private static int countLeadingBackslash(int verificationType, + CharSequence input, int start) { + if (verificationType == VERIFICATION_CMD_BAT) + return 0; + int j; + for (j = start - 1; j >= 0 && input.charAt(j) == BACKSLASH; j--) { + // just scanning backwards + } + return (start - 1) - j; // number of BACKSLASHES + } + + private static final char DOUBLEQUOTE = '\"'; + private static final char BACKSLASH = '\\'; + + private WinNT.HANDLE handle; + private OutputStream stdin_stream; + private InputStream stdout_stream; + private InputStream stderr_stream; + + private ProcessImplForWin32( + String username, + String password, + String cmd[], + final String envblock, + final String path, + final long[] stdHandles, + final boolean redirectErrorStream) + throws IOException + { + String cmdstr; + final SecurityManager security = System.getSecurityManager(); + GetPropertyAction action = new GetPropertyAction("jdk.lang.Process.allowAmbiguousCommands", + (security == null) ? "true" : "false"); + final boolean allowAmbiguousCommands = !"false".equalsIgnoreCase(action.run()); + if (allowAmbiguousCommands && security == null) { + // Legacy mode. + + // Normalize path if possible. + String executablePath = new File(cmd[0]).getPath(); + + // No worry about internal, unpaired ["], and redirection/piping. + if (needsEscaping(VERIFICATION_LEGACY, executablePath) ) + executablePath = quoteString(executablePath); + + cmdstr = createCommandLine( + //legacy mode doesn't worry about extended verification + VERIFICATION_LEGACY, + executablePath, + cmd); + } else { + String executablePath; + try { + executablePath = getExecutablePath(cmd[0]); + } catch (IllegalArgumentException e) { + // Workaround for the calls like + // Runtime.getRuntime().exec("\"C:\\Program Files\\foo\" bar") + + // No chance to avoid CMD/BAT injection, except to do the work + // right from the beginning. Otherwise we have too many corner + // cases from + // Runtime.getRuntime().exec(String[] cmd [, ...]) + // calls with internal ["] and escape sequences. + + // Restore original command line. + StringBuilder join = new StringBuilder(); + // terminal space in command line is ok + for (String s : cmd) + join.append(s).append(' '); + + // Parse the command line again. + cmd = getTokensFromCommand(join.toString()); + executablePath = getExecutablePath(cmd[0]); + + // Check new executable name once more + if (security != null) + security.checkExec(executablePath); + } + + // Quotation protects from interpretation of the [path] argument as + // start of longer path with spaces. Quotation has no influence to + // [.exe] extension heuristic. + boolean isShell = allowAmbiguousCommands ? isShellFile(executablePath) + : !isExe(executablePath); + cmdstr = createCommandLine( + // We need the extended verification procedures + isShell ? VERIFICATION_CMD_BAT + : (allowAmbiguousCommands ? VERIFICATION_WIN32 : VERIFICATION_WIN32_SAFE), + quoteString(executablePath), + cmd); + } + + handle = create(username, password, cmdstr, envblock, path, stdHandles, redirectErrorStream); + + AccessController.doPrivileged( + new PrivilegedAction() { + public Void run() { + if (stdHandles[0] == -1L) + stdin_stream = ProcessBuilderForWin32.NullOutputStream.INSTANCE; + else { + FileDescriptor stdin_fd = new FileDescriptor(); + fdAccess.setHandle(stdin_fd, stdHandles[0]); + stdin_stream = new BufferedOutputStream( + new FileOutputStream(stdin_fd)); + } + + if (stdHandles[1] == -1L) + stdout_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; + else { + FileDescriptor stdout_fd = new FileDescriptor(); + fdAccess.setHandle(stdout_fd, stdHandles[1]); + stdout_stream = new BufferedInputStream( + new FileInputStream(stdout_fd)); + } + + if (stdHandles[2] == -1L) + stderr_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; + else { + FileDescriptor stderr_fd = new FileDescriptor(); + fdAccess.setHandle(stderr_fd, stdHandles[2]); + stderr_stream = new FileInputStream(stderr_fd); + } + + return null; }}); + } + + public OutputStream getOutputStream() { + return stdin_stream; + } + + public InputStream getInputStream() { + return stdout_stream; + } + + public InputStream getErrorStream() { + return stderr_stream; + } + + protected void finalize() { + closeHandle(handle); + } + + public int exitValue() { + int exitCode = getExitCodeProcess(handle); + if (exitCode == STILL_ACTIVE) + throw new IllegalThreadStateException("process has not exited"); + return exitCode; + } + + public int waitFor() throws InterruptedException { + waitForInterruptibly(handle); + if (Thread.interrupted()) + throw new InterruptedException(); + return exitValue(); + } + + @Override + public boolean waitFor(long timeout, TimeUnit unit) + throws InterruptedException + { + if (getExitCodeProcess(handle) != STILL_ACTIVE) return true; + if (timeout <= 0) return false; + + long remainingNanos = unit.toNanos(timeout); + long deadline = System.nanoTime() + remainingNanos ; + + do { + // Round up to next millisecond + long msTimeout = TimeUnit.NANOSECONDS.toMillis(remainingNanos + 999_999L); + waitForTimeoutInterruptibly(handle, msTimeout); + if (Thread.interrupted()) + throw new InterruptedException(); + if (getExitCodeProcess(handle) != STILL_ACTIVE) { + return true; + } + remainingNanos = deadline - System.nanoTime(); + } while (remainingNanos > 0); + + return (getExitCodeProcess(handle) != STILL_ACTIVE); + } + + public void destroy() { terminateProcess(handle); } + + public Process destroyForcibly() { + destroy(); + return this; + } + + public boolean isAlive() { + return isProcessAlive(handle); + } + + private static boolean initHolder(WinNT.HANDLEByReference pjhandles, + WinNT.HANDLEByReference[] pipe, + int offset, + WinNT.HANDLEByReference phStd) { + if (!pjhandles.getValue().equals(JAVA_INVALID_HANDLE_VALUE)) { + phStd.setValue(pjhandles.getValue()); + pjhandles.setValue(JAVA_INVALID_HANDLE_VALUE); + } else { + if (!Kernel32.INSTANCE.CreatePipe(pipe[0], pipe[1], null, PIPE_SIZE)) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } else { + WinNT.HANDLE thisProcessEnd = offset == OFFSET_READ ? pipe[1].getValue() : pipe[0].getValue(); + phStd.setValue(pipe[offset].getValue()); + pjhandles.setValue(thisProcessEnd); + } + } + Kernel32.INSTANCE.SetHandleInformation(phStd.getValue(), Kernel32.HANDLE_FLAG_INHERIT, Kernel32.HANDLE_FLAG_INHERIT); + return true; + } + + private static void releaseHolder(boolean complete, WinNT.HANDLEByReference[] pipe, int offset) { + closeHandle(pipe[offset].getValue()); + if (complete) { + closeHandle(pipe[offset == OFFSET_READ ? OFFSET_WRITE : OFFSET_READ].getValue()); + } + } + + private static void prepareIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) { + for(int i = 0; i < HANDLE_STORAGE_SIZE; ++i) { + WinNT.HANDLE hstd = stdIOE[i]; + if (!Kernel32.INVALID_HANDLE_VALUE.equals(hstd)) { + inherit[i] = Boolean.TRUE; + Kernel32.INSTANCE.SetHandleInformation(hstd, Kernel32.HANDLE_FLAG_INHERIT, 0); + } + } + } + + private static void restoreIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) { + for (int i = HANDLE_STORAGE_SIZE - 1; i >= 0; --i) { + if (!Kernel32.INVALID_HANDLE_VALUE.equals(stdIOE[i])) { + Kernel32.INSTANCE.SetHandleInformation(stdIOE[i], Kernel32.HANDLE_FLAG_INHERIT, inherit[i] ? Kernel32.HANDLE_FLAG_INHERIT : 0); + } + } + } + + private static WinNT.HANDLE processCreate(String username, + String password, + String cmd, + final String envblock, + final String path, + final WinNT.HANDLEByReference[] stdHandles, + final boolean redirectErrorStream) { + WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0)); + + WinNT.HANDLE[] stdIOE = new WinNT.HANDLE[] { + Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE, + stdHandles[0].getValue(), stdHandles[1].getValue(), stdHandles[2].getValue() + }; + stdIOE[0] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_INPUT_HANDLE); + stdIOE[1] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_OUTPUT_HANDLE); + stdIOE[2] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_ERROR_HANDLE); + + Boolean[] inherit = new Boolean[] { + Boolean.FALSE, Boolean.FALSE, Boolean.FALSE, + Boolean.FALSE, Boolean.FALSE, Boolean.FALSE + }; + + prepareIOEHandleState(stdIOE, inherit); + + // input + WinNT.HANDLEByReference hStdInput = new WinNT.HANDLEByReference(); + WinNT.HANDLEByReference[] pipeIn = new WinNT.HANDLEByReference[] { + new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + + // output + WinNT.HANDLEByReference hStdOutput = new WinNT.HANDLEByReference(); + WinNT.HANDLEByReference[] pipeOut = new WinNT.HANDLEByReference[] { + new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + + // error + WinNT.HANDLEByReference hStdError = new WinNT.HANDLEByReference(); + WinNT.HANDLEByReference[] pipeError = new WinNT.HANDLEByReference[] { + new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + + boolean success; + if (initHolder(stdHandles[0], pipeIn, OFFSET_READ, hStdInput)) { + if (initHolder(stdHandles[1], pipeOut, OFFSET_WRITE, hStdOutput)) { + WinBase.STARTUPINFO si = new WinBase.STARTUPINFO(); + si.hStdInput = hStdInput.getValue(); + si.hStdOutput = hStdOutput.getValue(); + + if (redirectErrorStream) { + si.hStdError = si.hStdOutput; + stdHandles[2].setValue(JAVA_INVALID_HANDLE_VALUE); + success = true; + } else { + success = initHolder(stdHandles[2], pipeError, OFFSET_WRITE, hStdError); + si.hStdError = hStdError.getValue(); + } + + if (success) { + WTypes.LPSTR lpEnvironment = envblock == null ? new WTypes.LPSTR() : new WTypes.LPSTR(envblock); + Kernel32.PROCESS_INFORMATION pi = new WinBase.PROCESS_INFORMATION(); + si.dwFlags = Kernel32.STARTF_USESTDHANDLES; + if (!Advapi32.INSTANCE.CreateProcessWithLogonW( + username + , null + , password + , Advapi32.LOGON_WITH_PROFILE + , null + , cmd + , Kernel32.CREATE_NO_WINDOW + , lpEnvironment.getPointer() + , path + , si + , pi)) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } else { + closeHandle(pi.hThread); + ret = pi.hProcess; + } + } + releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeError, OFFSET_WRITE); + releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeOut, OFFSET_WRITE); + } + releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeIn, OFFSET_READ); + } + restoreIOEHandleState(stdIOE, inherit); + return ret; + } + + private static synchronized WinNT.HANDLE create(String username, + String password, + String cmd, + final String envblock, + final String path, + final long[] stdHandles, + final boolean redirectErrorStream) { + WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0)); + WinNT.HANDLEByReference[] handles = new WinNT.HANDLEByReference[stdHandles.length]; + for (int i = 0; i < stdHandles.length; i++) { + handles[i] = new WinNT.HANDLEByReference(new WinNT.HANDLE(Pointer.createConstant(stdHandles[i]))); + } + + if (cmd != null) { + if (username != null && password != null) { + ret = processCreate(username, password, cmd, envblock, path, handles, redirectErrorStream); + } + } + + for (int i = 0; i < stdHandles.length; i++) { + stdHandles[i] = handles[i].getPointer().getLong(0); + } + + return ret; + } + + private static int getExitCodeProcess(WinNT.HANDLE handle) { + IntByReference exitStatus = new IntByReference(); + if (!Kernel32.INSTANCE.GetExitCodeProcess(handle, exitStatus)) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + return exitStatus.getValue(); + } + + private static void terminateProcess(WinNT.HANDLE handle) { + Kernel32.INSTANCE.TerminateProcess(handle, 1); + } + + private static boolean isProcessAlive(WinNT.HANDLE handle) { + IntByReference exitStatus = new IntByReference(); + Kernel32.INSTANCE.GetExitCodeProcess(handle, exitStatus); + return exitStatus.getValue() == STILL_ACTIVE; + } + + private static void closeHandle(WinNT.HANDLE handle) { + Kernel32Util.closeHandle(handle); + } + + /** + * Opens a file for atomic append. The file is created if it doesn't + * already exist. + * + * @param path the file to open or create + * @return the native HANDLE + */ + private static long openForAtomicAppend(String path) throws IOException { + int access = Kernel32.GENERIC_READ | Kernel32.GENERIC_WRITE; + int sharing = Kernel32.FILE_SHARE_READ | Kernel32.FILE_SHARE_WRITE; + int disposition = Kernel32.OPEN_ALWAYS; + int flagsAndAttributes = Kernel32.FILE_ATTRIBUTE_NORMAL; + if (path == null || path.isEmpty()) { + return -1; + } else { + WinNT.HANDLE handle = Kernel32.INSTANCE.CreateFile(path, access, sharing, null, disposition, flagsAndAttributes, null); + if (handle == Kernel32.INVALID_HANDLE_VALUE) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + return handle.getPointer().getLong(0); + } + } + + private static void waitForInterruptibly(WinNT.HANDLE handle) { + int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, Kernel32.INFINITE); + if (result == Kernel32.WAIT_FAILED) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + } + + private static void waitForTimeoutInterruptibly(WinNT.HANDLE handle, long timeout) { + int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, (int) timeout); + if (result == Kernel32.WAIT_FAILED) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java new file mode 100644 index 0000000000..3280a9629f --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.junit.Assert; +import org.junit.Test; + +/** + * Constants Test + */ +public class ConstantsTest { + + /** + * Test PID via env + */ + @Test + public void testPID() { + if (OSUtils.isWindows()) { + Assert.assertEquals(Constants.PID, "handle"); + } else { + Assert.assertEquals(Constants.PID, "pid"); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 7106804aaf..b955787c69 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -39,16 +39,20 @@ public class OSUtilsTest { @Test public void testOSMetric(){ - double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); - Assert.assertTrue(availablePhysicalMemorySize > 0.0f); - double totalMemorySize = OSUtils.totalMemorySize(); - Assert.assertTrue(totalMemorySize > 0.0f); - double loadAverage = OSUtils.loadAverage(); - logger.info("loadAverage {}", loadAverage); - double memoryUsage = OSUtils.memoryUsage(); - Assert.assertTrue(memoryUsage > 0.0f); - double cpuUsage = OSUtils.cpuUsage(); - Assert.assertTrue(cpuUsage > 0.0f); + if (!OSUtils.isWindows()) { + double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); + Assert.assertTrue(availablePhysicalMemorySize > 0.0f); + double totalMemorySize = OSUtils.totalMemorySize(); + Assert.assertTrue(totalMemorySize > 0.0f); + double loadAverage = OSUtils.loadAverage(); + logger.info("loadAverage {}", loadAverage); + double memoryUsage = OSUtils.memoryUsage(); + Assert.assertTrue(memoryUsage > 0.0f); + double cpuUsage = OSUtils.cpuUsage(); + Assert.assertTrue(cpuUsage > 0.0f); + } else { + // TODO window ut + } } @Test diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java new file mode 100644 index 0000000000..ce04346743 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(OSUtils.class) +public class ProcessBuilderForWin32Test { + + private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class); + + @Before + public void before() { + PowerMockito.mockStatic(OSUtils.class); + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + } + + @Test + public void testCreateProcessBuilderForWin32() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + Assert.assertNotNull(builder); + + builder = new ProcessBuilderForWin32("net"); + Assert.assertNotNull(builder); + + builder = new ProcessBuilderForWin32(Collections.singletonList("net")); + Assert.assertNotNull(builder); + + builder = new ProcessBuilderForWin32((List) null); + Assert.assertNotNull(builder); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testBuildUser() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.user("test", StringUtils.EMPTY); + Assert.assertNotNull(builder); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testBuildCommand() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.command(Collections.singletonList("net")); + Assert.assertNotEquals(0, builder.command().size()); + + builder = new ProcessBuilderForWin32(); + builder.command("net"); + Assert.assertNotEquals(0, builder.command().size()); + + builder = new ProcessBuilderForWin32(); + builder.command((List) null); + Assert.assertNotEquals(0, builder.command().size()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testEnvironment() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + Assert.assertNotNull(builder.environment()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.environment(new String[]{ "a=123" }); + Assert.assertNotEquals(0, builder.environment().size()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testDirectory() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.directory(new File("/tmp")); + Assert.assertNotNull(builder.directory()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testStream() { + try { + InputStream in = ProcessBuilderForWin32.NullInputStream.INSTANCE; + Assert.assertNotNull(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(0, in.available()); + + OutputStream out = ProcessBuilderForWin32.NullOutputStream.INSTANCE; + Assert.assertNotNull(out); + out.write(new byte[] {1}); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testRedirect() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + + builder.redirectInput(new File("/tmp")); + Assert.assertNotNull(builder.redirectInput()); + Assert.assertNotNull(builder.redirectInput().file()); + + builder.redirectOutput(new File("/tmp")); + Assert.assertNotNull(builder.redirectOutput()); + Assert.assertNotNull(builder.redirectOutput().file()); + + builder.redirectError(new File("/tmp")); + Assert.assertNotNull(builder.redirectError()); + Assert.assertNotNull(builder.redirectError().file()); + + builder.redirectInput(builder.redirectOutput()); + builder.redirectOutput(builder.redirectInput()); + builder.redirectError(builder.redirectInput()); + + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.PIPE.type()); + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.PIPE.toString()); + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.INHERIT.type()); + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.INHERIT.toString()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testRedirectErrorStream() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.redirectErrorStream(true); + Assert.assertTrue(builder.redirectErrorStream()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void runCmdViaUser() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.user("test123", StringUtils.EMPTY); + + List commands = new ArrayList<>(); + commands.add("cmd.exe"); + commands.add("/c"); + commands.add("net user"); + builder.command(commands); + + Process process = builder.start(); + BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName("GBK"))); + String line; + StringBuilder sb = new StringBuilder(); + while ((line = inReader.readLine()) != null) { + sb.append(line); + } + logger.info("net user: {}", sb.toString()); + Assert.assertNotEquals(StringUtils.EMPTY, sb.toString()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java new file mode 100644 index 0000000000..00c54c0164 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({OSUtils.class, ProcessEnvironmentForWin32.class}) +public class ProcessEnvironmentForWin32Test { + + private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class); + + @Before + public void before() { + try { + PowerMockito.mockStatic(OSUtils.class); + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testPutAndGet() { + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.put("a", "123"); + Assert.assertEquals("123", processEnvironmentForWin32.get("a")); + Assert.assertTrue(processEnvironmentForWin32.containsKey("a")); + Assert.assertTrue(processEnvironmentForWin32.containsValue("123")); + Assert.assertEquals("123", processEnvironmentForWin32.remove("a")); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.put("b=", "123"); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.put("b", "\u0000"); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.get(null); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testEntrySet() { + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.clear(); + processEnvironmentForWin32.put("a", "123"); + Assert.assertEquals(0, processEnvironmentForWin32.entrySet().size()); + Assert.assertTrue(processEnvironmentForWin32.entrySet().isEmpty()); + for (Map.Entry entry : processEnvironmentForWin32.entrySet()) { + Assert.assertNotNull(entry); + Assert.assertNotNull(entry.getKey()); + Assert.assertNotNull(entry.getValue()); + Assert.assertNotNull(entry.setValue("123")); + } + + processEnvironmentForWin32.clear(); + Set keys = processEnvironmentForWin32.keySet(); + Assert.assertEquals(0, keys.size()); + Assert.assertTrue(keys.isEmpty()); + + processEnvironmentForWin32.clear(); + Collection values = processEnvironmentForWin32.values(); + Assert.assertEquals(0, keys.size()); + Assert.assertTrue(keys.isEmpty()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testToEnvironmentBlock() { + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + Assert.assertNotNull(processEnvironmentForWin32.toEnvironmentBlock()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java new file mode 100644 index 0000000000..3f8bcbfb66 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sun.security.action.GetPropertyAction; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({OSUtils.class, GetPropertyAction.class}) +public class ProcessImplForWin32Test { + + private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class); + + @Before + public void before() { + PowerMockito.mockStatic(OSUtils.class); + PowerMockito.mockStatic(GetPropertyAction.class); + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + } + + @Test + public void testStart() { + try { + Process process = ProcessImplForWin32.start( + "test123", StringUtils.EMPTY, new String[]{"net"}, + null, null, null, false); + Assert.assertNotNull(process); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + Process process = ProcessImplForWin32.start( + "test123", StringUtils.EMPTY, new String[]{"net"}, + null, null, new ProcessBuilderForWin32.Redirect[]{ + ProcessBuilderForWin32.Redirect.PIPE, + ProcessBuilderForWin32.Redirect.PIPE, + ProcessBuilderForWin32.Redirect.PIPE + }, false); + Assert.assertNotNull(process); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index bac498c150..8e0ccee16c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -16,21 +16,29 @@ */ package org.apache.dolphinscheduler.server.worker.task; +import com.sun.jna.platform.win32.Kernel32; +import com.sun.jna.platform.win32.WinNT; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.process.ProcessBuilderForWin32; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import java.io.*; import java.lang.reflect.Field; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -194,26 +202,49 @@ public abstract class AbstractCommandExecutor { * @throws IOException IO Exception */ private void buildProcess(String commandFile) throws IOException { + // command list + List command = new ArrayList<>(); + //init process builder - ProcessBuilder processBuilder = new ProcessBuilder(); - // setting up a working directory - processBuilder.directory(new File(taskDir)); - // merge error information to standard output stream - processBuilder.redirectErrorStream(true); - // setting up user to run commands - List command = new LinkedList<>(); - command.add("sudo"); - command.add("-u"); - command.add(tenantCode); - command.add(commandInterpreter()); - command.addAll(commandOptions()); - command.add(commandFile); - processBuilder.command(command); - - process = processBuilder.start(); + if (OSUtils.isWindows()) { + ProcessBuilderForWin32 processBuilder = new ProcessBuilderForWin32(); + // setting up a working directory + processBuilder.directory(new File(taskDir)); + processBuilder.user(tenantCode, StringUtils.EMPTY); + // merge error information to standard output stream + processBuilder.redirectErrorStream(true); + + // setting up user to run commands + command.add(commandInterpreter()); + command.add("/c"); + command.addAll(commandOptions()); + command.add(commandFile); + + // setting commands + processBuilder.command(command); + process = processBuilder.start(); + } else { + ProcessBuilder processBuilder = new ProcessBuilder(); + // setting up a working directory + processBuilder.directory(new File(taskDir)); + // merge error information to standard output stream + processBuilder.redirectErrorStream(true); + + // setting up user to run commands + command.add("sudo"); + command.add("-u"); + command.add(tenantCode); + command.add(commandInterpreter()); + command.addAll(commandOptions()); + command.add(commandFile); + + // setting commands + processBuilder.command(command); + process = processBuilder.start(); + } // print command - printCommand(processBuilder); + printCommand(command); } /** @@ -320,13 +351,13 @@ public abstract class AbstractCommandExecutor { /** * print command - * @param processBuilder process builder + * @param command command */ - private void printCommand(ProcessBuilder processBuilder) { + private void printCommand(List command) { String cmdStr; try { - cmdStr = ProcessUtils.buildCommandStr(processBuilder.command()); + cmdStr = ProcessUtils.buildCommandStr(command); logger.info("task run command:\n{}", cmdStr); } catch (IOException e) { logger.error(e.getMessage(), e); @@ -358,7 +389,11 @@ public abstract class AbstractCommandExecutor { BufferedReader inReader = null; try { - inReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + if (OSUtils.isWindows()) { + inReader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName("GBK"))); + } else { + inReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + } String line; long lastFlushTime = System.currentTimeMillis(); @@ -406,7 +441,7 @@ public abstract class AbstractCommandExecutor { } Thread.sleep(Constants.SLEEP_TIME_MILLIS); } - } + } } catch (Exception e) { logger.error("yarn applications: {} status failed ", appIds,e); result = false; @@ -510,12 +545,15 @@ public abstract class AbstractCommandExecutor { */ private int getProcessId(Process process) { int processId = 0; - try { Field f = process.getClass().getDeclaredField(Constants.PID); f.setAccessible(true); - - processId = f.getInt(process); + if (OSUtils.isWindows()) { + WinNT.HANDLE handle = (WinNT.HANDLE) f.get(process); + processId = Kernel32.INSTANCE.GetProcessId(handle); + } else { + processId = f.getInt(process); + } } catch (Throwable e) { logger.error(e.getMessage(), e); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java index db46d0d856..5d14e6b2a7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.server.worker.task; import org.apache.commons.io.FileUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.slf4j.Logger; import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Date; @@ -34,10 +35,15 @@ import java.util.function.Consumer; public class ShellCommandExecutor extends AbstractCommandExecutor { /** - * sh + * For Unix-like, using sh */ public static final String SH = "sh"; + /** + * For Windows, using cmd.exe + */ + public static final String CMD = "cmd.exe"; + /** * constructor * @param logHandler log handler @@ -66,7 +72,7 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { @Override protected String buildCommandFilePath() { // command file - return String.format("%s/%s.command", taskDir, taskAppId); + return String.format("%s/%s.%s", taskDir, taskAppId, OSUtils.isWindows() ? "bat" : "command"); } /** @@ -75,7 +81,7 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { */ @Override protected String commandInterpreter() { - return SH; + return OSUtils.isWindows() ? CMD : SH; } /** @@ -103,21 +109,26 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { logger.info("create command file:{}", commandFile); StringBuilder sb = new StringBuilder(); - sb.append("#!/bin/sh\n"); - sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); - sb.append("cd $BASEDIR\n"); - - if (envFile != null) { - sb.append("source " + envFile + "\n"); + if (OSUtils.isWindows()) { + sb.append("@echo off\n"); + sb.append("cd /d %~dp0\n"); + if (envFile != null) { + sb.append("call ").append(envFile).append("\n"); + } + } else { + sb.append("#!/bin/sh\n"); + sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); + sb.append("cd $BASEDIR\n"); + if (envFile != null) { + sb.append("source ").append(envFile).append("\n"); + } } - sb.append("\n\n"); sb.append(execCommand); - logger.info("command : {}",sb.toString()); + logger.info("command : {}", sb.toString()); // write data to file - FileUtils.writeStringToFile(new File(commandFile), sb.toString(), - Charset.forName("UTF-8")); + FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index ef941cd062..7537ca2edc 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -44,6 +44,7 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; @@ -338,7 +339,7 @@ public class DataxTask extends AbstractTask { private String buildShellCommandFile(String jobConfigFilePath) throws Exception { // generate scripts - String fileName = String.format("%s/%s_node.sh", taskDir, taskProps.getTaskAppId()); + String fileName = String.format("%s/%s_node.%s", taskDir, taskProps.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh"); Path path = new File(fileName).toPath(); if (Files.exists(path)) { @@ -370,7 +371,13 @@ public class DataxTask extends AbstractTask { // create shell command file Set perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X); FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - Files.createFile(path, attr); + + if (OSUtils.isWindows()) { + Files.createFile(path); + } else { + Files.createFile(path, attr); + } + Files.write(path, dataxCommand.getBytes(), StandardOpenOption.APPEND); return fileName; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index 5704c8052e..90661a690a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; @@ -123,7 +124,7 @@ public class ShellTask extends AbstractTask { */ private String buildCommand() throws Exception { // generate scripts - String fileName = String.format("%s/%s_node.sh", taskDir, taskProps.getTaskAppId()); + String fileName = String.format("%s/%s_node.%s", taskDir, taskProps.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh"); Path path = new File(fileName).toPath(); if (Files.exists(path)) { @@ -154,7 +155,11 @@ public class ShellTask extends AbstractTask { Set perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X); FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - Files.createFile(path, attr); + if (OSUtils.isWindows()) { + Files.createFile(path); + } else { + Files.createFile(path, attr); + } Files.write(path, shellParameters.getRawScript().getBytes(), StandardOpenOption.APPEND); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java index 77fc398702..1e0adaad9b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java @@ -21,7 +21,12 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + public class ProcessUtilsTest { + private static final Logger logger = LoggerFactory.getLogger(ProcessUtilsTest.class); @Test @@ -30,4 +35,16 @@ public class ProcessUtilsTest { Assert.assertNotEquals("The child process of process 1 should not be empty", pidList, ""); logger.info("Sub process list : {}", pidList); } + + @Test + public void testBuildCommandStr() { + List commands = new ArrayList<>(); + commands.add("sudo"); + try { + Assert.assertEquals(ProcessUtils.buildCommandStr(commands), "sudo"); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + } + } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java new file mode 100644 index 0000000000..5536665e26 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.worker.task.shell; + +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; +import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; + +import java.util.Date; + +/** + * shell task test + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest(OSUtils.class) +public class ShellTaskTest { + + private static final Logger logger = LoggerFactory.getLogger(ShellTaskTest.class); + + private ShellTask shellTask; + + private ProcessService processService; + + private ShellCommandExecutor shellCommandExecutor; + + private ApplicationContext applicationContext; + + @Before + public void before() throws Exception { + PowerMockito.mockStatic(OSUtils.class); + processService = PowerMockito.mock(ProcessService.class); + shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class); + + applicationContext = PowerMockito.mock(ApplicationContext.class); + SpringApplicationContext springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + PowerMockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + + TaskProps props = new TaskProps(); + props.setTaskDir("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstId(1); + props.setTenantCode("1"); + props.setEnvFile(".dolphinscheduler_env.sh"); + props.setTaskStartTime(new Date()); + props.setTaskTimeout(0); + props.setTaskParams("{\"rawScript\": \" echo 'hello world!'\"}"); + shellTask = new ShellTask(props, logger); + shellTask.init(); + + PowerMockito.when(processService.findDataSourceById(1)).thenReturn(getDataSource()); + PowerMockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource()); + PowerMockito.when(processService.findProcessInstanceByTaskId(1)).thenReturn(getProcessInstance()); + + String fileName = String.format("%s/%s_node.%s", props.getTaskDir(), props.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh"); + PowerMockito.when(shellCommandExecutor.run(fileName, processService)).thenReturn(0); + } + + private DataSource getDataSource() { + DataSource dataSource = new DataSource(); + dataSource.setType(DbType.MYSQL); + dataSource.setConnectionParams( + "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://127.0.0.1:3306\",\"database\":\"test\",\"jdbcUrl\":\"jdbc:mysql://127.0.0.1:3306/test\"}"); + dataSource.setUserId(1); + return dataSource; + } + + private ProcessInstance getProcessInstance() { + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setCommandType(CommandType.START_PROCESS); + processInstance.setScheduleTime(new Date()); + return processInstance; + } + + @After + public void after() {} + + /** + * Method: ShellTask() + */ + @Test + public void testShellTask() + throws Exception { + TaskProps props = new TaskProps(); + props.setTaskDir("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstId(1); + props.setTenantCode("1"); + ShellTask shellTaskTest = new ShellTask(props, logger); + Assert.assertNotNull(shellTaskTest); + } + + /** + * Method: init for Unix-like + */ + @Test + public void testInitForUnix() { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(false); + shellTask.init(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + /** + * Method: init for Windows + */ + @Test + public void testInitForWindows() { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + shellTask.init(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + /** + * Method: handle() for Unix-like + */ + @Test + public void testHandleForUnix() throws Exception { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(false); + shellTask.handle(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + if (!e.getMessage().contains("process error . exitCode is : -1") + && !System.getProperty("os.name").startsWith("Windows")) { + logger.error(e.getMessage()); + } + } + } + + /** + * Method: handle() for Windows + */ + @Test + public void testHandleForWindows() throws Exception { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + shellTask.handle(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + if (!e.getMessage().contains("process error . exitCode is : -1")) { + logger.error(e.getMessage()); + } + } + } + + /** + * Method: cancelApplication() + */ + @Test + public void testCancelApplication() throws Exception { + try { + shellTask.cancelApplication(true); + Assert.assertTrue(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/pom.xml b/pom.xml index dee1dce8b2..a7feec0e14 100644 --- a/pom.xml +++ b/pom.xml @@ -684,6 +684,9 @@ **/common/utils/*.java + **/common/utils/process/ProcessBuilderForWin32Test.java + **/common/utils/process/ProcessEnvironmentForWin32Test.java + **/common/utils/process/ProcessImplForWin32Test.java **/common/log/*.java **/common/threadutils/*.java **/common/graph/*.java @@ -732,6 +735,7 @@ **/alert/template/AlertTemplateFactoryTest.java **/alert/template/impl/DefaultHTMLTemplateTest.java **/server/worker/task/datax/DataxTaskTest.java + **/server/worker/task/shell/ShellTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/utils/DataxUtilsTest.java **/service/zk/DefaultEnsembleProviderTest.java From 4bb97dfb8b1c4aaee8e61a8d12793b1c337ee5f1 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 23:53:12 +0800 Subject: [PATCH 17/37] fix actions/checkout@v2 problem (#2095) --- .github/workflows/ci_backend.yml | 12 ++++++++++++ .github/workflows/ci_e2e.yml | 6 ++++++ .github/workflows/ci_frontend.yml | 12 ++++++++++++ .github/workflows/ci_ut.yml | 6 ++++++ 4 files changed, 36 insertions(+) diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index 1ca15c2813..0273251e99 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -46,6 +46,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: @@ -56,6 +62,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 7d9c5e3e72..fe818d0668 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -30,6 +30,12 @@ jobs: steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - uses: actions/cache@v1 with: path: ~/.m2/repository diff --git a/.github/workflows/ci_frontend.yml b/.github/workflows/ci_frontend.yml index 1f959b40eb..494d12dbae 100644 --- a/.github/workflows/ci_frontend.yml +++ b/.github/workflows/ci_frontend.yml @@ -35,6 +35,12 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up Node.js uses: actions/setup-node@v1 with: @@ -50,6 +56,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index e7e47437d4..48cd1e16ce 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -30,6 +30,12 @@ jobs: steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - uses: actions/cache@v1 with: path: ~/.m2/repository From 0e1dd8d7d95b945c8982f18b802c88b216cf0179 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sat, 7 Mar 2020 18:07:53 +0800 Subject: [PATCH 18/37] remove incorrect or unnecessary URL tag (#2099) --- dolphinscheduler-common/pom.xml | 2 +- dolphinscheduler-dao/pom.xml | 2 +- dolphinscheduler-remote/pom.xml | 2 -- dolphinscheduler-server/pom.xml | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index e7789f724b..ca75a84a62 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-common dolphinscheduler-common - http://maven.apache.org + jar UTF-8 diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index fd43458f87..3aea888f94 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-dao ${project.artifactId} - http://maven.apache.org + UTF-8 diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index b67b033ffa..39c7c6a7c0 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -12,8 +12,6 @@ dolphinscheduler-remote dolphinscheduler-remote - - http://www.example.com UTF-8 diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 86490197b6..e8e84297e2 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-server dolphinscheduler-server - http://maven.apache.org + jar UTF-8 From 89b42db3b4da98295c898d4243577ef1e0a0291d Mon Sep 17 00:00:00 2001 From: liwenhe1993 <32166572+liwenhe1993@users.noreply.github.com> Date: Sun, 8 Mar 2020 15:56:24 +0800 Subject: [PATCH 19/37] Add docker internal server check script (#2092) * add environment variables add checkpoint.sh file to docker image 1. add `POSTGRESQL_DATABASE` environment variable 2. add `DOLPHINSCHEDULER_DATA_BASEDIR_PATH` environment variable 3. add `DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH` environment variable 4. add `DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH` environment variable 5. add checkpoint.sh file to check server process 6. modify `README.md` and `README_zh_CN.md` * add environment variables add checkpoint.sh file to docker image 1. add `POSTGRESQL_DATABASE` environment variable 2. add `DOLPHINSCHEDULER_DATA_BASEDIR_PATH` environment variable 3. add `DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH` environment variable 4. add `DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH` environment variable 5. add checkpoint.sh file to check server process 6. modify `README.md` and `README_zh_CN.md` * add logger port and modify dolphinscheduler_env to dolphinscheduler_env.sh --- dockerfile/Dockerfile | 13 +++--- dockerfile/README.md | 40 ++++++++++++++----- dockerfile/README_zh_CN.md | 38 ++++++++++++++---- dockerfile/checkpoint.sh | 27 +++++++++++++ .../application.properties.tpl | 2 +- .../dolphinscheduler/common.properties.tpl | 12 +++--- ...nscheduler_env => dolphinscheduler_env.sh} | 0 .../dolphinscheduler/quartz.properties.tpl | 2 +- dockerfile/startup-init-conf.sh | 4 ++ 9 files changed, 108 insertions(+), 30 deletions(-) create mode 100644 dockerfile/checkpoint.sh rename dockerfile/conf/dolphinscheduler/env/{dolphinscheduler_env => dolphinscheduler_env.sh} (100%) diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index 1fc064c489..8fa4886f31 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -63,19 +63,22 @@ RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d #7. add configuration and modify permissions and set soft links +ADD ./checkpoint.sh /root/checkpoint.sh ADD ./startup-init-conf.sh /root/startup-init-conf.sh ADD ./startup.sh /root/startup.sh ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ -ADD ./conf/dolphinscheduler/env/dolphinscheduler_env /opt/dolphinscheduler/conf/env/ -RUN chmod +x /root/startup-init-conf.sh && \ +ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ +RUN chmod +x /root/checkpoint.sh && \ + chmod +x /root/startup-init-conf.sh && \ chmod +x /root/startup.sh && \ - chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \ + chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ chmod +x /opt/dolphinscheduler/script/*.sh && \ chmod +x /opt/dolphinscheduler/bin/*.sh && \ chmod +x /opt/zookeeper/bin/*.sh && \ + dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ - dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \ + dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ dos2unix /opt/zookeeper/bin/*.sh && \ @@ -87,6 +90,6 @@ RUN chmod +x /root/startup-init-conf.sh && \ RUN rm -rf /var/cache/apk/* #9. expose port -EXPOSE 2181 2888 3888 5432 12345 8888 +EXPOSE 2181 2888 3888 5432 12345 50051 8888 ENTRYPOINT ["/root/startup.sh"] \ No newline at end of file diff --git a/dockerfile/README.md b/dockerfile/README.md index b5a9d0d3aa..60af7fad8f 100644 --- a/dockerfile/README.md +++ b/dockerfile/README.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### You can start a dolphinscheduler instance ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ +-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -25,13 +25,13 @@ The default postgres user `root`, postgres password `root` and database `dolphin The default zookeeper is created in the `startup.sh`. -#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** +#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`** You can specify **existing postgres service**. Example: ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all @@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example: ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,7 +56,7 @@ You can start a standalone dolphinscheduler server. ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,7 +66,7 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,7 +75,7 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server @@ -85,7 +85,7 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. +**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. ## How to build a docker image @@ -140,14 +140,36 @@ This environment variable sets the port for PostgreSQL. The default value is `54 This environment variable sets the username for PostgreSQL. The default value is `root`. +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + **`POSTGRESQL_PASSWORD`** This environment variable sets the password for PostgreSQL. The default value is `root`. +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + +**`POSTGRESQL_DATABASE`** + +This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`. + +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + **`DOLPHINSCHEDULER_ENV_PATH`** This environment variable sets the runtime environment for task. The default value is `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`. +**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** + +User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler` + +**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`** + +Directory path for user data download. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/download` + +**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`** + +Process execute directory. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/exec` + **`TASK_QUEUE`** This environment variable sets the task queue for `master-server` and `worker-serverr`. The default value is `zookeeper`. diff --git a/dockerfile/README_zh_CN.md b/dockerfile/README_zh_CN.md index 5830e30b07..900c8b50d9 100644 --- a/dockerfile/README_zh_CN.md +++ b/dockerfile/README_zh_CN.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### 你可以运行一个dolphinscheduler实例 ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ +-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -31,7 +31,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all @@ -42,7 +42,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,7 +56,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,7 +66,7 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,7 +75,7 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server @@ -85,7 +85,7 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM`。 +**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`。 ## 如何构建一个docker镜像 @@ -140,14 +140,36 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 配置`PostgreSQL`的`USERNAME`, 默认值 `root`。 +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。 + **`POSTGRESQL_PASSWORD`** 配置`PostgreSQL`的`PASSWORD`, 默认值 `root`。 +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。 + +**`POSTGRESQL_DATABASE`** + +配置`PostgreSQL`的`DATABASE`, 默认值 `dolphinscheduler`。 + +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。 + **`DOLPHINSCHEDULER_ENV_PATH`** 任务执行时的环境变量配置文件, 默认值 `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`。 +**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** + +用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler`。 + +**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`** + +用户数据下载目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler/download`。 + +**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`** + +任务执行目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler/exec`。 + **`TASK_QUEUE`** 配置`master-server`和`worker-serverr`的`Zookeeper`任务队列名, 默认值 `zookeeper`。 diff --git a/dockerfile/checkpoint.sh b/dockerfile/checkpoint.sh new file mode 100644 index 0000000000..cd2774f9ce --- /dev/null +++ b/dockerfile/checkpoint.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +if [ "$(ps -ef | grep java | grep -c $1)" -eq 0 ]; then + echo "[ERROR] $1 process not exits." + exit 1 +else + echo "[INFO] $1 process exits." + exit 0 +fi diff --git a/dockerfile/conf/dolphinscheduler/application.properties.tpl b/dockerfile/conf/dolphinscheduler/application.properties.tpl index c643c414cd..6dd8a18e11 100644 --- a/dockerfile/conf/dolphinscheduler/application.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/application.properties.tpl @@ -19,7 +19,7 @@ spring.datasource.type=com.alibaba.druid.pool.DruidDataSource # postgre spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8 +spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 diff --git a/dockerfile/conf/dolphinscheduler/common.properties.tpl b/dockerfile/conf/dolphinscheduler/common.properties.tpl index ea03e0b78e..8134fc7a9b 100644 --- a/dockerfile/conf/dolphinscheduler/common.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/common.properties.tpl @@ -38,6 +38,12 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties # is development state? default "false" development.state=true +# user data directory path, self configuration, please make sure the directory exists and have read write permissions +data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} +# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions +data.download.basedir.path=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH} +# process execute directory. self configuration, please make sure the directory exists and have read write permissions +process.exec.basepath=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH} # resource upload startup type : HDFS,S3,NONE res.upload.startup.type=NONE @@ -49,12 +55,6 @@ res.upload.startup.type=NONE hdfs.root.user=hdfs # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended data.store2hdfs.basepath=/dolphinscheduler -# user data directory path, self configuration, please make sure the directory exists and have read write permissions -data.basedir.path=/tmp/dolphinscheduler -# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions -data.download.basedir.path=/tmp/dolphinscheduler/download -# process execute directory. self configuration, please make sure the directory exists and have read write permissions -process.exec.basepath=/tmp/dolphinscheduler/exec # whether kerberos starts hadoop.security.authentication.startup.state=false # java.security.krb5.conf path diff --git a/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env b/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh similarity index 100% rename from dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env rename to dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh diff --git a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl index de5496bc3d..7c7c92e425 100644 --- a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl @@ -22,7 +22,7 @@ org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate # postgre org.quartz.dataSource.myDs.driver = org.postgresql.Driver -org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8 +org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 org.quartz.dataSource.myDs.user = ${POSTGRESQL_USERNAME} org.quartz.dataSource.myDs.password = ${POSTGRESQL_PASSWORD} org.quartz.scheduler.instanceName = DolphinScheduler diff --git a/dockerfile/startup-init-conf.sh b/dockerfile/startup-init-conf.sh index db37976168..d2b50fa3e7 100644 --- a/dockerfile/startup-init-conf.sh +++ b/dockerfile/startup-init-conf.sh @@ -28,11 +28,15 @@ export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"} export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"} export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"} export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"} +export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"} #============================================================================ # System #============================================================================ export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} +export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} +export DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH:-"/tmp/dolphinscheduler/download"} +export DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH:-"/tmp/dolphinscheduler/exec"} #============================================================================ # Zookeeper From b2268b1af9bdab3162a4c6378c85a936ce00827f Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Sun, 8 Mar 2020 15:57:06 +0800 Subject: [PATCH 20/37] use stringbuilder to append string (#2108) --- .../service/zk/AbstractZKClient.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java index 135bfdabc6..fa1a0bfced 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java @@ -65,14 +65,16 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator { if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ return; } - String str = splits[0] + Constants.COMMA - + splits[1] + Constants.COMMA - + OSUtils.cpuUsage() + Constants.COMMA - + OSUtils.memoryUsage() + Constants.COMMA - + OSUtils.loadAverage() + Constants.COMMA - + splits[5] + Constants.COMMA - + DateUtils.dateToString(new Date()); - zkClient.setData().forPath(znode,str.getBytes()); + StringBuilder sb = new StringBuilder(); + sb.append(splits[0]).append(Constants.COMMA) + .append(splits[1]).append(Constants.COMMA) + .append(OSUtils.cpuUsage()).append(Constants.COMMA) + .append(OSUtils.memoryUsage()).append(Constants.COMMA) + .append(OSUtils.loadAverage()).append(Constants.COMMA) + .append(splits[5]).append(Constants.COMMA) + .append(DateUtils.dateToString(new Date())); + + zkClient.setData().forPath(znode, sb.toString().getBytes()); } catch (Exception e) { logger.error("heartbeat for zk failed", e); From 275df85250683b22691c658591ed2094d8a32ac3 Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Sun, 8 Mar 2020 16:00:47 +0800 Subject: [PATCH 21/37] fix: remove redundant import class (#2110) --- .../org/apache/dolphinscheduler/remote/command/Command.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java index 86ba79c884..ed46e1ff51 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java @@ -16,8 +16,6 @@ */ package org.apache.dolphinscheduler.remote.command; -import com.sun.org.apache.regexp.internal.RE; - import java.io.Serializable; import java.util.concurrent.atomic.AtomicLong; From ad381fcb716632deb966fd8ca0627ff8c17d5641 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sun, 8 Mar 2020 21:22:07 +0800 Subject: [PATCH 22/37] move UT class to reasonable package (#2116) --- .../dolphinscheduler/common}/utils/PreconditionsTest.java | 3 +-- .../dolphinscheduler/service/quartz}/cron/CronUtilsTest.java | 3 +-- .../dolphinscheduler/service}/queue/BaseTaskQueueTest.java | 2 +- .../dolphinscheduler/service}/queue/TaskQueueZKImplTest.java | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) rename {dolphinscheduler-service/src/test/java => dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common}/utils/PreconditionsTest.java (98%) rename dolphinscheduler-service/src/test/java/{ => org/apache/dolphinscheduler/service/quartz}/cron/CronUtilsTest.java (99%) rename dolphinscheduler-service/src/test/java/{ => org/apache/dolphinscheduler/service}/queue/BaseTaskQueueTest.java (96%) rename dolphinscheduler-service/src/test/java/{ => org/apache/dolphinscheduler/service}/queue/TaskQueueZKImplTest.java (99%) diff --git a/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java similarity index 98% rename from dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java rename to dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java index a1b85f1b12..47b24bb93c 100644 --- a/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package utils; +package org.apache.dolphinscheduler.common.utils; -import org.apache.dolphinscheduler.common.utils.Preconditions; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java similarity index 99% rename from dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java index 6a402b5e67..b4f864c5b4 100644 --- a/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package cron; +package org.apache.dolphinscheduler.service.quartz.cron; import com.cronutils.builder.CronBuilder; import com.cronutils.model.Cron; @@ -25,7 +25,6 @@ import com.cronutils.model.field.CronFieldName; import com.cronutils.model.field.expression.*; import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java similarity index 96% rename from dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java index 97ab9969a3..17e2ae4056 100644 --- a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; diff --git a/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java similarity index 99% rename from dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java index d29c5aa610..5d464ac3c9 100644 --- a/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.IpUtils; From 450a1f56fc73f088fce89a343a0b008706f2088c Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sun, 8 Mar 2020 21:28:49 +0800 Subject: [PATCH 23/37] Adapting partial code(file name start with E) to the sonar cloud rule (#1999) * Adapting partial code(file name start with E) to the sonar cloud rule * remove isEmpty invoke * resolve conflicts --- .../manager/EnterpriseWeChatManager.java | 4 +- .../alert/utils/EnterpriseWeChatUtils.java | 38 +++++++++---------- .../utils/EnterpriseWeChatUtilsTest.java | 8 ++-- .../api/controller/ExecutorController.java | 2 +- .../api/service/ExecutorService.java | 15 +++----- .../dao/mapper/ErrorCommandMapperTest.java | 4 +- 6 files changed, 34 insertions(+), 37 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java index 9bcad56c24..bb06be6561 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java @@ -42,8 +42,8 @@ public class EnterpriseWeChatManager { public Map send(Alert alert, String token){ Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); - String agentId = EnterpriseWeChatUtils.enterpriseWeChatAgentId; - String users = EnterpriseWeChatUtils.enterpriseWeChatUsers; + String agentId = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_AGENT_ID; + String users = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_USERS; List userList = Arrays.asList(users.split(",")); logger.info("send message {}",alert); String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alert)); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index ff8822421a..900c120cd4 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -43,24 +43,24 @@ public class EnterpriseWeChatUtils { public static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatUtils.class); - private static final String enterpriseWeChatCorpId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID); + private static final String ENTERPRISE_WE_CHAT_CORP_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID); - private static final String enterpriseWeChatSecret = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); + private static final String ENTERPRISE_WE_CHAT_SECRET = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); - private static final String enterpriseWeChatTokenUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); - private static String enterpriseWeChatTokenUrlReplace = enterpriseWeChatTokenUrl - .replaceAll("\\$corpId", enterpriseWeChatCorpId) - .replaceAll("\\$secret", enterpriseWeChatSecret); + private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); + private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL + .replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID) + .replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET); - private static final String enterpriseWeChatPushUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); + private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); - private static final String enterpriseWeChatTeamSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); + private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); - private static final String enterpriseWeChatUserSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); + private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); - public static final String enterpriseWeChatAgentId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); + public static final String ENTERPRISE_WE_CHAT_AGENT_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); - public static final String enterpriseWeChatUsers = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); + public static final String ENTERPRISE_WE_CHAT_USERS = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); /** * get Enterprise WeChat is enable @@ -87,7 +87,7 @@ public class EnterpriseWeChatUtils { CloseableHttpClient httpClient = HttpClients.createDefault(); try { - HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace); + HttpGet httpGet = new HttpGet(ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE); CloseableHttpResponse response = httpClient.execute(httpGet); try { HttpEntity entity = response.getEntity(); @@ -114,7 +114,7 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(String toParty, String agentId, String msg) { - return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", toParty) + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -128,7 +128,7 @@ public class EnterpriseWeChatUtils { */ public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { String listParty = FuncUtils.mkString(toParty, "|"); - return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", listParty) + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -141,7 +141,7 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeUserSendMsg(String toUser, String agentId, String msg) { - return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", toUser) + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -155,7 +155,7 @@ public class EnterpriseWeChatUtils { */ public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { String listUser = FuncUtils.mkString(toUser, "|"); - return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", listUser) + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -169,7 +169,7 @@ public class EnterpriseWeChatUtils { * @throws IOException the IOException */ public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { - String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token); + String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token); CloseableHttpClient httpClient = HttpClients.createDefault(); try { @@ -184,8 +184,8 @@ public class EnterpriseWeChatUtils { } finally { response.close(); } - logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", - enterpriseWeChatPushUrl, data, resp); + logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", + ENTERPRISE_WE_CHAT_PUSH_URL, data, resp); return resp; } finally { httpClient.close(); diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 3471f6efdd..15b92a622e 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -53,7 +53,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -68,7 +68,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -95,7 +95,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -110,7 +110,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index cae4993942..ffedd5703c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -149,7 +149,7 @@ public class ExecutorController extends BaseController { ) { try { logger.info("execute command, login user: {}, project:{}, process instance id:{}, execute type:{}", - loginUser.getUserName(), projectName, processInstanceId, executeType.toString()); + loginUser.getUserName(), projectName, processInstanceId, executeType); Map result = execService.execute(loginUser, projectName, processInstanceId, executeType); return returnDataList(result); } catch (Exception e) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 152292a21b..86b507f0a0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -259,10 +259,7 @@ public class ExecutorService extends BaseService{ // checkTenantExists(); Tenant tenant = processService.getTenantForProcess(processDefinition.getTenantId(), processDefinition.getUserId()); - if(tenant == null){ - return false; - } - return true; + return tenant != null; } /** @@ -298,6 +295,7 @@ public class ExecutorService extends BaseService{ if (executionStatus.typeIsPause()|| executionStatus.typeIsCancel()) { checkResult = true; } + break; default: break; } @@ -369,7 +367,7 @@ public class ExecutorService extends BaseService{ * @return check result code */ public Map startCheckByProcessDefinedId(int processDefineId) { - Map result = new HashMap(); + Map result = new HashMap<>(); if (processDefineId == 0){ logger.error("process definition id is null"); @@ -378,10 +376,9 @@ public class ExecutorService extends BaseService{ List ids = new ArrayList<>(); processService.recurseFindSubProcessId(processDefineId, ids); Integer[] idArray = ids.toArray(new Integer[ids.size()]); - if (ids.size() > 0){ - List processDefinitionList; - processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray); - if (processDefinitionList != null && processDefinitionList.size() > 0){ + if (!ids.isEmpty()){ + List processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray); + if (processDefinitionList != null){ for (ProcessDefinition processDefinition : processDefinitionList){ /** * if there is no online process, exit directly diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java index 3653e6a53d..5fb7dfc09f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java @@ -67,7 +67,7 @@ public class ErrorCommandMapperTest { //update errorCommand.setUpdateTime(new Date()); int update = errorCommandMapper.updateById(errorCommand); - Assert.assertEquals(update, 1); + Assert.assertEquals(1,update); errorCommandMapper.deleteById(errorCommand.getId()); } @@ -79,7 +79,7 @@ public class ErrorCommandMapperTest { ErrorCommand errorCommand = insertOne(); int delete = errorCommandMapper.deleteById(errorCommand.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1,delete); } /** From 9224b49b58b756d22c75d8929108f716283282b4 Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 9 Mar 2020 19:06:41 +0800 Subject: [PATCH 24/37] access field handle of FileDescriptor in ProcessImplForWin32 by reflection for portability (#2113) * access field handle of FileDescriptor in ProcessImplForWin32 by reflection for portability Current implementation relies on `sun.misc.JavaIOFileDescriptorAccess` which is only accessible on oraclejdk8. Basically the demand is getting & setting `handle` field of `FileDescriptor`, so we can directly do that with reflection. Though, I suspect the necessity we introduce ProcessImplForWin32. Maybe we could have a better way to support worker server to run bat script. * harden initialization of ProcessImplForWin32 * ignore ShellTaskTest#testHandleForWindows outside Windows --- .../common/utils/OSUtils.java | 16 ++++-- .../utils/process/ProcessImplForWin32.java | 57 +++++++++++++++---- .../worker/task/shell/ShellTaskTest.java | 3 +- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index acfca77c8c..b011c0bc4e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -400,8 +400,7 @@ public class OSUtils { * @return true if mac */ public static boolean isMacOS() { - String os = System.getProperty("os.name"); - return os.startsWith("Mac"); + return getOSName().startsWith("Mac"); } @@ -409,9 +408,16 @@ public class OSUtils { * whether is windows * @return true if windows */ - public static boolean isWindows() { - String os = System.getProperty("os.name"); - return os.startsWith("Windows"); + public static boolean isWindows() { ; + return getOSName().startsWith("Windows"); + } + + /** + * get current OS name + * @return current OS name + */ + public static String getOSName() { + return System.getProperty("os.name"); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java index 4583be8aff..9f2716a096 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java @@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.common.utils.process; import com.sun.jna.Pointer; import com.sun.jna.platform.win32.*; import com.sun.jna.ptr.IntByReference; +import java.lang.reflect.Field; +import org.apache.dolphinscheduler.common.utils.OSUtils; import sun.security.action.GetPropertyAction; import java.io.*; @@ -31,10 +33,25 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE; +import static java.util.Objects.requireNonNull; public class ProcessImplForWin32 extends Process { - private static final sun.misc.JavaIOFileDescriptorAccess fdAccess - = sun.misc.SharedSecrets.getJavaIOFileDescriptorAccess(); + + private static final Field FD_HANDLE; + + static { + if (!OSUtils.isWindows()) { + throw new RuntimeException("ProcessImplForWin32 can be only initialized in " + + "Windows environment, but current OS is " + OSUtils.getOSName()); + } + + try { + FD_HANDLE = requireNonNull(FileDescriptor.class.getDeclaredField("handle")); + FD_HANDLE.setAccessible(true); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } private static final int PIPE_SIZE = 4096 + 24; @@ -46,6 +63,22 @@ public class ProcessImplForWin32 extends Process { private static final WinNT.HANDLE JAVA_INVALID_HANDLE_VALUE = new WinNT.HANDLE(Pointer.createConstant(-1)); + private static void setHandle(FileDescriptor obj, long handle) { + try { + FD_HANDLE.set(obj, handle); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private static long getHandle(FileDescriptor obj) { + try { + return (Long) FD_HANDLE.get(obj); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + /** * Open a file for writing. If {@code append} is {@code true} then the file * is opened for atomic append directly and a FileOutputStream constructed @@ -63,7 +96,7 @@ public class ProcessImplForWin32 extends Process { sm.checkWrite(path); long handle = openForAtomicAppend(path); final FileDescriptor fd = new FileDescriptor(); - fdAccess.setHandle(fd, handle); + setHandle(fd, handle); return AccessController.doPrivileged( new PrivilegedAction() { public FileOutputStream run() { @@ -102,30 +135,30 @@ public class ProcessImplForWin32 extends Process { if (redirects[0] == ProcessBuilderForWin32.Redirect.PIPE) stdHandles[0] = -1L; else if (redirects[0] == ProcessBuilderForWin32.Redirect.INHERIT) - stdHandles[0] = fdAccess.getHandle(FileDescriptor.in); + stdHandles[0] = getHandle(FileDescriptor.in); else { f0 = new FileInputStream(redirects[0].file()); - stdHandles[0] = fdAccess.getHandle(f0.getFD()); + stdHandles[0] = getHandle(f0.getFD()); } if (redirects[1] == ProcessBuilderForWin32.Redirect.PIPE) stdHandles[1] = -1L; else if (redirects[1] == ProcessBuilderForWin32.Redirect.INHERIT) - stdHandles[1] = fdAccess.getHandle(FileDescriptor.out); + stdHandles[1] = getHandle(FileDescriptor.out); else { f1 = newFileOutputStream(redirects[1].file(), redirects[1].append()); - stdHandles[1] = fdAccess.getHandle(f1.getFD()); + stdHandles[1] = getHandle(f1.getFD()); } if (redirects[2] == ProcessBuilderForWin32.Redirect.PIPE) stdHandles[2] = -1L; else if (redirects[2] == ProcessBuilderForWin32.Redirect.INHERIT) - stdHandles[2] = fdAccess.getHandle(FileDescriptor.err); + stdHandles[2] = getHandle(FileDescriptor.err); else { f2 = newFileOutputStream(redirects[2].file(), redirects[2].append()); - stdHandles[2] = fdAccess.getHandle(f2.getFD()); + stdHandles[2] = getHandle(f2.getFD()); } } @@ -442,7 +475,7 @@ public class ProcessImplForWin32 extends Process { stdin_stream = ProcessBuilderForWin32.NullOutputStream.INSTANCE; else { FileDescriptor stdin_fd = new FileDescriptor(); - fdAccess.setHandle(stdin_fd, stdHandles[0]); + setHandle(stdin_fd, stdHandles[0]); stdin_stream = new BufferedOutputStream( new FileOutputStream(stdin_fd)); } @@ -451,7 +484,7 @@ public class ProcessImplForWin32 extends Process { stdout_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; else { FileDescriptor stdout_fd = new FileDescriptor(); - fdAccess.setHandle(stdout_fd, stdHandles[1]); + setHandle(stdout_fd, stdHandles[1]); stdout_stream = new BufferedInputStream( new FileInputStream(stdout_fd)); } @@ -460,7 +493,7 @@ public class ProcessImplForWin32 extends Process { stderr_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; else { FileDescriptor stderr_fd = new FileDescriptor(); - fdAccess.setHandle(stderr_fd, stdHandles[2]); + setHandle(stderr_fd, stdHandles[2]); stderr_stream = new FileInputStream(stderr_fd); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java index 5536665e26..ebe90147d1 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.After; import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -172,7 +173,7 @@ public class ShellTaskTest { @Test public void testHandleForWindows() throws Exception { try { - PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + Assume.assumeTrue(OSUtils.isWindows()); shellTask.handle(); Assert.assertTrue(true); } catch (Error | Exception e) { From ebf7a980d2922a5659d8f0da28f519556aa3a2f0 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Tue, 10 Mar 2020 18:59:45 +0800 Subject: [PATCH 25/37] Adapting partial code(file name start with F) to the sonar cloud rule (#2045) * Adapting partial code(file name start with F) to the sonar cloud rule * add more unit test * add License * add includes configuration to maven-surefire-plugin * fix getResourceFilesList incorrect logic --- .../alert/utils/FuncUtils.java | 2 +- .../alert/utils/FuncUtilsTest.java | 2 +- .../common/task/flink/FlinkParameters.java | 14 +++-- .../common/utils/FileUtils.java | 6 +- .../common/task/FlinkParametersTest.java | 55 +++++++++++++++++++ .../common/utils/FileUtilsTest.java | 17 +++--- .../server/utils/FlinkArgsUtils.java | 16 ++---- .../worker/runner/TaskScheduleThread.java | 4 +- .../server/utils/FlinkArgsUtilsTest.java | 24 ++++---- pom.xml | 1 + 10 files changed, 97 insertions(+), 44 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java index dd6ca4b8a6..d68532a82b 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java @@ -20,7 +20,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils; public class FuncUtils { - static public String mkString(Iterable list, String split) { + public static String mkString(Iterable list, String split) { if (null == list || StringUtils.isEmpty(split)){ return null; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java index e2b25d961d..a4aeea9c0c 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java @@ -46,7 +46,7 @@ public class FuncUtilsTest { logger.info(result); //Expected result string - assertEquals(result, "user1|user2|user3"); + assertEquals("user1|user2|user3", result); //Null list expected return null result = FuncUtils.mkString(null, split); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java index 0638b3858e..1fbd9ab354 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -207,12 +208,15 @@ public class FlinkParameters extends AbstractParameters { @Override public List getResourceFilesList() { - if(resourceList !=null ) { - this.resourceList.add(mainJar); - return resourceList.stream() - .map(p -> p.getRes()).collect(Collectors.toList()); + if(resourceList != null ) { + List resourceFiles = resourceList.stream() + .map(ResourceInfo::getRes).collect(Collectors.toList()); + if(mainJar != null) { + resourceFiles.add(mainJar.getRes()); + } + return resourceFiles; } - return null; + return Collections.emptyList(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index c84848fbae..dc60b04c59 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -44,7 +44,7 @@ public class FileUtils { String fileSuffix = ""; if (StringUtils.isNotEmpty(filename)) { - int lastIndex = filename.lastIndexOf("."); + int lastIndex = filename.lastIndexOf('.'); if (lastIndex > 0) { fileSuffix = filename.substring(lastIndex + 1); } @@ -325,10 +325,8 @@ public class FileUtils { } } else { File parent = file.getParentFile(); - if (parent != null) { - if (!parent.mkdirs() && !parent.isDirectory()) { + if (parent != null && !parent.mkdirs() && !parent.isDirectory()) { throw new IOException("Directory '" + parent + "' could not be created"); - } } } return new FileOutputStream(file, append); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java new file mode 100644 index 0000000000..7ce00e875a --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.task; + +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; +import org.junit.Assert; +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; + +public class FlinkParametersTest { + @Test + public void getResourceFilesList() { + FlinkParameters flinkParameters = new FlinkParameters(); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertTrue(flinkParameters.getResourceFilesList().isEmpty()); + + ResourceInfo mainResource = new ResourceInfo(); + mainResource.setRes("testFlinkMain-1.0.0-SNAPSHOT.jar"); + flinkParameters.setMainJar(mainResource); + + List resourceInfos = new LinkedList<>(); + ResourceInfo resourceInfo1 = new ResourceInfo(); + resourceInfo1.setRes("testFlinkParameters1.jar"); + resourceInfos.add(resourceInfo1); + + flinkParameters.setResourceList(resourceInfos); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertEquals(2, flinkParameters.getResourceFilesList().size()); + + ResourceInfo resourceInfo2 = new ResourceInfo(); + resourceInfo2.setRes("testFlinkParameters2.jar"); + resourceInfos.add(resourceInfo2); + + flinkParameters.setResourceList(resourceInfos); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertEquals(3, flinkParameters.getResourceFilesList().size()); + } +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index 89458f6f1c..96217842bf 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -30,29 +30,32 @@ public class FileUtilsTest { @Test public void suffix() { - Assert.assertEquals(FileUtils.suffix("ninfor.java"),"java"); + Assert.assertEquals("java", FileUtils.suffix("ninfor.java")); + Assert.assertEquals("", FileUtils.suffix(null)); + Assert.assertEquals("", FileUtils.suffix("")); + Assert.assertEquals("", FileUtils.suffix("ninfor-java")); } @Test public void testGetDownloadFilename() { PowerMockito.mockStatic(DateUtils.class); PowerMockito.when(DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059"); - Assert.assertEquals(FileUtils.getDownloadFilename("test"), - "/tmp/dolphinscheduler/download/20190101101059/test"); + Assert.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test", + FileUtils.getDownloadFilename("test")); } @Test public void testGetUploadFilename() { - Assert.assertEquals(FileUtils.getUploadFilename("aaa","bbb"), - "/tmp/dolphinscheduler/aaa/resources/bbb"); + Assert.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb", + FileUtils.getUploadFilename("aaa","bbb")); } @Test public void testGetProcessExecDir() { String dir = FileUtils.getProcessExecDir(1,2,3, 4); - Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3/4"); + Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3/4", dir); dir = FileUtils.getProcessExecDir(1,2,3); - Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3"); + Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3", dir); } @Test diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java index 4c33ef8db2..12c7eb2d56 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.server.utils; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; -import org.apache.commons.lang.StringUtils; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -32,12 +31,7 @@ import java.util.List; * spark args utils */ public class FlinkArgsUtils { - - /** - * logger of FlinkArgsUtils - */ - private static final org.slf4j.Logger logger = LoggerFactory.getLogger(FlinkArgsUtils.class); - + private static final String LOCAL_DEPLOY_MODE = "local"; /** * build args * @param param flink parameters @@ -52,7 +46,7 @@ public class FlinkArgsUtils { deployMode = tmpDeployMode; } - if (!"local".equals(deployMode)) { + if (!LOCAL_DEPLOY_MODE.equals(deployMode)) { args.add(Constants.FLINK_RUN_MODE); //-m args.add(Constants.FLINK_YARN_CLUSTER); //yarn-cluster @@ -113,12 +107,12 @@ public class FlinkArgsUtils { String queue = param.getQueue(); if (StringUtils.isNotEmpty(others)) { - if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) { + if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) { args.add(Constants.FLINK_QUEUE); args.add(param.getQueue()); } args.add(others); - } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) { + } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) { args.add(Constants.FLINK_QUEUE); args.add(param.getQueue()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java index 5e68acf94e..21ee1dfa40 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java @@ -296,9 +296,7 @@ public class TaskScheduleThread implements Runnable { if (baseParam != null) { List projectResourceFiles = baseParam.getResourceFilesList(); - if (projectResourceFiles != null) { - projectFiles.addAll(projectResourceFiles); - } + projectFiles.addAll(projectResourceFiles); } return new ArrayList<>(projectFiles); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java index 710d2c2505..2e4861e2a2 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java @@ -87,35 +87,35 @@ public class FlinkArgsUtilsTest { } //Expected values and order - assertEquals(result.size(),20); + assertEquals(20, result.size()); - assertEquals(result.get(0),"-m"); - assertEquals(result.get(1),"yarn-cluster"); + assertEquals("-m", result.get(0)); + assertEquals("yarn-cluster", result.get(1)); - assertEquals(result.get(2),"-ys"); + assertEquals("-ys", result.get(2)); assertSame(Integer.valueOf(result.get(3)),slot); - assertEquals(result.get(4),"-ynm"); + assertEquals("-ynm",result.get(4)); assertEquals(result.get(5),appName); - assertEquals(result.get(6),"-yn"); + assertEquals("-yn", result.get(6)); assertSame(Integer.valueOf(result.get(7)),taskManager); - assertEquals(result.get(8),"-yjm"); + assertEquals("-yjm", result.get(8)); assertEquals(result.get(9),jobManagerMemory); - assertEquals(result.get(10),"-ytm"); + assertEquals("-ytm", result.get(10)); assertEquals(result.get(11),taskManagerMemory); - assertEquals(result.get(12),"-d"); + assertEquals("-d", result.get(12)); - assertEquals(result.get(13),"-c"); + assertEquals("-c", result.get(13)); assertEquals(result.get(14),mainClass); assertEquals(result.get(15),mainJar.getRes()); assertEquals(result.get(16),mainArgs); - assertEquals(result.get(17),"--qu"); + assertEquals("--qu", result.get(17)); assertEquals(result.get(18),queue); assertEquals(result.get(19),others); @@ -125,7 +125,7 @@ public class FlinkArgsUtilsTest { param1.setQueue(queue); param1.setDeployMode(mode); result = FlinkArgsUtils.buildArgs(param1); - assertEquals(result.size(),5); + assertEquals(5, result.size()); } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index a7feec0e14..8b91add6d7 100644 --- a/pom.xml +++ b/pom.xml @@ -691,6 +691,7 @@ **/common/threadutils/*.java **/common/graph/*.java **/common/queue/*.java + **/common/task/FlinkParametersTest.java **/common/task/SqoopParameterEntityTest.java **/api/utils/CheckUtilsTest.java **/api/utils/FileUtilsTest.java From ce175310245cf78be91339545cf1f0b8cdd168d2 Mon Sep 17 00:00:00 2001 From: AhahaGe Date: Tue, 10 Mar 2020 22:00:40 +0800 Subject: [PATCH 26/37] add .asf.yaml file (#2136) Co-authored-by: guirong.ggr --- .asf.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .asf.yaml diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 0000000000..fa71a46552 --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,10 @@ +staging: + profile: ~ + whoami: dev + foo: trigger + +publish: + whoami: dev + +github: + description: “Dolphin Scheduler is a distributed and easy-to-extend visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)” \ No newline at end of file From 1db4ac6b63afedc2622aa7997cec701c2096e68f Mon Sep 17 00:00:00 2001 From: AhahaGe Date: Tue, 10 Mar 2020 22:28:40 +0800 Subject: [PATCH 27/37] edit README.md modify word from expand to extend (#2138) Co-authored-by: guirong.ggr --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a9a164b85..ebd620efee 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Dolphin Scheduler Official Website ### Design features: -A distributed and easy-to-expand visual DAG workflow scheduling system. Dedicated to solving the complex dependencies in data processing, making the scheduling system `out of the box` for data processing. +A distributed and easy-to-extend visual DAG workflow scheduling system. Dedicated to solving the complex dependencies in data processing, making the scheduling system `out of the box` for data processing. Its main objectives are as follows: - Associate the Tasks according to the dependencies of the tasks in a DAG graph, which can visualize the running state of task in real time. From b03e0a02c5533ac6c6dc7189cef96b7f686e5497 Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Thu, 12 Mar 2020 10:09:13 +0800 Subject: [PATCH 28/37] fix wrong sonar analysis with checkout V2. (#2148) --- .github/workflows/ci_ut.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 48cd1e16ce..12f7c04ed6 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -15,7 +15,7 @@ # limitations under the License. # -on: ["pull_request"] +on: ["pull_request", "push"] env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler @@ -52,7 +52,15 @@ jobs: run: | export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' mvn test -B -Dmaven.test.skip=false + - name: Upload coverage report to codecov + if: github.event_name == 'pull_request' + run: | CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) + - name: Git fetch unshallow + run: | + git fetch --unshallow + git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch origin - name: Run SonarCloud Analysis run: > mvn verify --batch-mode From ebfacbfcfa73d3dbbace14a539f8fce0fd7a894d Mon Sep 17 00:00:00 2001 From: liwenhe1993 <32166572+liwenhe1993@users.noreply.github.com> Date: Thu, 12 Mar 2020 10:11:07 +0800 Subject: [PATCH 29/37] docker run failed when docker command was `docker run -d` (#2125) * docker run failed when docker command was `docker run -d` 1. add tini 2. replace `tee` command by `tail` command in startup.sh 3. api-server need to support zookeeper * modify `check.sh` in dockerfile * add init directories and log files --- dockerfile/Dockerfile | 6 +++--- dockerfile/hooks/check | 2 +- dockerfile/startup.sh | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index 8fa4886f31..acd2cb99e5 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -23,11 +23,11 @@ ENV TZ Asia/Shanghai ENV LANG C.UTF-8 ENV DEBIAN_FRONTEND noninteractive -#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip kazoo. +#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip tini kazoo. #If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: #RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories RUN apk update && \ - apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip && \ + apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip tini && \ apk add --update procps && \ openrc boot && \ pip install kazoo @@ -92,4 +92,4 @@ RUN rm -rf /var/cache/apk/* #9. expose port EXPOSE 2181 2888 3888 5432 12345 50051 8888 -ENTRYPOINT ["/root/startup.sh"] \ No newline at end of file +ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] \ No newline at end of file diff --git a/dockerfile/hooks/check b/dockerfile/hooks/check index fbc3f8bd44..21f3f4db12 100644 --- a/dockerfile/hooks/check +++ b/dockerfile/hooks/check @@ -17,7 +17,7 @@ # echo "------ dolphinscheduler check - server - status -------" sleep 20 -server_num=$(docker top `docker container list | grep startup | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) +server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) if [ $server_num -eq 5 ] then echo "Server all start successfully" diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index 610a86f5d3..30439c2321 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -164,6 +164,7 @@ case "$1" in LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log ;; (api-server) + initZK initPostgreSQL initApiServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log @@ -187,6 +188,9 @@ case "$1" in ;; esac -echo "tee begin" -exec tee ${LOGFILE} +# init directories and log files +mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null >> ${LOGFILE} + +echo "tail begin" +exec bash -c "tail -n 1 -f ${LOGFILE}" From e27ba3fd978006a56d26bbf1f7d15255296af5fe Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Thu, 12 Mar 2020 17:38:38 +0800 Subject: [PATCH 30/37] use logger to print exception (#2151) --- .../org/apache/dolphinscheduler/common/shell/AbstractShell.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java index 101da18b2c..f846b19741 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java @@ -335,7 +335,7 @@ public abstract class AbstractShell { try{ entry.getValue().destroy(); } catch (Exception e) { - e.printStackTrace(); + logger.error("Destroy All Processes error", e); } } From e3065dbfcc026639420aab91e9fae4246e593c69 Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Thu, 12 Mar 2020 17:39:25 +0800 Subject: [PATCH 31/37] fix field access modifier (#2150) --- .../org/apache/dolphinscheduler/api/utils/ZooKeeperState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java index 5aa6be858a..a33754397d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java @@ -121,7 +121,7 @@ public class ZooKeeperState { private class SendThread extends Thread { private String cmd; - public String ret = ""; + private String ret = ""; public SendThread(String cmd) { this.cmd = cmd; From afb907b4b019cbc9f0ed85c25854e9c381561a4f Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Thu, 12 Mar 2020 17:41:10 +0800 Subject: [PATCH 32/37] fix unequal number of arguments and placeholder (#2149) --- .../java/org/apache/dolphinscheduler/alert/utils/MailUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 99efdc8a6a..130ad8ade6 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -334,7 +334,7 @@ public class MailUtils { * @param e the exception */ private static void handleException(Collection receivers, Map retMap, Exception e) { - logger.error("Send email to {} failed {}", receivers, e); + logger.error("Send email to {} failed", receivers, e); retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed," + e.toString()); } From 46e30b1e50727bba7a553baf64e32bb07de14478 Mon Sep 17 00:00:00 2001 From: t1mon <178317391@qq.com> Date: Thu, 12 Mar 2020 17:50:37 +0800 Subject: [PATCH 33/37] Add mail server configuration description information (#2143) * Add mail server configuration description information * Remove mail server configuration note in alert.properties --- dolphinscheduler-alert/src/main/resources/alert.properties | 1 - install.sh | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 000d0653b7..9f5acea188 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -28,7 +28,6 @@ mail.server.port=25 mail.sender=xxx@xxx.com mail.user=xxx@xxx.com mail.passwd=111111 - # TLS mail.smtp.starttls.enable=true # SSL diff --git a/install.sh b/install.sh index 8735543c47..ed66ce5bd9 100644 --- a/install.sh +++ b/install.sh @@ -109,6 +109,7 @@ mailProtocol="SMTP" mailServerHost="smtp.exmail.qq.com" # mail server port +# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct. mailServerPort="25" # sender @@ -118,6 +119,7 @@ mailSender="xxxxxxxxxx" mailUser="xxxxxxxxxx" # sender password +# note: The mail.passwd is email service authorization code, not the email login password. mailPassword="xxxxxxxxxx" # TLS mail protocol support From 712a1bc126ed32259edaaa9e2c8b6afb71fd2f6e Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 12 Mar 2020 17:59:16 +0800 Subject: [PATCH 34/37] Limit customization file content to no more than 3000 lines(#2128) (#2141) * Change DOM label * Change name to lowercase * Limit customization file content to no more than 3000 lines --- .../pages/file/pages/create/index.vue | 5 ++- .../resource/pages/file/pages/edit/index.vue | 38 ++++++++++++------- .../src/js/module/i18n/locale/en_US.js | 1 + .../src/js/module/i18n/locale/zh_CN.js | 1 + 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue index caa31ed4d8..df84f0f292 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue @@ -132,7 +132,10 @@ this.$message.warning(`${i18n.$t('Please enter the resource content')}`) return false } - + if (editor.doc.size>3000) { + this.$message.warning(`${i18n.$t('Resource content cannot exceed 3000 lines')}`) + return false + } return true }, /** diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue index a9aee1c9fd..a0d1d7d187 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue @@ -45,6 +45,7 @@ @@ -257,4 +272,4 @@ } } } - \ No newline at end of file + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue index eead745a06..6594ffcf74 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue @@ -65,7 +65,7 @@
@@ -186,8 +186,8 @@
@@ -385,7 +385,7 @@
@@ -556,7 +556,8 @@ targetType:"HDFS", sourceMysqlParams:{ - srcDatasource:-1, + srcType:"MYSQL", + srcDatasource:"", srcTable:"", srcQueryType:"1", srcQuerySql:'', @@ -588,7 +589,8 @@ }, targetMysqlParams:{ - targetDatasource:-1, + targetType:"MYSQL", + targetDatasource:"", targetTable:"", targetColumns:"", fieldsTerminated:"", @@ -680,6 +682,7 @@ * return data source */ _onSourceDsData (o) { + this.sourceMysqlParams.srcType = o.type this.sourceMysqlParams.srcDatasource = o.datasource }, @@ -687,6 +690,7 @@ * return data source */ _onTargetDsData (o) { + this.targetMysqlParams.targetType = o.type this.targetMysqlParams.targetDatasource = o.datasource }, @@ -697,7 +701,7 @@ var params = null switch(this.sourceType){ case "MYSQL": - this.sourceMysqlParams.srcQuerySql = editor.getValue() + this.sourceMysqlParams.srcQuerySql = editor ? editor.getValue() : this.sourceMysqlParams.srcQuerySql params = JSON.stringify(this.sourceMysqlParams) break; case "ORACLE": @@ -879,7 +883,9 @@ * Processing code highlighting */ _handlerEditor () { - editor = codemirror('code-sql-mirror', { + this._destroyEditor() + + editor = codemirror('code-sqoop-mirror', { mode: 'sql', readOnly: this.isDetails }) @@ -892,9 +898,15 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', this.changes) + editor.setValue(this.sourceMysqlParams.srcQuerySql) return editor @@ -906,6 +918,27 @@ _onLocalParams (a) { this.localParams = a }, + + _cacheParams () { + this.$emit('on-cache-params', { + concurrency:this.concurrency, + modelType:this.modelType, + sourceType:this.sourceType, + targetType:this.targetType, + sourceParams:this._handleSourceParams(), + targetParams:this._handleTargetParams(), + localParams:this.localParams + }); + }, + + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sqoop-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'changes', this.changes) + editor = null + } + }, }, watch: { // Listening to sqlType @@ -927,11 +960,12 @@ }, //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, created () { + this._destroyEditor() let o = this.backfillItem // Non-null objects represent backfill @@ -963,11 +997,28 @@ */ if (editor) { editor.toTextArea() // Uninstall - editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'changes', this.changes) + editor = null } }, computed: { + cacheParams () { + return { + concurrency:this.concurrency, + modelType:this.modelType, + sourceType:this.sourceType, + targetType:this.targetType, + localParams:this.localParams, + sourceMysqlParams:this.sourceMysqlParams, + sourceHdfsParams:this.sourceHdfsParams, + sourceHiveParams:this.sourceHiveParams, + targetHdfsParams:this.targetHdfsParams, + targetMysqlParams:this.targetMysqlParams, + targetHiveParams:this.targetHiveParams + } + } }, components: { mListBox, mDatasource, mLocalParams} }