Browse Source

[Feature-5987][Server] Support to set multiple environment configs for a certain worker. (#6082)

* support multi environments

* add some test cases

* add an environment vue component

* improve environment form

* improve environment form

* add environment worker group relation

* add environment worker group relation

* add the environment choice for formModel

* set an environment for the task

* modify the modal form of starting process

* add the environment config to TaskExecutionContext

* add the environment config to the timing form

* fix conflicts

* fix issues of the code style

* fix some issues of the code style

* fix some issues of the code style

* fix some issues of the code style

* fix some issues of the code style

* fix some issues of the code style

* fix some bugs in the code review

* add the same table and columns to support H2

* fix some bugs
2.0.7-release
Hua Jiang 3 years ago committed by GitHub
parent
commit
d7af95f98c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 240
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java
  2. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java
  3. 9
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java
  4. 129
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/EnvironmentDto.java
  5. 17
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
  6. 102
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java
  7. 41
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java
  8. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java
  9. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java
  10. 463
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java
  11. 76
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java
  12. 9
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
  13. 10
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java
  14. 208
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/EnvironmentControllerTest.java
  15. 310
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentServiceTest.java
  16. 69
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationServiceTest.java
  17. 15
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java
  18. 15
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
  19. 70
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java
  20. 142
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Environment.java
  21. 117
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/EnvironmentWorkerGroupRelation.java
  22. 63
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java
  23. 15
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java
  24. 14
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java
  25. 14
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java
  26. 1
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinitionLog.java
  27. 27
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
  28. 71
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java
  29. 57
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapper.java
  30. 1
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java
  31. 2
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml
  32. 55
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.xml
  33. 40
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapper.xml
  34. 2
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml
  35. 4
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ScheduleMapper.xml
  36. 4
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.xml
  37. 4
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml
  38. 4
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
  39. 199
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapperTest.java
  40. 109
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java
  41. 2
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapperTest.java
  42. 2
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java
  43. 5
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java
  44. 15
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java
  45. 15
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
  46. 26
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
  47. 11
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java
  48. 22
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java
  49. 27
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  50. 2
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
  51. 10
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue
  52. 120
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/relatedEnvironment.vue
  53. 31
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue
  54. 16
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue
  55. 18
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
  56. 226
      dolphinscheduler-ui/src/js/conf/home/pages/security/pages/environment/_source/createEnvironment.vue
  57. 116
      dolphinscheduler-ui/src/js/conf/home/pages/security/pages/environment/_source/list.vue
  58. 163
      dolphinscheduler-ui/src/js/conf/home/pages/security/pages/environment/index.vue
  59. 8
      dolphinscheduler-ui/src/js/conf/home/router/index.js
  60. 71
      dolphinscheduler-ui/src/js/conf/home/store/security/actions.js
  61. 1
      dolphinscheduler-ui/src/js/conf/home/store/security/state.js
  62. 9
      dolphinscheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js
  63. 12
      dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
  64. 12
      dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
  65. 3
      pom.xml
  66. 43
      sql/dolphinscheduler_h2.sql
  67. 40
      sql/dolphinscheduler_mysql.sql
  68. 41
      sql/dolphinscheduler_postgre.sql
  69. 44
      sql/upgrade/1.4.0_schema/mysql/dolphinscheduler_ddl.sql
  70. 58
      sql/upgrade/1.4.0_schema/postgresql/dolphinscheduler_ddl.sql

240
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java

@ -0,0 +1,240 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.controller;
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_ENVIRONMENT_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_ENVIRONMENT_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ENVIRONMENT_BY_CODE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ENVIRONMENT_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ENVIRONMENT_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_ENVIRONMENT_ERROR;
import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.EnvironmentService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestAttribute;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import springfox.documentation.annotations.ApiIgnore;
/**
* environment controller
*/
@Api(tags = "ENVIRONMENT_TAG")
@RestController
@RequestMapping("environment")
public class EnvironmentController extends BaseController {
@Autowired
private EnvironmentService environmentService;
/**
* create environment
*
* @param loginUser login user
* @param name environment name
* @param config config
* @param description description
* @return returns an error if it exists
*/
@ApiOperation(value = "createEnvironment", notes = "CREATE_ENVIRONMENT_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "name", value = "ENVIRONMENT_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "config", value = "CONFIG", required = true, dataType = "String"),
@ApiImplicitParam(name = "description", value = "ENVIRONMENT_DESC", dataType = "String"),
@ApiImplicitParam(name = "workerGroups", value = "WORKER_GROUP_LIST", dataType = "String")
})
@PostMapping(value = "/create")
@ResponseStatus(HttpStatus.CREATED)
@ApiException(CREATE_ENVIRONMENT_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result createProject(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("name") String name,
@RequestParam("config") String config,
@RequestParam(value = "description", required = false) String description,
@RequestParam(value = "workerGroups", required = false) String workerGroups) {
Map<String, Object> result = environmentService.createEnvironment(loginUser, name, config, description, workerGroups);
return returnDataList(result);
}
/**
* update environment
*
* @param loginUser login user
* @param code environment code
* @param name environment name
* @param config environment config
* @param description description
* @return update result code
*/
@ApiOperation(value = "updateEnvironment", notes = "UPDATE_ENVIRONMENT_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "code", value = "ENVIRONMENT_CODE", required = true, dataType = "Long", example = "100"),
@ApiImplicitParam(name = "name", value = "ENVIRONMENT_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "config", value = "ENVIRONMENT_CONFIG", required = true, dataType = "String"),
@ApiImplicitParam(name = "description", value = "ENVIRONMENT_DESC", dataType = "String"),
@ApiImplicitParam(name = "workerGroups", value = "WORKER_GROUP_LIST", dataType = "String")
})
@PostMapping(value = "/update")
@ResponseStatus(HttpStatus.OK)
@ApiException(UPDATE_ENVIRONMENT_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result updateEnvironment(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("code") Long code,
@RequestParam("name") String name,
@RequestParam("config") String config,
@RequestParam(value = "description", required = false) String description,
@RequestParam(value = "workerGroups", required = false) String workerGroups) {
Map<String, Object> result = environmentService.updateEnvironmentByCode(loginUser, code, name, config, description, workerGroups);
return returnDataList(result);
}
/**
* query environment details by code
*
* @param environmentCode environment code
* @return environment detail information
*/
@ApiOperation(value = "queryEnvironmentByCode", notes = "QUERY_ENVIRONMENT_BY_CODE_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", required = true, dataType = "Long", example = "100")
})
@GetMapping(value = "/query-by-code")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_ENVIRONMENT_BY_CODE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result queryEnvironmentByCode(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("environmentCode") Long environmentCode) {
Map<String, Object> result = environmentService.queryEnvironmentByCode(environmentCode);
return returnDataList(result);
}
/**
* query environment list paging
*
* @param searchVal search value
* @param pageSize page size
* @param pageNo page number
* @return environment list which the login user have permission to see
*/
@ApiOperation(value = "queryEnvironmentListPaging", notes = "QUERY_ENVIRONMENT_LIST_PAGING_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", dataType = "String"),
@ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", required = true, dataType = "Int", example = "20"),
@ApiImplicitParam(name = "pageNo", value = "PAGE_NO", required = true, dataType = "Int", example = "1")
})
@GetMapping(value = "/list-paging")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_ENVIRONMENT_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result queryEnvironmentListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "searchVal", required = false) String searchVal,
@RequestParam("pageSize") Integer pageSize,
@RequestParam("pageNo") Integer pageNo
) {
Result result = checkPageParams(pageNo, pageSize);
if (!result.checkResult()) {
return result;
}
searchVal = ParameterUtils.handleEscapes(searchVal);
result = environmentService.queryEnvironmentListPaging(pageNo, pageSize, searchVal);
return result;
}
/**
* delete environment by code
*
* @param loginUser login user
* @param environmentCode environment code
* @return delete result code
*/
@ApiOperation(value = "deleteEnvironmentByCode", notes = "DELETE_ENVIRONMENT_BY_CODE_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", required = true, dataType = "Long", example = "100")
})
@PostMapping(value = "/delete")
@ResponseStatus(HttpStatus.OK)
@ApiException(DELETE_ENVIRONMENT_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result deleteEnvironment(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("environmentCode") Long environmentCode
) {
Map<String, Object> result = environmentService.deleteEnvironmentByCode(loginUser, environmentCode);
return returnDataList(result);
}
/**
* query all environment list
*
* @param loginUser login user
* @return all environment list
*/
@ApiOperation(value = "queryAllEnvironmentList", notes = "QUERY_ALL_ENVIRONMENT_LIST_NOTES")
@GetMapping(value = "/query-environment-list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_ENVIRONMENT_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result queryAllEnvironmentList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
Map<String, Object> result = environmentService.queryAllEnvironmentList();
return returnDataList(result);
}
/**
* verify environment and environment name
*
* @param loginUser login user
* @param environmentName environment name
* @return true if the environment name not exists, otherwise return false
*/
@ApiOperation(value = "verifyEnvironment", notes = "VERIFY_ENVIRONMENT_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "environmentName", value = "ENVIRONMENT_NAME", required = true, dataType = "String")
})
@PostMapping(value = "/verify-environment")
@ResponseStatus(HttpStatus.OK)
@ApiException(VERIFY_ENVIRONMENT_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result verifyEnvironment(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "environmentName") String environmentName
) {
Map<String, Object> result = environmentService.verifyEnvironment(environmentName);
return returnDataList(result);
}
}

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java

@ -99,8 +99,9 @@ public class ExecutorController extends BaseController {
@ApiImplicitParam(name = "runMode", value = "RUN_MODE", dataType = "RunMode"),
@ApiImplicitParam(name = "processInstancePriority", value = "PROCESS_INSTANCE_PRIORITY", required = true, dataType = "Priority"),
@ApiImplicitParam(name = "workerGroup", value = "WORKER_GROUP", dataType = "String", example = "default"),
@ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", dataType = "Long", example = "default"),
@ApiImplicitParam(name = "timeout", value = "TIMEOUT", dataType = "Int", example = "100"),
@ApiImplicitParam(name = "expectedParallelismNumber", value = "EXPECTED_PARALLELISM_NUMBER", dataType = "Int", example = "8"),
@ApiImplicitParam(name = "expectedParallelismNumber", value = "EXPECTED_PARALLELISM_NUMBER", dataType = "Int", example = "8")
})
@PostMapping(value = "start-process-instance")
@ResponseStatus(HttpStatus.OK)
@ -119,6 +120,7 @@ public class ExecutorController extends BaseController {
@RequestParam(value = "runMode", required = false) RunMode runMode,
@RequestParam(value = "processInstancePriority", required = false) Priority processInstancePriority,
@RequestParam(value = "workerGroup", required = false, defaultValue = "default") String workerGroup,
@RequestParam(value = "environmentCode", required = false, defaultValue = "-1") Long environmentCode,
@RequestParam(value = "timeout", required = false) Integer timeout,
@RequestParam(value = "startParams", required = false) String startParams,
@RequestParam(value = "expectedParallelismNumber", required = false) Integer expectedParallelismNumber
@ -133,7 +135,7 @@ public class ExecutorController extends BaseController {
}
Map<String, Object> result = execService.execProcessInstance(loginUser, projectName, processDefinitionId, scheduleTime, execType, failureStrategy,
startNodeList, taskDependType, warningType,
warningGroupId, runMode, processInstancePriority, workerGroup, timeout, startParamMap, expectedParallelismNumber);
warningGroupId, runMode, processInstancePriority, workerGroup, environmentCode, timeout, startParamMap, expectedParallelismNumber);
return returnDataList(result);
}

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java

@ -74,7 +74,6 @@ public class SchedulerController extends BaseController {
@Autowired
private SchedulerService schedulerService;
/**
* create schedule
*
@ -99,6 +98,7 @@ public class SchedulerController extends BaseController {
@ApiImplicitParam(name = "failureStrategy", value = "FAILURE_STRATEGY", type = "FailureStrategy"),
@ApiImplicitParam(name = "workerGroupId", value = "WORKER_GROUP_ID", dataType = "Int", example = "100"),
@ApiImplicitParam(name = "workerGroup", value = "WORKER_GROUP", dataType = "String"),
@ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", dataType = "Long"),
@ApiImplicitParam(name = "processInstancePriority", value = "PROCESS_INSTANCE_PRIORITY", type = "Priority"),
})
@PostMapping("/create")
@ -113,9 +113,10 @@ public class SchedulerController extends BaseController {
@RequestParam(value = "warningGroupId", required = false, defaultValue = DEFAULT_NOTIFY_GROUP_ID) int warningGroupId,
@RequestParam(value = "failureStrategy", required = false, defaultValue = DEFAULT_FAILURE_POLICY) FailureStrategy failureStrategy,
@RequestParam(value = "workerGroup", required = false, defaultValue = "default") String workerGroup,
@RequestParam(value = "environmentCode", required = false, defaultValue = "-1") Long environmentCode,
@RequestParam(value = "processInstancePriority", required = false, defaultValue = DEFAULT_PROCESS_INSTANCE_PRIORITY) Priority processInstancePriority) {
Map<String, Object> result = schedulerService.insertSchedule(loginUser, projectName, processDefinitionId, schedule,
warningType, warningGroupId, failureStrategy, processInstancePriority, workerGroup);
warningType, warningGroupId, failureStrategy, processInstancePriority, workerGroup, environmentCode);
return returnDataList(result);
}
@ -145,6 +146,7 @@ public class SchedulerController extends BaseController {
@ApiImplicitParam(name = "failureStrategy", value = "FAILURE_STRATEGY", type = "FailureStrategy"),
@ApiImplicitParam(name = "workerGroupId", value = "WORKER_GROUP_ID", dataType = "Int", example = "100"),
@ApiImplicitParam(name = "workerGroup", value = "WORKER_GROUP", dataType = "String"),
@ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", dataType = "Long"),
@ApiImplicitParam(name = "processInstancePriority", value = "PROCESS_INSTANCE_PRIORITY", type = "Priority")
})
@PostMapping("/update")
@ -158,10 +160,11 @@ public class SchedulerController extends BaseController {
@RequestParam(value = "warningGroupId", required = false) int warningGroupId,
@RequestParam(value = "failureStrategy", required = false, defaultValue = "END") FailureStrategy failureStrategy,
@RequestParam(value = "workerGroup", required = false, defaultValue = "default") String workerGroup,
@RequestParam(value = "environmentCode", required = false, defaultValue = "-1") Long environmentCode,
@RequestParam(value = "processInstancePriority", required = false) Priority processInstancePriority) {
Map<String, Object> result = schedulerService.updateSchedule(loginUser, projectName, id, schedule,
warningType, warningGroupId, failureStrategy, null, processInstancePriority, workerGroup);
warningType, warningGroupId, failureStrategy, null, processInstancePriority, workerGroup, environmentCode);
return returnDataList(result);
}

129
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/EnvironmentDto.java

@ -0,0 +1,129 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto;
import java.util.Date;
import java.util.List;
/**
* EnvironmentDto
*/
public class EnvironmentDto {
private int id;
/**
* environment code
*/
private Long code;
/**
* environment name
*/
private String name;
/**
* config content
*/
private String config;
private String description;
private List<String> workerGroups;
/**
* operator user id
*/
private Integer operator;
private Date createTime;
private Date updateTime;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Long getCode() {
return this.code;
}
public void setCode(Long code) {
this.code = code;
}
public String getConfig() {
return this.config;
}
public void setConfig(String config) {
this.config = config;
}
public String getDescription() {
return this.description;
}
public void setDescription(String description) {
this.description = description;
}
public Integer getOperator() {
return this.operator;
}
public void setOperator(Integer operator) {
this.operator = operator;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
public List<String> getWorkerGroups() {
return workerGroups;
}
public void setWorkerGroups(List<String> workerGroups) {
this.workerGroups = workerGroups;
}
}

17
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@ -310,7 +310,22 @@ public enum Status {
LIST_PAGING_ALERT_PLUGIN_INSTANCE_ERROR(110011, "query plugin instance page error", "分页查询告警实例失败"),
DELETE_ALERT_PLUGIN_INSTANCE_ERROR_HAS_ALERT_GROUP_ASSOCIATED(110012, "failed to delete the alert instance, there is an alarm group associated with this alert instance",
"删除告警实例失败,存在与此告警实例关联的警报组"),
PROCESS_DEFINITION_VERSION_IS_USED(110013,"this process definition version is used","此工作流定义版本被使用");
PROCESS_DEFINITION_VERSION_IS_USED(110013,"this process definition version is used","此工作流定义版本被使用"),
CREATE_ENVIRONMENT_ERROR(120001, "create environment error", "创建环境失败"),
ENVIRONMENT_NAME_EXISTS(120002,"this enviroment name [{0}] already exists","环境名称[{0}]已经存在"),
ENVIRONMENT_NAME_IS_NULL(120003,"this enviroment name shouldn't be empty.","环境名称不能为空"),
ENVIRONMENT_CONFIG_IS_NULL(120004,"this enviroment config shouldn't be empty.","环境配置信息不能为空"),
UPDATE_ENVIRONMENT_ERROR(120005, "update environment [{0}] info error", "更新环境[{0}]信息失败"),
DELETE_ENVIRONMENT_ERROR(120006, "delete environment error", "删除环境信息失败"),
DELETE_ENVIRONMENT_RELATED_TASK_EXISTS(120007, "this environment has been used in tasks,so you can't delete it.", "该环境已经被任务使用,所以不能删除该环境信息"),
QUERY_ENVIRONMENT_BY_NAME_ERROR(1200008, "not found environment [{0}] ", "查询环境名称[{0}]信息不存在"),
QUERY_ENVIRONMENT_BY_CODE_ERROR(1200009, "not found environment [{0}] ", "查询环境编码[{0}]不存在"),
QUERY_ENVIRONMENT_ERROR(1200010, "login user query environment error", "分页查询环境列表错误"),
VERIFY_ENVIRONMENT_ERROR(1200011, "verify environment error", "验证环境信息错误"),
ENVIRONMENT_WORKER_GROUPS_IS_INVALID(1200012, "environment worker groups is invalid format", "环境关联的工作组参数解析错误"),
UPDATE_ENVIRONMENT_WORKER_GROUP_RELATION_ERROR(1200013,"You can't modify the worker group, because the worker group [{0}] and this environment [{1}] already be used in the task [{2}]",
"您不能修改工作组选项,因为该工作组 [{0}] 和 该环境 [{1}] 已经被用在任务 [{2}] 中");
private final int code;

102
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java

@ -0,0 +1,102 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.dao.entity.User;
import java.util.Map;
/**
* environment service
*/
public interface EnvironmentService {
/**
* create environment
*
* @param loginUser login user
* @param name environment name
* @param config environment config
* @param desc environment desc
* @param workerGroups worker groups
*/
Map<String, Object> createEnvironment(User loginUser, String name, String config, String desc, String workerGroups);
/**
* query environment
*
* @param name environment name
*/
Map<String, Object> queryEnvironmentByName(String name);
/**
* query environment
*
* @param code environment code
*/
Map<String, Object> queryEnvironmentByCode(Long code);
/**
* delete environment
*
* @param loginUser login user
* @param code environment code
*/
Map<String, Object> deleteEnvironmentByCode(User loginUser, Long code);
/**
* update environment
*
* @param loginUser login user
* @param code environment code
* @param name environment name
* @param config environment config
* @param desc environment desc
* @param workerGroups worker groups
*/
Map<String, Object> updateEnvironmentByCode(User loginUser, Long code, String name, String config, String desc, String workerGroups);
/**
* query environment paging
*
* @param pageNo page number
* @param searchVal search value
* @param pageSize page size
* @return environment list page
*/
Result queryEnvironmentListPaging(Integer pageNo, Integer pageSize, String searchVal);
/**
* query all environment
*
* @return all environment list
*/
Map<String, Object> queryAllEnvironmentList();
/**
* verify environment name
*
* @param environmentName environment name
* @return true if the environment name not exists, otherwise return false
*/
Map<String, Object> verifyEnvironment(String environmentName);
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import java.util.Map;
/**
* environment worker group relation service
*/
public interface EnvironmentWorkerGroupRelationService {
/**
* query environment worker group relation
*
* @param environmentCode environment code
*/
Map<String, Object> queryEnvironmentWorkerGroupRelation(Long environmentCode);
/**
* query all environment worker group relation
*
* @return all relation list
*/
Map<String, Object> queryAllEnvironmentWorkerGroupRelationList();
}

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java

@ -49,6 +49,7 @@ public interface ExecutorService {
* @param warningGroupId notify group id
* @param processInstancePriority process instance priority
* @param workerGroup worker group name
* @param environmentCode environment code
* @param runMode run mode
* @param timeout timeout
* @param startParams the global param values which pass to new process instance
@ -60,7 +61,7 @@ public interface ExecutorService {
FailureStrategy failureStrategy, String startNodeList,
TaskDependType taskDependType, WarningType warningType, int warningGroupId,
RunMode runMode,
Priority processInstancePriority, String workerGroup, Integer timeout,
Priority processInstancePriority, String workerGroup, Long environmentCode, Integer timeout,
Map<String, String> startParams, Integer expectedParallelismNumber);
/**

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java

@ -43,6 +43,7 @@ public interface SchedulerService {
* @param failureStrategy failure strategy
* @param processInstancePriority process instance priority
* @param workerGroup worker group
* @param environmentCode environment code
* @return create result code
*/
Map<String, Object> insertSchedule(User loginUser, String projectName,
@ -52,7 +53,8 @@ public interface SchedulerService {
int warningGroupId,
FailureStrategy failureStrategy,
Priority processInstancePriority,
String workerGroup);
String workerGroup,
Long environmentCode);
/**
* updateProcessInstance schedule
@ -65,6 +67,7 @@ public interface SchedulerService {
* @param warningGroupId warning group id
* @param failureStrategy failure strategy
* @param workerGroup worker group
* @param environmentCode environment code
* @param processInstancePriority process instance priority
* @param scheduleStatus schedule status
* @return update result code
@ -78,7 +81,8 @@ public interface SchedulerService {
FailureStrategy failureStrategy,
ReleaseState scheduleStatus,
Priority processInstancePriority,
String workerGroup);
String workerGroup,
Long environmentCode);
/**

463
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java

@ -0,0 +1,463 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service.impl;
import org.apache.dolphinscheduler.api.dto.EnvironmentDto;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.EnvironmentService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.commons.collections4.SetUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.core.type.TypeReference;
/**
* task definition service impl
*/
@Service
public class EnvironmentServiceImpl extends BaseServiceImpl implements EnvironmentService {
private static final Logger logger = LoggerFactory.getLogger(EnvironmentServiceImpl.class);
@Autowired
private EnvironmentMapper environmentMapper;
@Autowired
private EnvironmentWorkerGroupRelationMapper relationMapper;
@Autowired
private TaskDefinitionMapper taskDefinitionMapper;
/**
* create environment
*
* @param loginUser login user
* @param name environment name
* @param config environment config
* @param desc environment desc
* @param workerGroups worker groups
*/
@Transactional(rollbackFor = RuntimeException.class)
@Override
public Map<String, Object> createEnvironment(User loginUser, String name, String config, String desc, String workerGroups) {
Map<String, Object> result = new HashMap<>();
if (isNotAdmin(loginUser, result)) {
return result;
}
Map<String, Object> checkResult = checkParams(name,config,workerGroups);
if (checkResult.get(Constants.STATUS) != Status.SUCCESS) {
return checkResult;
}
Environment environment = environmentMapper.queryByEnvironmentName(name);
if (environment != null) {
putMsg(result, Status.ENVIRONMENT_NAME_EXISTS, name);
return result;
}
Environment env = new Environment();
env.setName(name);
env.setConfig(config);
env.setDescription(desc);
env.setOperator(loginUser.getId());
env.setCreateTime(new Date());
env.setUpdateTime(new Date());
long code = 0L;
try {
code = SnowFlakeUtils.getInstance().nextId();
env.setCode(code);
} catch (SnowFlakeException e) {
logger.error("Environment code get error, ", e);
}
if (code == 0L) {
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating environment code");
return result;
}
if (environmentMapper.insert(env) > 0) {
if (StringUtils.isNotEmpty(workerGroups)) {
List<String> workerGroupList = JSONUtils.parseObject(workerGroups, new TypeReference<List<String>>(){});
if (CollectionUtils.isNotEmpty(workerGroupList)) {
workerGroupList.stream().forEach(workerGroup -> {
if (StringUtils.isNotEmpty(workerGroup)) {
EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation();
relation.setEnvironmentCode(env.getCode());
relation.setWorkerGroup(workerGroup);
relation.setOperator(loginUser.getId());
relation.setCreateTime(new Date());
relation.setUpdateTime(new Date());
relationMapper.insert(relation);
}
});
}
}
result.put(Constants.DATA_LIST, env.getCode());
putMsg(result, Status.SUCCESS);
} else {
putMsg(result, Status.CREATE_ENVIRONMENT_ERROR);
}
return result;
}
/**
* query environment paging
*
* @param pageNo page number
* @param searchVal search value
* @param pageSize page size
* @return environment list page
*/
@Override
public Result queryEnvironmentListPaging(Integer pageNo, Integer pageSize, String searchVal) {
Result result = new Result();
Page<Environment> page = new Page<>(pageNo, pageSize);
IPage<Environment> environmentIPage = environmentMapper.queryEnvironmentListPaging(page, searchVal);
PageInfo<EnvironmentDto> pageInfo = new PageInfo<>(pageNo, pageSize);
pageInfo.setTotal((int) environmentIPage.getTotal());
if (CollectionUtils.isNotEmpty(environmentIPage.getRecords())) {
Map<Long, List<String>> relationMap = relationMapper.selectList(null).stream()
.collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode,Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup,Collectors.toList())));
List<EnvironmentDto> dtoList = environmentIPage.getRecords().stream().map(environment -> {
EnvironmentDto dto = new EnvironmentDto();
BeanUtils.copyProperties(environment,dto);
List<String> workerGroups = relationMap.getOrDefault(environment.getCode(),new ArrayList<String>());
dto.setWorkerGroups(workerGroups);
return dto;
}).collect(Collectors.toList());
pageInfo.setTotalList(dtoList);
} else {
pageInfo.setTotalList(new ArrayList<>());
}
result.setData(pageInfo);
putMsg(result, Status.SUCCESS);
return result;
}
/**
* query all environment
*
* @return all environment list
*/
@Override
public Map<String, Object> queryAllEnvironmentList() {
Map<String,Object> result = new HashMap<>();
List<Environment> environmentList = environmentMapper.queryAllEnvironmentList();
if (CollectionUtils.isNotEmpty(environmentList)) {
Map<Long, List<String>> relationMap = relationMapper.selectList(null).stream()
.collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode,Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup,Collectors.toList())));
List<EnvironmentDto> dtoList = environmentList.stream().map(environment -> {
EnvironmentDto dto = new EnvironmentDto();
BeanUtils.copyProperties(environment,dto);
List<String> workerGroups = relationMap.getOrDefault(environment.getCode(),new ArrayList<String>());
dto.setWorkerGroups(workerGroups);
return dto;
}).collect(Collectors.toList());
result.put(Constants.DATA_LIST,dtoList);
} else {
result.put(Constants.DATA_LIST, new ArrayList<>());
}
putMsg(result,Status.SUCCESS);
return result;
}
/**
* query environment
*
* @param code environment code
*/
@Override
public Map<String, Object> queryEnvironmentByCode(Long code) {
Map<String, Object> result = new HashMap<>();
Environment env = environmentMapper.queryByEnvironmentCode(code);
if (env == null) {
putMsg(result, Status.QUERY_ENVIRONMENT_BY_CODE_ERROR, code);
} else {
List<String> workerGroups = relationMapper.queryByEnvironmentCode(env.getCode()).stream()
.map(item -> item.getWorkerGroup())
.collect(Collectors.toList());
EnvironmentDto dto = new EnvironmentDto();
BeanUtils.copyProperties(env,dto);
dto.setWorkerGroups(workerGroups);
result.put(Constants.DATA_LIST, dto);
putMsg(result, Status.SUCCESS);
}
return result;
}
/**
* query environment
*
* @param name environment name
*/
@Override
public Map<String, Object> queryEnvironmentByName(String name) {
Map<String, Object> result = new HashMap<>();
Environment env = environmentMapper.queryByEnvironmentName(name);
if (env == null) {
putMsg(result, Status.QUERY_ENVIRONMENT_BY_NAME_ERROR, name);
} else {
List<String> workerGroups = relationMapper.queryByEnvironmentCode(env.getCode()).stream()
.map(item -> item.getWorkerGroup())
.collect(Collectors.toList());
EnvironmentDto dto = new EnvironmentDto();
BeanUtils.copyProperties(env,dto);
dto.setWorkerGroups(workerGroups);
result.put(Constants.DATA_LIST, dto);
putMsg(result, Status.SUCCESS);
}
return result;
}
/**
* delete environment
*
* @param loginUser login user
* @param code environment code
*/
@Transactional(rollbackFor = RuntimeException.class)
@Override
public Map<String, Object> deleteEnvironmentByCode(User loginUser, Long code) {
Map<String, Object> result = new HashMap<>();
if (isNotAdmin(loginUser, result)) {
return result;
}
Integer relatedTaskNumber = taskDefinitionMapper
.selectCount(new QueryWrapper<TaskDefinition>().lambda().eq(TaskDefinition::getEnvironmentCode,code));
if (relatedTaskNumber > 0) {
putMsg(result, Status.DELETE_ENVIRONMENT_RELATED_TASK_EXISTS);
return result;
}
int delete = environmentMapper.deleteByCode(code);
if (delete > 0) {
relationMapper.delete(new QueryWrapper<EnvironmentWorkerGroupRelation>()
.lambda()
.eq(EnvironmentWorkerGroupRelation::getEnvironmentCode,code));
putMsg(result, Status.SUCCESS);
} else {
putMsg(result, Status.DELETE_ENVIRONMENT_ERROR);
}
return result;
}
/**
* update environment
*
* @param loginUser login user
* @param code environment code
* @param name environment name
* @param config environment config
* @param desc environment desc
* @param workerGroups worker groups
*/
@Transactional(rollbackFor = RuntimeException.class)
@Override
public Map<String, Object> updateEnvironmentByCode(User loginUser, Long code, String name, String config, String desc, String workerGroups) {
Map<String, Object> result = new HashMap<>();
if (isNotAdmin(loginUser, result)) {
return result;
}
Map<String, Object> checkResult = checkParams(name,config,workerGroups);
if (checkResult.get(Constants.STATUS) != Status.SUCCESS) {
return checkResult;
}
Environment environment = environmentMapper.queryByEnvironmentName(name);
if (environment != null && !environment.getCode().equals(code)) {
putMsg(result, Status.ENVIRONMENT_NAME_EXISTS, name);
return result;
}
Set<String> workerGroupSet;
if (StringUtils.isNotEmpty(workerGroups)) {
workerGroupSet = JSONUtils.parseObject(workerGroups, new TypeReference<Set<String>>() {});
} else {
workerGroupSet = new TreeSet<>();
}
Set<String> existWorkerGroupSet = relationMapper
.queryByEnvironmentCode(code)
.stream()
.map(item -> item.getWorkerGroup())
.collect(Collectors.toSet());
Set<String> deleteWorkerGroupSet = SetUtils.difference(existWorkerGroupSet,workerGroupSet).toSet();
Set<String> addWorkerGroupSet = SetUtils.difference(workerGroupSet,existWorkerGroupSet).toSet();
// verify whether the relation of this environment and worker groups can be adjusted
checkResult = checkUsedEnvironmentWorkerGroupRelation(deleteWorkerGroupSet, name, code);
if (checkResult.get(Constants.STATUS) != Status.SUCCESS) {
return checkResult;
}
Environment env = new Environment();
env.setCode(code);
env.setName(name);
env.setConfig(config);
env.setDescription(desc);
env.setOperator(loginUser.getId());
env.setUpdateTime(new Date());
int update = environmentMapper.update(env, new UpdateWrapper<Environment>().lambda().eq(Environment::getCode,code));
if (update > 0) {
deleteWorkerGroupSet.stream().forEach(key -> {
if (StringUtils.isNotEmpty(key)) {
relationMapper.delete(new QueryWrapper<EnvironmentWorkerGroupRelation>()
.lambda()
.eq(EnvironmentWorkerGroupRelation::getEnvironmentCode,code));
}
});
addWorkerGroupSet.stream().forEach(key -> {
if (StringUtils.isNotEmpty(key)) {
EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation();
relation.setEnvironmentCode(code);
relation.setWorkerGroup(key);
relation.setUpdateTime(new Date());
relation.setCreateTime(new Date());
relation.setOperator(loginUser.getId());
relationMapper.insert(relation);
}
});
putMsg(result, Status.SUCCESS);
} else {
putMsg(result, Status.UPDATE_ENVIRONMENT_ERROR, name);
}
return result;
}
/**
* verify environment name
*
* @param environmentName environment name
* @return true if the environment name not exists, otherwise return false
*/
@Override
public Map<String, Object> verifyEnvironment(String environmentName) {
Map<String, Object> result = new HashMap<>();
if (StringUtils.isEmpty(environmentName)) {
putMsg(result, Status.ENVIRONMENT_NAME_IS_NULL);
return result;
}
Environment environment = environmentMapper.queryByEnvironmentName(environmentName);
if (environment != null) {
putMsg(result, Status.ENVIRONMENT_NAME_EXISTS, environmentName);
return result;
}
result.put(Constants.STATUS, Status.SUCCESS);
return result;
}
private Map<String, Object> checkUsedEnvironmentWorkerGroupRelation(Set<String> deleteKeySet,String environmentName, Long environmentCode) {
Map<String, Object> result = new HashMap<>();
for (String workerGroup : deleteKeySet) {
TaskDefinition taskDefinition = taskDefinitionMapper
.selectOne(new QueryWrapper<TaskDefinition>().lambda()
.eq(TaskDefinition::getEnvironmentCode,environmentCode)
.eq(TaskDefinition::getWorkerGroup,workerGroup));
if (Objects.nonNull(taskDefinition)) {
putMsg(result, Status.UPDATE_ENVIRONMENT_WORKER_GROUP_RELATION_ERROR,workerGroup,environmentName,taskDefinition.getName());
return result;
}
}
result.put(Constants.STATUS, Status.SUCCESS);
return result;
}
public Map<String, Object> checkParams(String name, String config, String workerGroups) {
Map<String, Object> result = new HashMap<>();
if (StringUtils.isEmpty(name)) {
putMsg(result, Status.ENVIRONMENT_NAME_IS_NULL);
return result;
}
if (StringUtils.isEmpty(config)) {
putMsg(result, Status.ENVIRONMENT_CONFIG_IS_NULL);
return result;
}
if (StringUtils.isNotEmpty(workerGroups)) {
List<String> workerGroupList = JSONUtils.parseObject(workerGroups, new TypeReference<List<String>>(){});
if (Objects.isNull(workerGroupList)) {
putMsg(result, Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID);
return result;
}
}
result.put(Constants.STATUS, Status.SUCCESS);
return result;
}
}

76
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service.impl;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.EnvironmentWorkerGroupRelationService;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
/**
* task definition service impl
*/
@Service
public class EnvironmentWorkerGroupRelationServiceImpl extends BaseServiceImpl implements
EnvironmentWorkerGroupRelationService {
private static final Logger logger = LoggerFactory.getLogger(EnvironmentWorkerGroupRelationServiceImpl.class);
@Autowired
private EnvironmentWorkerGroupRelationMapper environmentWorkerGroupRelationMapper;
/**
* query environment worker group relation
*
* @param environmentCode environment code
*/
@Override
public Map<String, Object> queryEnvironmentWorkerGroupRelation(Long environmentCode) {
Map<String, Object> result = new HashMap<>();
List<EnvironmentWorkerGroupRelation> relations = environmentWorkerGroupRelationMapper.queryByEnvironmentCode(environmentCode);
result.put(Constants.DATA_LIST, relations);
putMsg(result, Status.SUCCESS);
return result;
}
/**
* query all environment worker group relation
*
* @return all relation list
*/
@Override
public Map<String, Object> queryAllEnvironmentWorkerGroupRelationList() {
Map<String, Object> result = new HashMap<>();
List<EnvironmentWorkerGroupRelation> relations = environmentWorkerGroupRelationMapper.selectList(null);
result.put(Constants.DATA_LIST,relations);
putMsg(result,Status.SUCCESS);
return result;
}
}

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java

@ -118,6 +118,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
* @param warningGroupId notify group id
* @param processInstancePriority process instance priority
* @param workerGroup worker group name
* @param environmentCode environment code
* @param runMode run mode
* @param timeout timeout
* @param startParams the global param values which pass to new process instance
@ -130,7 +131,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
FailureStrategy failureStrategy, String startNodeList,
TaskDependType taskDependType, WarningType warningType, int warningGroupId,
RunMode runMode,
Priority processInstancePriority, String workerGroup, Integer timeout,
Priority processInstancePriority, String workerGroup, Long environmentCode, Integer timeout,
Map<String, String> startParams, Integer expectedParallelismNumber) {
Map<String, Object> result = new HashMap<>();
// timeout is invalid
@ -168,7 +169,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
*/
int create = this.createCommand(commandType, processDefinitionId,
taskDependType, failureStrategy, startNodeList, cronTime, warningType, loginUser.getId(),
warningGroupId, runMode, processInstancePriority, workerGroup, startParams, expectedParallelismNumber);
warningGroupId, runMode, processInstancePriority, workerGroup, environmentCode, startParams, expectedParallelismNumber);
if (create > 0) {
processDefinition.setWarningGroupId(warningGroupId);
@ -495,13 +496,14 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
* @param runMode runMode
* @param processInstancePriority processInstancePriority
* @param workerGroup workerGroup
* @param environmentCode environmentCode
* @return command id
*/
private int createCommand(CommandType commandType, int processDefineId,
TaskDependType nodeDep, FailureStrategy failureStrategy,
String startNodeList, String schedule, WarningType warningType,
int executorId, int warningGroupId,
RunMode runMode, Priority processInstancePriority, String workerGroup,
RunMode runMode, Priority processInstancePriority, String workerGroup, Long environmentCode,
Map<String, String> startParams, Integer expectedParallelismNumber) {
/**
@ -537,6 +539,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
command.setWarningGroupId(warningGroupId);
command.setProcessInstancePriority(processInstancePriority);
command.setWorkerGroup(workerGroup);
command.setEnvironmentCode(environmentCode);
Date start = null;
Date end = null;

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java

@ -106,6 +106,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe
* @param failureStrategy failure strategy
* @param processInstancePriority process instance priority
* @param workerGroup worker group
* @param environmentCode environment code
* @return create result code
*/
@Override
@ -117,7 +118,8 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe
int warningGroupId,
FailureStrategy failureStrategy,
Priority processInstancePriority,
String workerGroup) {
String workerGroup,
Long environmentCode) {
Map<String, Object> result = new HashMap<>();
@ -169,6 +171,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe
scheduleObj.setReleaseState(ReleaseState.OFFLINE);
scheduleObj.setProcessInstancePriority(processInstancePriority);
scheduleObj.setWorkerGroup(workerGroup);
scheduleObj.setEnvironmentCode(environmentCode);
scheduleMapper.insert(scheduleObj);
/**
@ -196,6 +199,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe
* @param warningGroupId warning group id
* @param failureStrategy failure strategy
* @param workerGroup worker group
* @param environmentCode environment code
* @param processInstancePriority process instance priority
* @param scheduleStatus schedule status
* @return update result code
@ -211,7 +215,8 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe
FailureStrategy failureStrategy,
ReleaseState scheduleStatus,
Priority processInstancePriority,
String workerGroup) {
String workerGroup,
Long environmentCode) {
Map<String, Object> result = new HashMap<>();
Project project = projectMapper.queryByName(projectName);
@ -277,6 +282,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe
schedule.setReleaseState(scheduleStatus);
}
schedule.setWorkerGroup(workerGroup);
schedule.setEnvironmentCode(environmentCode);
schedule.setUpdateTime(now);
schedule.setProcessInstancePriority(processInstancePriority);
scheduleMapper.updateById(schedule);

208
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/EnvironmentControllerTest.java

@ -0,0 +1,208 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.controller;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.Preconditions;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import com.fasterxml.jackson.core.type.TypeReference;
/**
* environment controller test
*/
public class EnvironmentControllerTest extends AbstractControllerTest {
private static Logger logger = LoggerFactory.getLogger(EnvironmentControllerTest.class);
private String environmentCode;
public static final String environmentName = "Env1";
public static final String config = "this is config content";
public static final String desc = "this is environment description";
@Before
public void before() throws Exception {
testCreateEnvironment();
}
@After
public void after() throws Exception {
testDeleteEnvironment();
}
public void testCreateEnvironment() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("name",environmentName);
paramsMap.add("config",config);
paramsMap.add("description",desc);
MvcResult mvcResult = mockMvc.perform(post("/environment/create")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isCreated())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference<Result<String>>() {});
logger.info(result.toString());
Assert.assertTrue(result != null && result.isSuccess());
Assert.assertNotNull(result.getData());
logger.info("create environment return result:{}", mvcResult.getResponse().getContentAsString());
environmentCode = (String)result.getData();
}
@Test
public void testUpdateEnvironment() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("code", environmentCode);
paramsMap.add("name","environment_test_update");
paramsMap.add("config","this is config content");
paramsMap.add("desc","the test environment update");
MvcResult mvcResult = mockMvc.perform(post("/environment/update")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
logger.info(result.toString());
Assert.assertTrue(result != null && result.isSuccess());
logger.info("update environment return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testQueryEnvironmentByCode() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("environmentCode", environmentCode);
MvcResult mvcResult = mockMvc.perform(get("/environment/query-by-code")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
logger.info(result.toString());
Assert.assertTrue(result != null && result.isSuccess());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("query environment by id :{}, return result:{}", environmentCode, mvcResult.getResponse().getContentAsString());
}
@Test
public void testQueryEnvironmentListPaging() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("searchVal","test");
paramsMap.add("pageSize","2");
paramsMap.add("pageNo","2");
MvcResult mvcResult = mockMvc.perform(get("/environment/list-paging")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
logger.info(result.toString());
Assert.assertTrue(result != null && result.isSuccess());
logger.info("query list-paging environment return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testQueryAllEnvironmentList() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
MvcResult mvcResult = mockMvc.perform(get("/environment/query-environment-list")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
logger.info(result.toString());
Assert.assertTrue(result != null && result.isSuccess());
logger.info("query all environment return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testVerifyEnvironment() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("environmentName",environmentName);
MvcResult mvcResult = mockMvc.perform(post("/environment/verify-environment")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
logger.info(result.toString());
Assert.assertTrue(result.isStatus(Status.ENVIRONMENT_NAME_EXISTS));
logger.info("verify environment return result:{}", mvcResult.getResponse().getContentAsString());
}
private void testDeleteEnvironment() throws Exception {
Preconditions.checkNotNull(environmentCode);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("environmentCode", environmentCode);
MvcResult mvcResult = mockMvc.perform(post("/environment/delete")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
logger.info(result.toString());
Assert.assertTrue(result != null && result.isSuccess());
logger.info("delete environment return result:{}", mvcResult.getResponse().getContentAsString());
}
}

310
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentServiceTest.java

@ -0,0 +1,310 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.EnvironmentServiceImpl;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.assertj.core.util.Lists;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
/**
* environment service test
*/
@RunWith(MockitoJUnitRunner.class)
public class EnvironmentServiceTest {
public static final Logger logger = LoggerFactory.getLogger(EnvironmentServiceTest.class);
@InjectMocks
private EnvironmentServiceImpl environmentService;
@Mock
private EnvironmentMapper environmentMapper;
@Mock
private EnvironmentWorkerGroupRelationMapper relationMapper;
@Mock
private TaskDefinitionMapper taskDefinitionMapper;
public static final String testUserName = "environmentServerTest";
public static final String environmentName = "Env1";
public static final String workerGroups = "[\"default\"]";
@Before
public void setUp(){
}
@After
public void after(){
}
@Test
public void testCreateEnvironment() {
User loginUser = getGeneralUser();
Map<String, Object> result = environmentService.createEnvironment(loginUser,environmentName,getConfig(),getDesc(),workerGroups);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
loginUser = getAdminUser();
result = environmentService.createEnvironment(loginUser,environmentName,"",getDesc(),workerGroups);
logger.info(result.toString());
Assert.assertEquals(Status.ENVIRONMENT_CONFIG_IS_NULL, result.get(Constants.STATUS));
result = environmentService.createEnvironment(loginUser,"",getConfig(),getDesc(),workerGroups);
logger.info(result.toString());
Assert.assertEquals(Status.ENVIRONMENT_NAME_IS_NULL, result.get(Constants.STATUS));
result = environmentService.createEnvironment(loginUser,environmentName,getConfig(),getDesc(),"test");
logger.info(result.toString());
Assert.assertEquals(Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID, result.get(Constants.STATUS));
Mockito.when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(getEnvironment());
result = environmentService.createEnvironment(loginUser,environmentName,getConfig(),getDesc(),workerGroups);
logger.info(result.toString());
Assert.assertEquals(Status.ENVIRONMENT_NAME_EXISTS, result.get(Constants.STATUS));
Mockito.when(environmentMapper.insert(Mockito.any(Environment.class))).thenReturn(1);
Mockito.when(relationMapper.insert(Mockito.any(EnvironmentWorkerGroupRelation.class))).thenReturn(1);
result = environmentService.createEnvironment(loginUser,"testName","test","test",workerGroups);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testCheckParams() {
Map<String, Object> result = environmentService.checkParams(environmentName,getConfig(),"test");
Assert.assertEquals(Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID, result.get(Constants.STATUS));
}
@Test
public void testUpdateEnvironmentByCode() {
User loginUser = getGeneralUser();
Map<String, Object> result = environmentService.updateEnvironmentByCode(loginUser,1L,environmentName,getConfig(),getDesc(),workerGroups);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
loginUser = getAdminUser();
result = environmentService.updateEnvironmentByCode(loginUser,1L,environmentName,"",getDesc(),workerGroups);
logger.info(result.toString());
Assert.assertEquals(Status.ENVIRONMENT_CONFIG_IS_NULL, result.get(Constants.STATUS));
result = environmentService.updateEnvironmentByCode(loginUser,1L,"",getConfig(),getDesc(),workerGroups);
logger.info(result.toString());
Assert.assertEquals(Status.ENVIRONMENT_NAME_IS_NULL, result.get(Constants.STATUS));
result = environmentService.updateEnvironmentByCode(loginUser,1L,environmentName,getConfig(),getDesc(),"test");
logger.info(result.toString());
Assert.assertEquals(Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID, result.get(Constants.STATUS));
Mockito.when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(getEnvironment());
result = environmentService.updateEnvironmentByCode(loginUser,2L,environmentName,getConfig(),getDesc(),workerGroups);
logger.info(result.toString());
Assert.assertEquals(Status.ENVIRONMENT_NAME_EXISTS, result.get(Constants.STATUS));
Mockito.when(environmentMapper.update(Mockito.any(Environment.class),Mockito.any(Wrapper.class))).thenReturn(1);
result = environmentService.updateEnvironmentByCode(loginUser,1L,"testName","test","test",workerGroups);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testQueryAllEnvironmentList() {
Mockito.when(environmentMapper.queryAllEnvironmentList()).thenReturn(Lists.newArrayList(getEnvironment()));
Map<String, Object> result = environmentService.queryAllEnvironmentList();
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
List<Environment> list = (List<Environment>)(result.get(Constants.DATA_LIST));
Assert.assertEquals(1,list.size());
}
@Test
public void testQueryEnvironmentListPaging() {
IPage<Environment> page = new Page<>(1, 10);
page.setRecords(getList());
page.setTotal(1L);
Mockito.when(environmentMapper.queryEnvironmentListPaging(Mockito.any(Page.class), Mockito.eq(environmentName))).thenReturn(page);
Result result = environmentService.queryEnvironmentListPaging(1, 10, environmentName);
logger.info(result.toString());
PageInfo<Environment> pageInfo = (PageInfo<Environment>) result.getData();
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList()));
}
@Test
public void testQueryEnvironmentByName() {
Mockito.when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(null);
Map<String, Object> result = environmentService.queryEnvironmentByName(environmentName);
logger.info(result.toString());
Assert.assertEquals(Status.QUERY_ENVIRONMENT_BY_NAME_ERROR,result.get(Constants.STATUS));
Mockito.when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(getEnvironment());
result = environmentService.queryEnvironmentByName(environmentName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testQueryEnvironmentByCode() {
Mockito.when(environmentMapper.queryByEnvironmentCode(1L)).thenReturn(null);
Map<String, Object> result = environmentService.queryEnvironmentByCode(1L);
logger.info(result.toString());
Assert.assertEquals(Status.QUERY_ENVIRONMENT_BY_CODE_ERROR,result.get(Constants.STATUS));
Mockito.when(environmentMapper.queryByEnvironmentCode(1L)).thenReturn(getEnvironment());
result = environmentService.queryEnvironmentByCode(1L);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testDeleteEnvironmentByCode() {
User loginUser = getGeneralUser();
Map<String, Object> result = environmentService.deleteEnvironmentByCode(loginUser,1L);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
loginUser = getAdminUser();
Mockito.when(taskDefinitionMapper.selectCount(Mockito.any(LambdaQueryWrapper.class))).thenReturn(1);
result = environmentService.deleteEnvironmentByCode(loginUser,1L);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_ENVIRONMENT_RELATED_TASK_EXISTS, result.get(Constants.STATUS));
Mockito.when(taskDefinitionMapper.selectCount(Mockito.any(LambdaQueryWrapper.class))).thenReturn(0);
Mockito.when(environmentMapper.deleteByCode(1L)).thenReturn(1);
result = environmentService.deleteEnvironmentByCode(loginUser,1L);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testVerifyEnvironment() {
Map<String, Object> result = environmentService.verifyEnvironment("");
logger.info(result.toString());
Assert.assertEquals(Status.ENVIRONMENT_NAME_IS_NULL, result.get(Constants.STATUS));
Mockito.when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(getEnvironment());
result = environmentService.verifyEnvironment(environmentName);
logger.info(result.toString());
Assert.assertEquals(Status.ENVIRONMENT_NAME_EXISTS, result.get(Constants.STATUS));
}
private Environment getEnvironment() {
Environment environment = new Environment();
environment.setId(1);
environment.setCode(1L);
environment.setName(environmentName);
environment.setConfig(getConfig());
environment.setDescription(getDesc());
environment.setOperator(1);
return environment;
}
/**
* create an environment description
*/
private String getDesc() {
return "create an environment to test ";
}
/**
* create an environment config
*/
private String getConfig() {
return "export HADOOP_HOME=/opt/hadoop-2.6.5\n"
+ "export HADOOP_CONF_DIR=/etc/hadoop/conf\n"
+ "export SPARK_HOME1=/opt/soft/spark1\n"
+ "export SPARK_HOME2=/opt/soft/spark2\n"
+ "export PYTHON_HOME=/opt/soft/python\n"
+ "export JAVA_HOME=/opt/java/jdk1.8.0_181-amd64\n"
+ "export HIVE_HOME=/opt/soft/hive\n"
+ "export FLINK_HOME=/opt/soft/flink\n"
+ "export DATAX_HOME=/opt/soft/datax\n"
+ "export YARN_CONF_DIR=\"/etc/hadoop/conf\"\n"
+ "\n"
+ "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH\n"
+ "\n"
+ "export HADOOP_CLASSPATH=`hadoop classpath`\n"
+ "\n"
+ "#echo \"HADOOP_CLASSPATH=\"$HADOOP_CLASSPATH";
}
/**
* create general user
*/
private User getGeneralUser() {
User loginUser = new User();
loginUser.setUserType(UserType.GENERAL_USER);
loginUser.setUserName(testUserName);
loginUser.setId(1);
return loginUser;
}
/**
* create admin user
*/
private User getAdminUser() {
User loginUser = new User();
loginUser.setUserType(UserType.ADMIN_USER);
loginUser.setUserName(testUserName);
loginUser.setId(1);
return loginUser;
}
private List<Environment> getList() {
List<Environment> list = new ArrayList<>();
list.add(getEnvironment());
return list;
}
}

69
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationServiceTest.java

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.EnvironmentWorkerGroupRelationServiceImpl;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper;
import java.util.Map;
import org.assertj.core.util.Lists;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* environment service test
*/
@RunWith(MockitoJUnitRunner.class)
public class EnvironmentWorkerGroupRelationServiceTest {
public static final Logger logger = LoggerFactory.getLogger(EnvironmentWorkerGroupRelationServiceTest.class);
@InjectMocks
private EnvironmentWorkerGroupRelationServiceImpl relationService;
@Mock
private EnvironmentWorkerGroupRelationMapper relationMapper;
@Test
public void testQueryEnvironmentWorkerGroupRelation() {
Mockito.when(relationMapper.queryByEnvironmentCode(1L)).thenReturn(Lists.newArrayList(new EnvironmentWorkerGroupRelation()));
Map<String, Object> result = relationService.queryEnvironmentWorkerGroupRelation(1L);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testQueryAllEnvironmentWorkerGroupRelationList() {
Mockito.when(relationMapper.selectList(Mockito.any())).thenReturn(Lists.newArrayList(new EnvironmentWorkerGroupRelation()));
Map<String, Object> result = relationService.queryAllEnvironmentWorkerGroupRelationList();
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
}

15
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java

@ -153,7 +153,7 @@ public class ExecutorService2Test {
null, null,
null, null, 0,
RunMode.RUN_MODE_SERIAL,
Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110, null, 4);
Priority.LOW, Constants.DEFAULT_WORKER_GROUP,-1L, 110, null, 4);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
verify(processService, times(1)).createCommand(any(Command.class));
@ -171,13 +171,12 @@ public class ExecutorService2Test {
null, "n1,n2",
null, null, 0,
RunMode.RUN_MODE_SERIAL,
Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110, null, null);
Priority.LOW, Constants.DEFAULT_WORKER_GROUP,-1L, 110, null, null);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
verify(processService, times(1)).createCommand(any(Command.class));
}
/**
* date error
*/
@ -190,7 +189,7 @@ public class ExecutorService2Test {
null, null,
null, null, 0,
RunMode.RUN_MODE_SERIAL,
Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110, null, null);
Priority.LOW, Constants.DEFAULT_WORKER_GROUP,-1L, 110, null, null);
Assert.assertEquals(Status.START_PROCESS_INSTANCE_ERROR, result.get(Constants.STATUS));
verify(processService, times(0)).createCommand(any(Command.class));
}
@ -207,7 +206,7 @@ public class ExecutorService2Test {
null, null,
null, null, 0,
RunMode.RUN_MODE_SERIAL,
Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110, null, null);
Priority.LOW, Constants.DEFAULT_WORKER_GROUP,-1L, 110, null, null);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
verify(processService, times(1)).createCommand(any(Command.class));
@ -225,7 +224,7 @@ public class ExecutorService2Test {
null, null,
null, null, 0,
RunMode.RUN_MODE_PARALLEL,
Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110, null, null);
Priority.LOW, Constants.DEFAULT_WORKER_GROUP,-1L, 110, null, null);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
verify(processService, times(31)).createCommand(any(Command.class));
@ -243,7 +242,7 @@ public class ExecutorService2Test {
null, null,
null, null, 0,
RunMode.RUN_MODE_PARALLEL,
Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110, null, 4);
Priority.LOW, Constants.DEFAULT_WORKER_GROUP,-1L, 110, null, 4);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
verify(processService, times(4)).createCommand(any(Command.class));
@ -258,7 +257,7 @@ public class ExecutorService2Test {
null, null,
null, null, 0,
RunMode.RUN_MODE_PARALLEL,
Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110, null, 4);
Priority.LOW, Constants.DEFAULT_WORKER_GROUP,-1L, 110, null, 4);
Assert.assertEquals(result.get(Constants.STATUS), Status.MASTER_NOT_EXISTS);
}

15
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java

@ -143,6 +143,11 @@ public class TaskNode {
*/
private String workerGroup;
/**
* environment code
*/
private Long environmentCode;
/**
* task time out
*/
@ -262,6 +267,7 @@ public class TaskNode {
&& Objects.equals(runFlag, taskNode.runFlag)
&& Objects.equals(dependence, taskNode.dependence)
&& Objects.equals(workerGroup, taskNode.workerGroup)
&& Objects.equals(environmentCode, taskNode.environmentCode)
&& Objects.equals(conditionResult, taskNode.conditionResult)
&& CollectionUtils.equalLists(depList, taskNode.depList);
}
@ -422,11 +428,20 @@ public class TaskNode {
+ ", conditionResult='" + conditionResult + '\''
+ ", taskInstancePriority=" + taskInstancePriority
+ ", workerGroup='" + workerGroup + '\''
+ ", environmentCode=" + environmentCode
+ ", timeout='" + timeout + '\''
+ ", delayTime=" + delayTime
+ '}';
}
public void setEnvironmentCode(Long environmentCode) {
this.environmentCode = environmentCode;
}
public Long getEnvironmentCode() {
return this.environmentCode;
}
public String getSwitchResult() {
return switchResult;
}

70
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java

@ -14,15 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.entity;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.TaskDependType;
import org.apache.dolphinscheduler.common.enums.WarningType;
import java.util.Date;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import org.apache.dolphinscheduler.common.enums.*;
import java.util.Date;
/**
* command
@ -114,6 +120,12 @@ public class Command {
@TableField("worker_group")
private String workerGroup;
/**
* environment code
*/
@TableField("environment_code")
private Long environmentCode;
public Command() {
this.taskDependType = TaskDependType.TASK_POST;
this.failureStrategy = FailureStrategy.CONTINUE;
@ -132,6 +144,7 @@ public class Command {
int warningGroupId,
Date scheduleTime,
String workerGroup,
Long environmentCode,
Priority processInstancePriority) {
this.commandType = commandType;
this.executorId = executorId;
@ -145,10 +158,10 @@ public class Command {
this.startTime = new Date();
this.updateTime = new Date();
this.workerGroup = workerGroup;
this.environmentCode = environmentCode;
this.processInstancePriority = processInstancePriority;
}
public TaskDependType getTaskDependType() {
return taskDependType;
}
@ -181,7 +194,6 @@ public class Command {
this.processDefinitionId = processDefinitionId;
}
public FailureStrategy getFailureStrategy() {
return failureStrategy;
}
@ -262,6 +274,14 @@ public class Command {
this.workerGroup = workerGroup;
}
public Long getEnvironmentCode() {
return this.environmentCode;
}
public void setEnvironmentCode(Long environmentCode) {
this.environmentCode = environmentCode;
}
@Override
public boolean equals(Object o) {
if (this == o) {
@ -285,6 +305,11 @@ public class Command {
if (workerGroup != null ? workerGroup.equals(command.workerGroup) : command.workerGroup == null) {
return false;
}
if (environmentCode != null ? environmentCode.equals(command.environmentCode) : command.environmentCode == null) {
return false;
}
if (commandType != command.commandType) {
return false;
}
@ -332,26 +357,29 @@ public class Command {
result = 31 * result + (processInstancePriority != null ? processInstancePriority.hashCode() : 0);
result = 31 * result + (updateTime != null ? updateTime.hashCode() : 0);
result = 31 * result + (workerGroup != null ? workerGroup.hashCode() : 0);
result = 31 * result + (environmentCode != null ? environmentCode.hashCode() : 0);
return result;
}
@Override
public String toString() {
return "Command{" +
"id=" + id +
", commandType=" + commandType +
", processDefinitionId=" + processDefinitionId +
", executorId=" + executorId +
", commandParam='" + commandParam + '\'' +
", taskDependType=" + taskDependType +
", failureStrategy=" + failureStrategy +
", warningType=" + warningType +
", warningGroupId=" + warningGroupId +
", scheduleTime=" + scheduleTime +
", startTime=" + startTime +
", processInstancePriority=" + processInstancePriority +
", updateTime=" + updateTime +
", workerGroup='" + workerGroup + '\'' +
'}';
return "Command{"
+ "id=" + id
+ ", commandType=" + commandType
+ ", processDefinitionId=" + processDefinitionId
+ ", executorId=" + executorId
+ ", commandParam='" + commandParam + '\''
+ ", taskDependType=" + taskDependType
+ ", failureStrategy=" + failureStrategy
+ ", warningType=" + warningType
+ ", warningGroupId=" + warningGroupId
+ ", scheduleTime=" + scheduleTime
+ ", startTime=" + startTime
+ ", processInstancePriority=" + processInstancePriority
+ ", updateTime=" + updateTime
+ ", workerGroup='" + workerGroup + '\''
+ ", environmentCode='" + environmentCode + '\''
+ '}';
}
}

142
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Environment.java

@ -0,0 +1,142 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.entity;
import java.util.Date;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
/**
* Environment
*/
@TableName("t_ds_environment")
public class Environment {
@TableId(value = "id", type = IdType.AUTO)
private int id;
/**
* environment code
*/
private Long code;
/**
* environment name
*/
private String name;
/**
* config content
*/
private String config;
private String description;
/**
* operator user id
*/
private Integer operator;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private Date createTime;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private Date updateTime;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Long getCode() {
return this.code;
}
public void setCode(Long code) {
this.code = code;
}
public String getConfig() {
return this.config;
}
public void setConfig(String config) {
this.config = config;
}
public String getDescription() {
return this.description;
}
public void setDescription(String description) {
this.description = description;
}
public Integer getOperator() {
return this.operator;
}
public void setOperator(Integer operator) {
this.operator = operator;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
@Override
public String toString() {
return "Environment{"
+ "id= " + id
+ ", code= " + code
+ ", name= " + name
+ ", config= " + config
+ ", description= " + description
+ ", operator= " + operator
+ ", createTime= " + createTime
+ ", updateTime= " + updateTime
+ "}";
}
}

117
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/EnvironmentWorkerGroupRelation.java

@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.entity;
import java.util.Date;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
/**
* EnvironmentWorkerGroupRelation
*/
@TableName("t_ds_environment_worker_group_relation")
public class EnvironmentWorkerGroupRelation {
@TableId(value = "id", type = IdType.AUTO)
private int id;
/**
* environment code
*/
private Long environmentCode;
/**
* worker group id
*/
private String workerGroup;
/**
* operator user id
*/
private Integer operator;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private Date createTime;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private Date updateTime;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getWorkerGroup() {
return workerGroup;
}
public void setWorkerGroup(String workerGroup) {
this.workerGroup = workerGroup;
}
public Long getEnvironmentCode() {
return this.environmentCode;
}
public void setEnvironmentCode(Long environmentCode) {
this.environmentCode = environmentCode;
}
public Integer getOperator() {
return this.operator;
}
public void setOperator(Integer operator) {
this.operator = operator;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
@Override
public String toString() {
return "EnvironmentWorkerGroupRelation{"
+ "id= " + id
+ ", environmentCode= " + environmentCode
+ ", workerGroup= " + workerGroup
+ ", operator= " + operator
+ ", createTime= " + createTime
+ ", updateTime= " + updateTime
+ "}";
}
}

63
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java

@ -14,15 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.entity;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.TaskDependType;
import org.apache.dolphinscheduler.common.enums.WarningType;
import java.util.Date;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import org.apache.dolphinscheduler.common.enums.*;
import java.util.Date;
/**
* command
@ -109,6 +115,11 @@ public class ErrorCommand {
*/
private String workerGroup;
/**
* environment code
*/
private Long environmentCode;
public ErrorCommand(){}
public ErrorCommand(Command command, String message) {
@ -124,6 +135,7 @@ public class ErrorCommand {
this.failureStrategy = command.getFailureStrategy();
this.startTime = command.getStartTime();
this.updateTime = command.getUpdateTime();
this.environmentCode = command.getEnvironmentCode();
this.processInstancePriority = command.getProcessInstancePriority();
this.message = message;
}
@ -155,7 +167,6 @@ public class ErrorCommand {
this.message = message;
}
public TaskDependType getTaskDependType() {
return taskDependType;
}
@ -188,7 +199,6 @@ public class ErrorCommand {
this.processDefinitionId = processDefinitionId;
}
public FailureStrategy getFailureStrategy() {
return failureStrategy;
}
@ -277,24 +287,33 @@ public class ErrorCommand {
this.message = message;
}
public Long getEnvironmentCode() {
return this.environmentCode;
}
public void setEnvironmentCode(Long environmentCode) {
this.environmentCode = environmentCode;
}
@Override
public String toString() {
return "ErrorCommand{" +
"id=" + id +
", commandType=" + commandType +
", processDefinitionId=" + processDefinitionId +
", executorId=" + executorId +
", commandParam='" + commandParam + '\'' +
", taskDependType=" + taskDependType +
", failureStrategy=" + failureStrategy +
", warningType=" + warningType +
", warningGroupId=" + warningGroupId +
", scheduleTime=" + scheduleTime +
", startTime=" + startTime +
", processInstancePriority=" + processInstancePriority +
", updateTime=" + updateTime +
", message='" + message + '\'' +
", workerGroup='" + workerGroup + '\'' +
'}';
return "ErrorCommand{"
+ "id=" + id
+ ", commandType=" + commandType
+ ", processDefinitionId=" + processDefinitionId
+ ", executorId=" + executorId
+ ", commandParam='" + commandParam + '\''
+ ", taskDependType=" + taskDependType
+ ", failureStrategy=" + failureStrategy
+ ", warningType=" + warningType
+ ", warningGroupId=" + warningGroupId
+ ", scheduleTime=" + scheduleTime
+ ", startTime=" + startTime
+ ", processInstancePriority=" + processInstancePriority
+ ", updateTime=" + updateTime
+ ", message='" + message + '\''
+ ", workerGroup='" + workerGroup + '\''
+ ", environmentCode='" + environmentCode + '\''
+ '}';
}
}

15
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java

@ -226,6 +226,11 @@ public class ProcessInstance {
*/
private String workerGroup;
/**
* environment code
*/
private Long environmentCode;
/**
* process timeout for warning
*/
@ -505,6 +510,14 @@ public class ProcessInstance {
this.executorName = executorName;
}
public Long getEnvironmentCode() {
return this.environmentCode;
}
public void setEnvironmentCode(Long environmentCode) {
this.environmentCode = environmentCode;
}
/**
* add command to history
*
@ -666,6 +679,8 @@ public class ProcessInstance {
+ ", workerGroup='"
+ workerGroup
+ '\''
+ ", environmentCode="
+ environmentCode
+ ", timeout="
+ timeout
+ ", tenantId="

14
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java

@ -139,6 +139,11 @@ public class Schedule {
*/
private String workerGroup;
/**
* environment code
*/
private Long environmentCode;
public int getWarningGroupId() {
return warningGroupId;
}
@ -286,6 +291,14 @@ public class Schedule {
this.workerGroup = workerGroup;
}
public Long getEnvironmentCode() {
return this.environmentCode;
}
public void setEnvironmentCode(Long environmentCode) {
this.environmentCode = environmentCode;
}
@Override
public String toString() {
return "Schedule{"
@ -308,6 +321,7 @@ public class Schedule {
+ ", warningGroupId=" + warningGroupId
+ ", processInstancePriority=" + processInstancePriority
+ ", workerGroup='" + workerGroup + '\''
+ ", environmentCode='" + environmentCode + '\''
+ '}';
}

14
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java

@ -128,6 +128,11 @@ public class TaskDefinition {
*/
private String workerGroup;
/**
* environment code
*/
private Long environmentCode;
/**
* fail retry times
*/
@ -395,6 +400,14 @@ public class TaskDefinition {
this.delayTime = delayTime;
}
public Long getEnvironmentCode() {
return this.environmentCode;
}
public void setEnvironmentCode(Long environmentCode) {
this.environmentCode = environmentCode;
}
@Override
public String toString() {
return "TaskDefinition{"
@ -414,6 +427,7 @@ public class TaskDefinition {
+ ", userName='" + userName + '\''
+ ", projectName='" + projectName + '\''
+ ", workerGroup='" + workerGroup + '\''
+ ", environmentCode='" + environmentCode + '\''
+ ", failRetryTimes=" + failRetryTimes
+ ", failRetryInterval=" + failRetryInterval
+ ", timeoutFlag=" + timeoutFlag

1
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinitionLog.java

@ -53,6 +53,7 @@ public class TaskDefinitionLog extends TaskDefinition {
this.setUserId(taskDefinition.getUserId());
this.setUserName(taskDefinition.getUserName());
this.setWorkerGroup(taskDefinition.getWorkerGroup());
this.setEnvironmentCode(taskDefinition.getEnvironmentCode());
this.setProjectCode(taskDefinition.getProjectCode());
this.setProjectName(taskDefinition.getProjectName());
this.setResourceIds(taskDefinition.getResourceIds());

27
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java

@ -220,6 +220,15 @@ public class TaskInstance implements Serializable {
*/
private String workerGroup;
/**
* environment code
*/
private Long environmentCode;
/**
* environment config
*/
private String environmentConfig;
/**
* executor id
@ -421,6 +430,22 @@ public class TaskInstance implements Serializable {
this.appLink = appLink;
}
public Long getEnvironmentCode() {
return this.environmentCode;
}
public void setEnvironmentCode(Long environmentCode) {
this.environmentCode = environmentCode;
}
public String getEnvironmentConfig() {
return this.environmentConfig;
}
public void setEnvironmentConfig(String environmentConfig) {
this.environmentConfig = environmentConfig;
}
public DependentParameters getDependency() {
if (this.dependency == null) {
Map<String, Object> taskParamsMap = JSONUtils.toMap(this.getTaskParams(), String.class, Object.class);
@ -623,6 +648,8 @@ public class TaskInstance implements Serializable {
+ ", processInstancePriority=" + processInstancePriority
+ ", dependentResult='" + dependentResult + '\''
+ ", workerGroup='" + workerGroup + '\''
+ ", environmentCode=" + environmentCode
+ ", environmentConfig='" + environmentConfig + '\''
+ ", executorId=" + executorId
+ ", executorName='" + executorName + '\''
+ ", delayTime=" + delayTime

71
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.ibatis.annotations.Param;
import java.util.List;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
/**
* environment mapper interface
*/
public interface EnvironmentMapper extends BaseMapper<Environment> {
/**
* query environment by name
*
* @param name name
* @return environment
*/
Environment queryByEnvironmentName(@Param("environmentName") String name);
/**
* query environment by code
*
* @param environmentCode environmentCode
* @return environment
*/
Environment queryByEnvironmentCode(@Param("environmentCode") Long environmentCode);
/**
* query all environment list
* @return environment list
*/
List<Environment> queryAllEnvironmentList();
/**
* environment page
* @param page page
* @param searchName searchName
* @return environment IPage
*/
IPage<Environment> queryEnvironmentListPaging(IPage<Environment> page, @Param("searchName") String searchName);
/**
* delete environment by code
*
* @param code code
* @return int
*/
int deleteByCode(@Param("code") Long code);
}

57
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapper.java

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.ibatis.annotations.Param;
import java.util.List;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* environment worker group relation mapper interface
*/
public interface EnvironmentWorkerGroupRelationMapper extends BaseMapper<EnvironmentWorkerGroupRelation> {
/**
* environment worker group relation by environmentCode
*
* @param environmentCode environmentCode
* @return EnvironmentWorkerGroupRelation list
*/
List<EnvironmentWorkerGroupRelation> queryByEnvironmentCode(@Param("environmentCode") Long environmentCode);
/**
* environment worker group relation by workerGroupName
*
* @param workerGroupName workerGroupName
* @return EnvironmentWorkerGroupRelation list
*/
List<EnvironmentWorkerGroupRelation> queryByWorkerGroupName(@Param("workerGroupName") String workerGroupName);
/**
* delete environment worker group relation by processCode
*
* @param environmentCode environmentCode
* @param workerGroupName workerGroupName
* @return int
*/
int deleteByCode(@Param("environmentCode") Long environmentCode, @Param("workerGroupName") String workerGroupName);
}

1
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade.shell;
import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager;

2
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml

@ -21,7 +21,7 @@
<select id="getOneToRun" resultType="org.apache.dolphinscheduler.dao.entity.Command">
select cmd.id, cmd.command_type, cmd.process_definition_id, cmd.command_param, cmd.task_depend_type, cmd.failure_strategy,
cmd.warning_type, cmd.warning_group_id, cmd.schedule_time, cmd.start_time, cmd.executor_id, cmd.update_time,
cmd.process_instance_priority, cmd.worker_group
cmd.process_instance_priority, cmd.worker_group, cmd.environment_code
from t_ds_command cmd
join t_ds_process_definition definition on cmd.process_definition_id = definition.id
where definition.release_state = 1 AND definition.flag = 1

55
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.xml

@ -0,0 +1,55 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper">
<sql id="baseSql">
id, code, name, config, description, operator, create_time, update_time
</sql>
<select id="queryByEnvironmentName" resultType="org.apache.dolphinscheduler.dao.entity.Environment">
select
<include refid="baseSql"/>
from t_ds_environment
WHERE name = #{environmentName}
</select>
<select id="queryAllEnvironmentList" resultType="org.apache.dolphinscheduler.dao.entity.Environment">
select
<include refid="baseSql"/>
from t_ds_environment
order by create_time desc
</select>
<select id="queryEnvironmentListPaging" resultType="org.apache.dolphinscheduler.dao.entity.Environment">
select
<include refid="baseSql"/>
from t_ds_environment
where 1=1
<if test="searchName!=null and searchName != ''">
and name like concat('%', #{searchName}, '%')
</if>
order by create_time desc
</select>
<select id="queryByEnvironmentCode" resultType="org.apache.dolphinscheduler.dao.entity.Environment">
select
<include refid="baseSql"/>
from t_ds_environment
where code = #{environmentCode}
</select>
<delete id="deleteByCode">
delete from t_ds_environment where code = #{code}
</delete>
</mapper>

40
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapper.xml

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper">
<sql id="baseSql">
id, environment_code, worker_group, operator, create_time, update_time
</sql>
<select id="queryByEnvironmentCode" resultType="org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation">
select
<include refid="baseSql"/>
from t_ds_environment_worker_group_relation
WHERE environment_code = #{environmentCode}
</select>
<select id="queryByWorkerGroupName" resultType="org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation">
select
<include refid="baseSql"/>
from t_ds_environment_worker_group_relation
WHERE worker_group = #{workerGroupName}
</select>
<delete id="deleteByCode">
delete from t_ds_environment_worker_group_relation
WHERE environment_code = #{environmentCode} and worker_group = #{workerGroupName}
</delete>
</mapper>

2
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml

@ -23,7 +23,7 @@
command_type, command_param, task_depend_type, max_try_times, failure_strategy, warning_type,
warning_group_id, schedule_time, command_start_time, global_params, flag,
update_time, is_sub_process, executor_id, history_cmd,
process_instance_priority, worker_group, timeout, tenant_id, var_pool
process_instance_priority, worker_group,environment_code, timeout, tenant_id, var_pool
</sql>
<select id="queryDetailById" resultType="org.apache.dolphinscheduler.dao.entity.ProcessInstance">
select

4
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ScheduleMapper.xml

@ -20,11 +20,11 @@
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ScheduleMapper">
<sql id="baseSql">
id, process_definition_id, start_time, end_time, timezone_id, crontab, failure_strategy, user_id, release_state,
warning_type, warning_group_id, process_instance_priority, worker_group, create_time, update_time
warning_type, warning_group_id, process_instance_priority, worker_group, environment_code, create_time, update_time
</sql>
<sql id="baseSqlV2">
${alias}.id, ${alias}.process_definition_id, ${alias}.start_time, ${alias}.end_time, ${alias}.timezone_id, ${alias}.crontab, ${alias}.failure_strategy, ${alias}.user_id, ${alias}.release_state,
${alias}.warning_type, ${alias}.warning_group_id, ${alias}.process_instance_priority, ${alias}.worker_group, ${alias}.create_time, ${alias}.update_time
${alias}.warning_type, ${alias}.warning_group_id, ${alias}.process_instance_priority, ${alias}.worker_group, ${alias}.environment_code, ${alias}.create_time, ${alias}.update_time
</sql>
<select id="queryByProcessDefineIdPaging" resultType="org.apache.dolphinscheduler.dao.entity.Schedule">
select p_f.name as process_definition_name, p.name as project_name,u.user_name,

4
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.xml

@ -20,12 +20,12 @@
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper">
<sql id="baseSql">
id, code, name, version, description, project_code, user_id, task_type, task_params, flag, task_priority,
worker_group, fail_retry_times, fail_retry_interval, timeout_flag, timeout_notify_strategy, timeout, delay_time,
worker_group, environment_code, fail_retry_times, fail_retry_interval, timeout_flag, timeout_notify_strategy, timeout, delay_time,
resource_ids, operator, operate_time, create_time, update_time
</sql>
<select id="queryByDefinitionName" resultType="org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog">
select td.id, td.code, td.name, td.version, td.description, td.project_code, td.user_id, td.task_type, td.task_params,
td.flag, td.task_priority, td.worker_group, td.fail_retry_times, td.fail_retry_interval, td.timeout_flag, td.timeout_notify_strategy,
td.flag, td.task_priority, td.worker_group, td.environment_code, td.fail_retry_times, td.fail_retry_interval, td.timeout_flag, td.timeout_notify_strategy,
td.timeout, td.delay_time, td.resource_ids, td.operator,td.operate_time, td.create_time, td.update_time,
u.user_name,p.name as project_name
from t_ds_task_definition_log td

4
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml

@ -20,7 +20,7 @@
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper">
<sql id="baseSql">
id, code, name, version, description, project_code, user_id, task_type, task_params, flag, task_priority,
worker_group, fail_retry_times, fail_retry_interval, timeout_flag, timeout_notify_strategy, timeout, delay_time,
worker_group, environment_code, fail_retry_times, fail_retry_interval, timeout_flag, timeout_notify_strategy, timeout, delay_time,
resource_ids, create_time, update_time
</sql>
<select id="queryByDefinitionName" resultType="org.apache.dolphinscheduler.dao.entity.TaskDefinition">
@ -63,7 +63,7 @@
</select>
<select id="queryByDefinitionId" resultType="org.apache.dolphinscheduler.dao.entity.TaskDefinition">
select td.id, td.code, td.name, td.version, td.description, td.project_code, td.user_id, td.task_type, td.task_params,
td.flag, td.task_priority, td.worker_group, td.fail_retry_times, td.fail_retry_interval, td.timeout_flag, td.timeout_notify_strategy,
td.flag, td.task_priority, td.worker_group, td.environment_code, td.fail_retry_times, td.fail_retry_interval, td.timeout_flag, td.timeout_notify_strategy,
td.timeout, td.delay_time, td.resource_ids, td.create_time, td.update_time, u.user_name,p.name as project_name
from t_ds_task_definition td
JOIN t_ds_user u ON td.user_id = u.id

4
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

@ -21,13 +21,13 @@
<sql id="baseSql">
id, name, task_type, process_instance_id, task_code, task_definition_version, state, submit_time,
start_time, end_time, host, execute_path, log_path, alert_flag, retry_times, pid, app_link,
flag, retry_interval, max_retry_times, task_instance_priority, worker_group, executor_id,
flag, retry_interval, max_retry_times, task_instance_priority, worker_group,environment_code , executor_id,
first_submit_time, delay_time, task_params, var_pool
</sql>
<sql id="baseSqlV2">
${alias}.id, ${alias}.name, ${alias}.task_type, ${alias}.task_code, ${alias}.task_definition_version, ${alias}.process_instance_id, ${alias}.state, ${alias}.submit_time,
${alias}.start_time, ${alias}.end_time, ${alias}.host, ${alias}.execute_path, ${alias}.log_path, ${alias}.alert_flag, ${alias}.retry_times, ${alias}.pid, ${alias}.app_link,
${alias}.flag, ${alias}.retry_interval, ${alias}.max_retry_times, ${alias}.task_instance_priority, ${alias}.worker_group, ${alias}.executor_id,
${alias}.flag, ${alias}.retry_interval, ${alias}.max_retry_times, ${alias}.task_instance_priority, ${alias}.worker_group,${alias}.environment_code , ${alias}.executor_id,
${alias}.first_submit_time, ${alias}.delay_time, ${alias}.task_params, ${alias}.var_pool
</sql>
<update id="setFailoverByHostAndStateArray">

199
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapperTest.java

@ -0,0 +1,199 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.Environment;
import java.util.Date;
import java.util.List;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@RunWith(SpringRunner.class)
@SpringBootTest
@Transactional
@Rollback(false)
public class EnvironmentMapperTest {
@Autowired
EnvironmentMapper environmentMapper;
/**
* insert
*
* @return Environment
*/
private Environment insertOne() {
//insertOne
Environment environment = new Environment();
environment.setName("testEnv");
environment.setCode(1L);
environment.setOperator(1);
environment.setConfig(getConfig());
environment.setDescription(getDesc());
environment.setCreateTime(new Date());
environment.setUpdateTime(new Date());
environmentMapper.insert(environment);
return environment;
}
@Before
public void setUp() {
clearTestData();
}
@After
public void after() {
clearTestData();
}
public void clearTestData() {
environmentMapper.queryAllEnvironmentList().stream().forEach(environment -> {
environmentMapper.deleteByCode(environment.getCode());
});
}
/**
* test update
*/
@Test
public void testUpdate() {
//insertOne
Environment environment = insertOne();
environment.setDescription("new description info");
//update
int update = environmentMapper.updateById(environment);
Assert.assertEquals(update, 1);
}
/**
* test delete
*/
@Test
public void testDelete() {
Environment environment = insertOne();
int delete = environmentMapper.deleteById(environment.getId());
Assert.assertEquals(delete, 1);
}
/**
* test query
*/
@Test
public void testQuery() {
insertOne();
//query
List<Environment> environments = environmentMapper.selectList(null);
Assert.assertEquals(environments.size(), 1);
}
/**
* test query environment by name
*/
@Test
public void testQueryByEnvironmentName() {
Environment entity = insertOne();
Environment environment = environmentMapper.queryByEnvironmentName(entity.getName());
Assert.assertEquals(entity.toString(),environment.toString());
}
/**
* test query environment by code
*/
@Test
public void testQueryByEnvironmentCode() {
Environment entity = insertOne();
Environment environment = environmentMapper.queryByEnvironmentCode(entity.getCode());
Assert.assertEquals(entity.toString(),environment.toString());
}
/**
* test query all environments
*/
@Test
public void testQueryAllEnvironmentList() {
Environment entity = insertOne();
List<Environment> environments = environmentMapper.queryAllEnvironmentList();
Assert.assertEquals(environments.size(), 1);
Assert.assertEquals(entity.toString(),environments.get(0).toString());
}
/**
* test query environment list paging
*/
@Test
public void testQueryEnvironmentListPaging() {
Environment entity = insertOne();
Page<Environment> page = new Page<>(1, 10);
IPage<Environment> environmentIPage = environmentMapper.queryEnvironmentListPaging(page,"");
List<Environment> environmentList = environmentIPage.getRecords();
Assert.assertEquals(environmentList.size(), 1);
environmentIPage = environmentMapper.queryEnvironmentListPaging(page,"abc");
environmentList = environmentIPage.getRecords();
Assert.assertEquals(environmentList.size(), 0);
}
/**
* test query all environments
*/
@Test
public void testDeleteByCode() {
Environment entity = insertOne();
int delete = environmentMapper.deleteByCode(entity.getCode());
Assert.assertEquals(delete, 1);
}
private String getDesc() {
return "create an environment to test ";
}
/**
* create an environment config
*/
private String getConfig() {
return "export HADOOP_HOME=/opt/hadoop-2.6.5\n"
+ "export HADOOP_CONF_DIR=/etc/hadoop/conf\n"
+ "export SPARK_HOME1=/opt/soft/spark1\n"
+ "export SPARK_HOME2=/opt/soft/spark2\n"
+ "export PYTHON_HOME=/opt/soft/python\n"
+ "export JAVA_HOME=/opt/java/jdk1.8.0_181-amd64\n"
+ "export HIVE_HOME=/opt/soft/hive\n"
+ "export FLINK_HOME=/opt/soft/flink\n"
+ "export DATAX_HOME=/opt/soft/datax\n"
+ "export YARN_CONF_DIR=\"/etc/hadoop/conf\"\n"
+ "\n"
+ "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH\n"
+ "\n"
+ "export HADOOP_CLASSPATH=`hadoop classpath`\n"
+ "\n"
+ "#echo \"HADOOP_CLASSPATH=\"$HADOOP_CLASSPATH";
}
}

109
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java

@ -0,0 +1,109 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import java.util.Date;
import java.util.List;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
@RunWith(SpringRunner.class)
@SpringBootTest
@Transactional
@Rollback(true)
public class EnvironmentWorkerGroupRelationMapperTest {
@Autowired
EnvironmentWorkerGroupRelationMapper environmentWorkerGroupRelationMapper;
@Before
public void setUp() {
clearTestData();
}
@After
public void after() {
clearTestData();
}
public void clearTestData() {
environmentWorkerGroupRelationMapper.selectList(null).stream().forEach(environment -> {
environmentWorkerGroupRelationMapper.deleteById(environment.getId());
});
}
/**
* insert
*
* @return ProcessDefinition
*/
private EnvironmentWorkerGroupRelation insertOne() {
//insertOne
EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation();
relation.setEnvironmentCode(1L);
relation.setWorkerGroup("default");
relation.setOperator(1);
relation.setUpdateTime(new Date());
relation.setCreateTime(new Date());
environmentWorkerGroupRelationMapper.insert(relation);
return relation;
}
/**
* test query
*/
@Test
public void testQuery() {
insertOne();
//query
List<EnvironmentWorkerGroupRelation> relations = environmentWorkerGroupRelationMapper.selectList(null);
Assert.assertEquals(relations.size(), 1);
}
@Test
public void testQueryByEnvironmentCode() {
EnvironmentWorkerGroupRelation relation = insertOne();
List<EnvironmentWorkerGroupRelation> environmentWorkerGroupRelations = environmentWorkerGroupRelationMapper.queryByEnvironmentCode(1L);
Assert.assertNotEquals(environmentWorkerGroupRelations.size(), 0);
}
@Test
public void testQueryByWorkerGroupName() {
EnvironmentWorkerGroupRelation relation = insertOne();
List<EnvironmentWorkerGroupRelation> environmentWorkerGroupRelations = environmentWorkerGroupRelationMapper.queryByWorkerGroupName("default");
Assert.assertNotEquals(environmentWorkerGroupRelations.size(), 0);
}
@Test
public void testDeleteByCode() {
EnvironmentWorkerGroupRelation relation = insertOne();
int i = environmentWorkerGroupRelationMapper.deleteByCode(1L, "default");
Assert.assertNotEquals(i, 0);
}
}

2
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapperTest.java

@ -65,6 +65,8 @@ public class TaskDefinitionLogMapperTest {
taskDefinition.setProjectCode(1L);
taskDefinition.setTaskType(TaskType.SHELL.getDesc());
taskDefinition.setUserId(userId);
taskDefinition.setEnvironmentCode(1L);
taskDefinition.setWorkerGroup("default");
taskDefinition.setVersion(1);
taskDefinition.setCreateTime(new Date());
taskDefinition.setUpdateTime(new Date());

2
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java

@ -63,6 +63,8 @@ public class TaskDefinitionMapperTest {
taskDefinition.setTaskType(TaskType.SHELL.getDesc());
taskDefinition.setUserId(userId);
taskDefinition.setResourceIds("1");
taskDefinition.setWorkerGroup("default");
taskDefinition.setEnvironmentCode(1L);
taskDefinition.setVersion(1);
taskDefinition.setCreateTime(new Date());
taskDefinition.setUpdateTime(new Date());

5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java

@ -56,6 +56,7 @@ public class TaskExecutionContextBuilder {
taskExecutionContext.setTaskType(taskInstance.getTaskType());
taskExecutionContext.setLogPath(taskInstance.getLogPath());
taskExecutionContext.setWorkerGroup(taskInstance.getWorkerGroup());
taskExecutionContext.setEnvironmentConfig(taskInstance.getEnvironmentConfig());
taskExecutionContext.setHost(taskInstance.getHost());
taskExecutionContext.setResources(taskInstance.getResources());
taskExecutionContext.setDelayTime(taskInstance.getDelayTime());
@ -76,7 +77,6 @@ public class TaskExecutionContextBuilder {
return this;
}
/**
* build processInstance related info
*
@ -107,7 +107,6 @@ public class TaskExecutionContextBuilder {
return this;
}
/**
* build SQLTask related info
*
@ -119,7 +118,6 @@ public class TaskExecutionContextBuilder {
return this;
}
/**
* build DataxTask related info
*
@ -153,7 +151,6 @@ public class TaskExecutionContextBuilder {
return this;
}
/**
* create
*

15
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java

@ -157,6 +157,12 @@ public class TaskExecutionContext implements Serializable {
*/
private String envFile;
/**
* environmentConfig
*/
private String environmentConfig;
/**
* definedParams
*/
@ -424,6 +430,14 @@ public class TaskExecutionContext implements Serializable {
this.envFile = envFile;
}
public String getEnvironmentConfig() {
return environmentConfig;
}
public void setEnvironmentConfig(String config) {
this.environmentConfig = config;
}
public Map<String, String> getDefinedParams() {
return definedParams;
}
@ -566,6 +580,7 @@ public class TaskExecutionContext implements Serializable {
+ ", taskTimeoutStrategy=" + taskTimeoutStrategy
+ ", taskTimeout=" + taskTimeout
+ ", workerGroup='" + workerGroup + '\''
+ ", environmentConfig='" + environmentConfig + '\''
+ ", delayTime=" + delayTime
+ ", resources=" + resources
+ ", sqlTaskExecutionContext=" + sqlTaskExecutionContext

15
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java

@ -50,6 +50,7 @@ import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
@ -77,6 +78,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
@ -725,10 +727,23 @@ public class WorkflowExecuteThread implements Runnable {
String processWorkerGroup = processInstance.getWorkerGroup();
processWorkerGroup = StringUtils.isBlank(processWorkerGroup) ? DEFAULT_WORKER_GROUP : processWorkerGroup;
String taskWorkerGroup = StringUtils.isBlank(taskNode.getWorkerGroup()) ? processWorkerGroup : taskNode.getWorkerGroup();
Long processEnvironmentCode = Objects.isNull(processInstance.getEnvironmentCode()) ? -1 : processInstance.getEnvironmentCode();
Long taskEnvironmentCode = Objects.isNull(taskNode.getEnvironmentCode()) ? processEnvironmentCode : taskNode.getEnvironmentCode();
if (!processWorkerGroup.equals(DEFAULT_WORKER_GROUP) && taskWorkerGroup.equals(DEFAULT_WORKER_GROUP)) {
taskInstance.setWorkerGroup(processWorkerGroup);
taskInstance.setEnvironmentCode(processEnvironmentCode);
} else {
taskInstance.setWorkerGroup(taskWorkerGroup);
taskInstance.setEnvironmentCode(taskEnvironmentCode);
}
if (!taskInstance.getEnvironmentCode().equals(-1L)) {
Environment environment = processService.findEnvironmentByCode(taskInstance.getEnvironmentCode());
if (Objects.nonNull(environment) && StringUtils.isNotEmpty(environment.getConfig())) {
taskInstance.setEnvironmentConfig(environment.getConfig());
}
}
// delay execution time
taskInstance.setDelayTime(taskNode.getDelayTime());

26
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.server.worker.task;
import java.util.Arrays;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
@ -122,6 +123,11 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
@Override
protected String commandInterpreter() {
String pythonHome = getPythonHome(taskExecutionContext.getEnvFile());
if (StringUtils.isNotBlank(taskExecutionContext.getEnvironmentConfig())) {
pythonHome = getPythonHomeFromEnvironmentConfig(taskExecutionContext.getEnvironmentConfig());
}
logger.info("PYTHON_HOME={}",pythonHome);
return getPythonCommand(pythonHome);
}
@ -185,4 +191,24 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
return null;
}
/**
* get python home from the environment config
*
* @param environmentConfig env config
* @return python home
*/
public static String getPythonHomeFromEnvironmentConfig(String environmentConfig) {
String[] lines = environmentConfig.split("\n");
String pythonHomeConfig = Arrays.stream(lines).filter(line -> line.contains(Constants.PYTHON_HOME)).findFirst().get();
if (StringUtils.isEmpty(pythonHomeConfig)) {
return null;
}
String[] arrs = pythonHomeConfig.split(Constants.EQUAL_SIGN);
if (arrs.length == 2) {
return arrs[1];
}
return null;
}
}

11
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java

@ -21,13 +21,13 @@ import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.util.Strings;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Consumer;
@ -97,18 +97,25 @@ public class ShellCommandExecutor extends AbstractCommandExecutor {
if (OSUtils.isWindows()) {
sb.append("@echo off\n");
sb.append("cd /d %~dp0\n");
if (Strings.isNotBlank(taskExecutionContext.getEnvironmentConfig())) {
sb.append(taskExecutionContext.getEnvironmentConfig()).append("\n");
} else {
if (taskExecutionContext.getEnvFile() != null) {
sb.append("call ").append(taskExecutionContext.getEnvFile()).append("\n");
}
}
} else {
sb.append("#!/bin/sh\n");
sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n");
sb.append("cd $BASEDIR\n");
if (Strings.isNotBlank(taskExecutionContext.getEnvironmentConfig())) {
sb.append(taskExecutionContext.getEnvironmentConfig()).append("\n");
} else {
if (taskExecutionContext.getEnvFile() != null) {
sb.append("source ").append(taskExecutionContext.getEnvFile()).append("\n");
}
}
}
sb.append(execCommand);
logger.info("command : {}", sb.toString());

22
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java

@ -37,6 +37,28 @@ public class PythonCommandExecutorTest {
Assert.assertNotNull(pythonHome);
}
@Test
public void testGetPythonHomeFromEnvironmentConfig() {
String environmentConfig = "export HADOOP_HOME=/opt/hadoop-2.6.5\n"
+ "export HADOOP_CONF_DIR=/etc/hadoop/conf\n"
+ "export SPARK_HOME=/opt/soft/spark\n"
+ "\n"
+ "export PYTHON_HOME=/opt/soft/python\n"
+ "export JAVA_HOME=/opt/java/jdk1.8.0_181-amd64\n"
+ "export HIVE_HOME=/opt/soft/hive\n"
+ "export FLINK_HOME=/opt/soft/flink\n"
+ "export DATAX_HOME=/opt/soft/datax\n"
+ "export YARN_CONF_DIR=/etc/hadoop/conf\n"
+ "export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH\n"
+ "export HADOOP_CLASSPATH=`hadoop classpath`";
String expected = "/opt/soft/python";
String pythonHome = PythonCommandExecutor.getPythonHomeFromEnvironmentConfig(environmentConfig);
logger.info(pythonHome);
Assert.assertEquals(expected,pythonHome);
}
@Test
public void testGetPythonCommand() {
String pythonCommand = PythonCommandExecutor.getPythonCommand(null);

27
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -66,6 +66,7 @@ import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.CycleDependency;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.ErrorCommand;
import org.apache.dolphinscheduler.dao.entity.ProcessData;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
@ -86,6 +87,7 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper;
import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
@ -207,6 +209,9 @@ public class ProcessService {
@Autowired
private ProcessTaskRelationLogMapper processTaskRelationLogMapper;
@Autowired
private EnvironmentMapper environmentMapper;
/**
* handle Command (construct ProcessInstance from Command) , wrapped in transaction
*
@ -495,7 +500,6 @@ public class ProcessService {
public void recurseFindSubProcessId(int parentId, List<Integer> ids) {
List<TaskDefinition> taskNodeList = this.getTaskNodeListByDefinitionId(parentId);
if (taskNodeList != null && !taskNodeList.isEmpty()) {
for (TaskDefinition taskNode : taskNodeList) {
@ -546,6 +550,7 @@ public class ProcessService {
processInstance.getWarningGroupId(),
processInstance.getScheduleTime(),
processInstance.getWorkerGroup(),
processInstance.getEnvironmentCode(),
processInstance.getProcessInstancePriority()
);
saveCommand(command);
@ -637,6 +642,7 @@ public class ProcessService {
processInstance.setProcessInstancePriority(command.getProcessInstancePriority());
String workerGroup = StringUtils.isBlank(command.getWorkerGroup()) ? Constants.DEFAULT_WORKER_GROUP : command.getWorkerGroup();
processInstance.setWorkerGroup(workerGroup);
processInstance.setEnvironmentCode(Objects.isNull(command.getEnvironmentCode()) ? -1 : command.getEnvironmentCode());
processInstance.setTimeout(processDefinition.getTimeout());
processInstance.setTenantId(processDefinition.getTenantId());
return processInstance;
@ -694,6 +700,21 @@ public class ProcessService {
return tenant;
}
/**
* get an environment
* use the code of the environment to find a environment.
*
* @param environmentCode environmentCode
* @return Environment
*/
public Environment findEnvironmentByCode(Long environmentCode) {
Environment environment = null;
if (environmentCode >= 0) {
environment = environmentMapper.queryByEnvironmentCode(environmentCode);
}
return environment;
}
/**
* check command parameters is valid
*
@ -1224,6 +1245,7 @@ public class ProcessService {
parentProcessInstance.getWarningGroupId(),
parentProcessInstance.getScheduleTime(),
task.getWorkerGroup(),
task.getEnvironmentCode(),
parentProcessInstance.getProcessInstancePriority()
);
}
@ -1680,7 +1702,6 @@ public class ProcessService {
taskInstance.setTaskParams(JSONUtils.toJsonString(taskParams));
}
/**
* convert integer list to string list
*
@ -2222,6 +2243,7 @@ public class ProcessService {
taskDefinition.setFlag(taskNode.isForbidden() ? Flag.NO : Flag.YES);
taskDefinition.setTaskPriority(taskNode.getTaskInstancePriority());
taskDefinition.setWorkerGroup(taskNode.getWorkerGroup());
taskDefinition.setEnvironmentCode(Objects.isNull(taskNode.getEnvironmentCode()) ? -1 : taskNode.getEnvironmentCode());
taskDefinition.setFailRetryTimes(taskNode.getMaxRetryTimes());
taskDefinition.setFailRetryInterval(taskNode.getRetryInterval());
taskDefinition.setTimeoutFlag(taskNode.getTaskTimeoutParameter().getEnable() ? TimeoutFlag.OPEN : TimeoutFlag.CLOSE);
@ -2517,6 +2539,7 @@ public class ProcessService {
v.setParams(JSONUtils.toJsonString(taskParamsMap));
v.setTaskInstancePriority(taskDefinitionLog.getTaskPriority());
v.setWorkerGroup(taskDefinitionLog.getWorkerGroup());
v.setEnvironmentCode(taskDefinitionLog.getEnvironmentCode());
v.setTimeout(JSONUtils.toJsonString(new TaskTimeoutParameter(taskDefinitionLog.getTimeoutFlag() == TimeoutFlag.OPEN,
taskDefinitionLog.getTimeoutNotifyStrategy(),
taskDefinitionLog.getTimeout())));

2
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java

@ -429,7 +429,7 @@ public class ProcessServiceTest {
+ "\"retryInterval\":0,\"params\":{},\"preTasks\":[\"unit-test\"],"
+ "\"preTaskNodeList\":[{\"code\":2,\"name\":\"unit-test\",\"version\":0}],"
+ "\"extras\":null,\"depList\":[\"unit-test\"],\"dependence\":null,\"conditionResult\":null,"
+ "\"switchResult\":null,\"taskInstancePriority\":null,\"workerGroup\":null,"
+ "\"switchResult\":null,\"taskInstancePriority\":null,\"workerGroup\":null,\"environmentCode\":null,"
+ "\"timeout\":{\"enable\":false,\"strategy\":null,\"interval\":0},\"delayTime\":0}],"
+ "\"globalParams\":[],\"timeout\":0,\"tenantId\":0}";

10
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue

@ -159,7 +159,15 @@
:visible.sync="nodeDrawer"
size=""
:with-header="false">
<m-form-model v-if="nodeDrawer" :nodeData=nodeData @seeHistory="seeHistory" @addTaskInfo="addTaskInfo" @cacheTaskInfo="cacheTaskInfo" @close="close" @onSubProcess="onSubProcess"></m-form-model>
<m-form-model
v-if="nodeDrawer"
:nodeData=nodeData
@seeHistory="seeHistory"
@addTaskInfo="addTaskInfo"
@cacheTaskInfo="cacheTaskInfo"
@close="close"
@onSubProcess="onSubProcess">
</m-form-model>
</el-drawer>
<el-drawer
:visible.sync="lineDrawer"

120
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/relatedEnvironment.vue

@ -0,0 +1,120 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<template>
<el-select
:disabled="isDetails"
clearable
@change="_onChange"
v-model="selectedValue"
size="small"
style="width: 180px">
<el-option
v-for="item in environmentOptions"
:key="item.code"
:value="item.code"
:label="item.name">
</el-option>
</el-select>
</template>
<script>
import disabledState from '@/module/mixin/disabledState'
export default {
name: 'form-related-environment',
data () {
return {
selectedValue: '',
selectedWorkerGroup: this.workerGroup,
environmentOptions: [],
environmentList: []
}
},
mixins: [disabledState],
props: {
value: {
type: String
},
workerGroup: {
type: String
}
},
model: {
prop: 'value',
event: 'environmentCodeEvent'
},
methods: {
_onChange (o) {
this.$emit('environmentCodeEvent', o)
},
_getEnvironmentAll () {
return new Promise((resolve, reject) => {
this.store.dispatch('security/getEnvironmentAll').then(res => {
resolve(res)
}).catch(e => {
reject(e)
})
})
},
_initEnvironmentOptions (workerGroup) {
this.environmentOptions = []
if (this.environmentList && workerGroup) {
this.environmentList.forEach(item => {
if (item.workerGroups && item.workerGroups.length > 0) {
if (item.workerGroups.indexOf(workerGroup) >= 0) {
this.environmentOptions.push({ code: item.code, name: item.name })
if (item.code === this.value) {
this.selectedValue = this.value
}
}
}
})
}
if (this.environmentOptions.length > 0) {
/// default to select this environment when only have one environment
if (this.environmentOptions.length === 1 && this.selectedValue === '') {
this.selectedValue = this.environmentOptions[0].code
this.$emit('environmentCodeEvent', this.selectedValue)
}
} else {
this.selectedValue = ''
this.$emit('environmentCodeEvent', this.selectedValue)
}
}
},
watch: {
value: function (val) {
},
workerGroup: function (val) {
this.selectedWorkerGroup = val
this._initEnvironmentOptions(this.selectedWorkerGroup)
}
},
created () {
let stateEnvironmentList = this.store.state.security.environmentListAll || []
if (stateEnvironmentList.length && stateEnvironmentList.length > 0) {
this.environmentList = stateEnvironmentList
this._initEnvironmentOptions(this.workerGroup)
} else {
this._getEnvironmentAll().then(res => {
this.environmentList = res
this._initEnvironmentOptions(this.workerGroup)
})
}
}
}
</script>

31
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue

@ -77,8 +77,18 @@
<span class="label-box" style="width: 193px;display: inline-block;">
<m-priority v-model="taskInstancePriority"></m-priority>
</span>
<span class="text-b">{{$t('Worker group')}}</span>
</div>
</m-list-box>
<!-- Worker group and environment -->
<m-list-box>
<div slot="text">{{$t('Worker group')}}</div>
<div slot="content">
<span class="label-box" style="width: 193px;display: inline-block;">
<m-worker-groups v-model="workerGroup"></m-worker-groups>
</span>
<span class="text-b">{{$t('Environment Name')}}</span>
<m-related-environment v-model="environmentCode" :workerGroup="workerGroup" v-on:environmentCodeEvent="_onUpdateEnvironmentCode"></m-related-environment>
</div>
</m-list-box>
@ -307,6 +317,7 @@
import mTimeoutAlarm from './_source/timeoutAlarm'
import mDependentTimeout from './_source/dependentTimeout'
import mWorkerGroups from './_source/workerGroups'
import mRelatedEnvironment from './_source/relatedEnvironment'
import mPreTasks from './tasks/pre_tasks'
import clickoutside from '@/module/util/clickoutside'
import disabledState from '@/module/mixin/disabledState'
@ -363,6 +374,9 @@
taskInstancePriority: 'MEDIUM',
// worker group id
workerGroup: 'default',
// selected environment
environmentCode: '',
selectedWorkerGroup: '',
stateList: [
{
value: 'success',
@ -467,18 +481,22 @@
})
}
},
_onUpdateWorkerGroup (o) {
this.selectedWorkerGroup = o
},
/**
* return params
*/
_onParams (o) {
this.params = Object.assign({}, o)
},
_onCacheParams (o) {
this.params = Object.assign(this.params, {}, o)
this._cacheItem()
},
_onUpdateEnvironmentCode (o) {
this.environmentCode = o
},
_cacheItem () {
this.conditionResult.successNode[0] = this.successBranch
this.conditionResult.failedNode[0] = this.failedBranch
@ -501,6 +519,7 @@
waitStartTimeout: this.waitStartTimeout,
taskInstancePriority: this.taskInstancePriority,
workerGroup: this.workerGroup,
environmentCode: this.environmentCode,
status: this.status,
branch: this.branch
},
@ -626,6 +645,7 @@
waitStartTimeout: this.waitStartTimeout,
taskInstancePriority: this.taskInstancePriority,
workerGroup: this.workerGroup,
environmentCode: this.environmentCode,
status: this.status,
branch: this.branch
},
@ -726,6 +746,7 @@
break
}
}
if (o.workerGroup === undefined) {
this.store.dispatch('dag/getTaskInstanceList', {
pageSize: 10, pageNo: 1, processInstanceId: this.nodeData.instanceId, name: o.name
@ -735,7 +756,7 @@
} else {
this.workerGroup = o.workerGroup
}
this.environmentCode = o.environmentCode
this.params = o.params || {}
this.dependence = o.dependence || {}
this.cacheDependence = o.dependence || {}
@ -793,6 +814,7 @@
waitStartTimeout: this.waitStartTimeout,
taskInstancePriority: this.taskInstancePriority,
workerGroup: this.workerGroup,
environmentCode: this.environmentCode,
successBranch: this.successBranch,
failedBranch: this.failedBranch
}
@ -821,6 +843,7 @@
mDependentTimeout,
mPriority,
mWorkerGroups,
mRelatedEnvironment,
mPreTasks
}
}

16
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue

@ -76,6 +76,14 @@
<m-worker-groups v-model="workerGroup"></m-worker-groups>
</div>
</div>
<div class="clearfix list">
<div class="text">
{{$t('Environment Name')}}
</div>
<div class="cont">
<m-related-environment v-model="environmentCode" :workerGroup="workerGroup" v-on:environmentCodeEvent="_onUpdateEnvironmentCode"></m-related-environment>
</div>
</div>
<div class="clearfix list">
<div class="text">
{{$t('Alarm group')}}
@ -189,6 +197,7 @@
import { warningTypeList } from './util'
import mPriority from '@/module/components/priority/priority'
import mWorkerGroups from '@/conf/home/pages/dag/_source/formModel/_source/workerGroups'
import mRelatedEnvironment from '@/conf/home/pages/dag/_source/formModel/_source/relatedEnvironment'
import mLocalParams from '@/conf/home/pages/dag/_source/formModel/tasks/_source/localParams'
import disabledState from '@/module/mixin/disabledState'
import { mapMutations } from 'vuex'
@ -214,6 +223,7 @@
runMode: 'RUN_MODE_SERIAL',
processInstancePriority: 'MEDIUM',
workerGroup: 'default',
environmentCode: '',
// Global custom parameters
definitionGlobalParams: [],
udpList: []
@ -242,6 +252,9 @@
_datepicker (val) {
this.scheduleTime = val
},
_onUpdateEnvironmentCode (o) {
this.environmentCode = o
},
_verification () {
if (this.enableCustomParallelism && !this.parallismNumber) {
this.$message.warning(`${i18n.$t('Parallelism number should be positive integer')}`)
@ -276,6 +289,7 @@
runMode: this.runMode,
processInstancePriority: this.processInstancePriority,
workerGroup: this.workerGroup,
environmentCode: this.environmentCode,
startParams: !_.isEmpty(startParams) ? JSON.stringify(startParams) : '',
expectedParallelismNumber: this.parallismNumber
}
@ -362,7 +376,7 @@
this.workflowName = this.startData.name
},
computed: {},
components: { mPriority, mWorkerGroups, mLocalParams }
components: { mPriority, mWorkerGroups, mLocalParams, mRelatedEnvironment }
}
</script>

18
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue

@ -128,6 +128,14 @@
<m-worker-groups v-model="workerGroup"></m-worker-groups>
</div>
</div>
<div class="clearfix list">
<div class="text">
{{$t('Environment Name')}}
</div>
<div class="cont">
<m-related-environment v-model="environmentCode" :workerGroup="workerGroup" v-on:environmentCodeEvent="_onUpdateEnvironmentCode"></m-related-environment>
</div>
</div>
<div class="clearfix list">
<div class="text">
{{$t('Alarm group')}}
@ -167,6 +175,7 @@
import { formatDate } from '@/module/filter/filter'
import mPriority from '@/module/components/priority/priority'
import mWorkerGroups from '@/conf/home/pages/dag/_source/formModel/_source/workerGroups'
import mRelatedEnvironment from '@/conf/home/pages/dag/_source/formModel/_source/relatedEnvironment'
export default {
name: 'timing-process',
@ -188,6 +197,7 @@
i18n: i18n.globalScope.LOCALE,
processInstancePriority: 'MEDIUM',
workerGroup: '',
environmentCode: '',
previewTimes: []
}
},
@ -215,6 +225,9 @@
}
return true
},
_onUpdateEnvironmentCode (o) {
this.environmentCode = o
},
_timing () {
if (this._verification()) {
let api = ''
@ -229,7 +242,8 @@
warningType: this.warningType,
processInstancePriority: this.processInstancePriority,
warningGroupId: this.warningGroupId === '' ? 0 : this.warningGroupId,
workerGroup: this.workerGroup
workerGroup: this.workerGroup,
environmentCode: this.environmentCode
}
let msg = ''
@ -360,7 +374,7 @@
}).catch(() => { this.warningGroupId = '' })
}
},
components: { vCrontab, mPriority, mWorkerGroups }
components: { vCrontab, mPriority, mWorkerGroups, mRelatedEnvironment }
}
</script>

226
dolphinscheduler-ui/src/js/conf/home/pages/security/pages/environment/_source/createEnvironment.vue

@ -0,0 +1,226 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<template>
<m-popover ref="popover" :ok-text="item && item.name ? $t('Edit') : $t('Submit')" @ok="_ok" @close="close">
<template slot="content">
<div class="create-environment-model">
<m-list-box-f>
<template slot="name"><strong>*</strong>{{$t('Environment Name')}}</template>
<template slot="content">
<el-input
type="input"
v-model="name"
maxlength="60"
size="mini"
:placeholder="$t('Please enter name')">
</el-input>
</template>
</m-list-box-f>
<m-list-box-f>
<template slot="name"><strong>*</strong>{{$t('Environment Config')}}</template>
<template slot="content">
<el-input
type="textarea"
:autosize="{ minRows: 10, maxRows: 20 }"
v-model="config"
:placeholder="configExample">
</el-input>
</template>
</m-list-box-f>
<m-list-box-f>
<template slot="name"><strong>*</strong>{{$t('Environment Desc')}}</template>
<template slot="content">
<el-input
type="input"
v-model="description"
maxlength="60"
size="mini"
:placeholder="$t('Please enter environment desc')">
</el-input>
</template>
</m-list-box-f>
<m-list-box-f>
<template slot="name">{{$t('Environment Worker Group')}}</template>
<template slot="content">
<el-select
v-model="workerGroups"
size="mini"
multiple
collapse-tags
style="display: block;"
:placeholder="$t('Please select worker groups')">
<el-option
v-for="item in workerGroupOptions"
:key="item.id"
:label="item.id"
:value="item.name">
</el-option>
</el-select>
</template>
</m-list-box-f>
</div>
</template>
</m-popover>
</template>
<script>
import _ from 'lodash'
import { mapActions } from 'vuex'
import i18n from '@/module/i18n'
import store from '@/conf/home/store'
import mPopover from '@/module/components/popup/popover'
import mListBoxF from '@/module/components/listBoxF/listBoxF'
export default {
name: 'create-environment',
data () {
return {
store,
workerGroups: [],
workerGroupOptions: [],
environment: '',
name: '',
config: '',
description: '',
configExample: 'export HADOOP_HOME=/opt/hadoop-2.6.5\n' +
'export HADOOP_CONF_DIR=/etc/hadoop/conf\n' +
'export SPARK_HOME=/opt/soft/spark\n' +
'export PYTHON_HOME=/opt/soft/python\n' +
'export JAVA_HOME=/opt/java/jdk1.8.0_181-amd64\n' +
'export HIVE_HOME=/opt/soft/hive\n' +
'export FLINK_HOME=/opt/soft/flink\n' +
'export DATAX_HOME=/opt/soft/datax\n' +
'export YARN_CONF_DIR=/etc/hadoop/conf\n' +
'export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH\n' +
'export HADOOP_CLASSPATH=`hadoop classpath`\n'
}
},
props: {
item: Object
},
methods: {
...mapActions('security', ['getWorkerGroupsAll']),
_getWorkerGroupList () {
this.getWorkerGroupsAll().then(res => {
this.workerGroups = res
console.log('get Worker Group List')
console.log(this.workerGroups)
})
},
_ok () {
if (!this._verification()) {
return
}
let param = {
name: _.trim(this.name),
config: _.trim(this.config),
description: _.trim(this.description),
workerGroups: JSON.stringify(this.workerGroups)
}
let $then = (res) => {
this.$emit('onUpdate')
this.$message.success(res.msg)
this.$refs.popover.spinnerLoading = false
}
let $catch = (e) => {
this.$message.error(e.msg || '')
this.$refs.popover.spinnerLoading = false
}
if (this.item && this.item.name) {
this.$refs.popover.spinnerLoading = true
let updateParam = {
code: this.item.code,
name: _.trim(this.name),
config: _.trim(this.config),
description: _.trim(this.description),
workerGroups: JSON.stringify(this.workerGroups)
}
this.store.dispatch('security/updateEnvironment', updateParam).then(res => {
$then(res)
}).catch(e => {
$catch(e)
})
} else {
this._verifyName(param).then(() => {
this.$refs.popover.spinnerLoading = true
this.store.dispatch('security/createEnvironment', param).then(res => {
$then(res)
}).catch(e => {
$catch(e)
})
}).catch(e => {
this.$message.error(e.msg || '')
})
}
},
_verification () {
if (!this.name.replace(/\s*/g, '')) {
this.$message.warning(`${i18n.$t('Please enter name')}`)
return false
}
if (!this.config.replace(/\s*/g, '')) {
this.$message.warning(`${i18n.$t('Please enter environment config')}`)
return false
}
if (!this.description.replace(/\s*/g, '')) {
this.$message.warning(`${i18n.$t('Please enter environment desc')}`)
return false
}
return true
},
_verifyName (param) {
return new Promise((resolve, reject) => {
this.store.dispatch('security/verifyEnvironment', { environmentName: param.name }).then(res => {
resolve()
}).catch(e => {
reject(e)
})
})
},
close () {
this.$emit('close')
}
},
watch: {
item: {
handler (val, oldVal) {
this.name = val.name
this.config = val.config
this.description = val.description
this.workerGroups = val.workerGroups
this.workerGroupOptions = val.workerGroupOptions
},
deep: true
}
},
created () {
if (this.item && this.item.name) {
this.name = this.item.name
this.config = this.item.config
this.description = this.item.description
this.workerGroups = this.item.workerGroups
}
this.workerGroupOptions = this.item.workerGroupOptions
},
mounted () {
},
components: { mPopover, mListBoxF }
}
</script>

116
dolphinscheduler-ui/src/js/conf/home/pages/security/pages/environment/_source/list.vue

@ -0,0 +1,116 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<template>
<div class="list-model">
<div class="table-box">
<el-table :data="list" size="mini" style="width: 100%">
<el-table-column type="index" :label="$t('#')" width="50"></el-table-column>
<el-table-column prop="name" :label="$t('Environment Name')" width="150"></el-table-column>
<el-table-column prop="config" :label="$t('Environment Config')"></el-table-column>
<el-table-column prop="description" :label="$t('Environment Desc')" min-width="50"></el-table-column>
<el-table-column :label="$t('Environment Worker Group')" min-width="50">
<template slot-scope="scope">
<span>{{ scope.row.workerGroups ? scope.row.workerGroups.join(",") : "" }}</span>
</template>
</el-table-column>
<el-table-column :label="$t('Create Time')" min-width="50">
<template slot-scope="scope">
<span>{{scope.row.createTime | formatDate}}</span>
</template>
</el-table-column>
<el-table-column :label="$t('Update Time')" min-width="50">
<template slot-scope="scope">
<span>{{scope.row.updateTime | formatDate}}</span>
</template>
</el-table-column>
<el-table-column :label="$t('Operation')" width="100">
<template slot-scope="scope">
<el-tooltip :content="$t('Edit')" placement="top">
<el-button type="primary" size="mini" icon="el-icon-edit-outline" @click="_edit(scope.row)" circle></el-button>
</el-tooltip>
<el-tooltip :content="$t('Delete')" placement="top">
<el-button type="danger" size="mini" icon="el-icon-delete" circle></el-button>
<el-popconfirm
:confirmButtonText="$t('Confirm')"
:cancelButtonText="$t('Cancel')"
icon="el-icon-info"
iconColor="red"
:title="$t('Delete?')"
@onConfirm="_delete(scope.row,scope.row.id)"
>
<el-button type="danger" size="mini" icon="el-icon-delete" circle slot="reference"></el-button>
</el-popconfirm>
</el-tooltip>
</template>
</el-table-column>
</el-table>
</div>
</div>
</template>
<script>
import { mapActions } from 'vuex'
export default {
name: 'environment-list',
data () {
return {
list: []
}
},
props: {
environmentList: Array,
pageNo: Number,
pageSize: Number
},
methods: {
...mapActions('security', ['deleteEnvironment']),
_delete (item, i) {
this.deleteEnvironment({
environmentCode: item.code
}).then(res => {
let newList = []
this.list.forEach(item => {
if (item.id !== i) {
newList.push(item)
}
})
this.list = newList
this.$message.success(res.msg)
}).catch(e => {
this.$message.error(e.msg || '')
})
},
_edit (item) {
this.$emit('on-edit', item)
}
},
watch: {
environmentList (a) {
this.list = []
setTimeout(() => {
this.list = a
})
}
},
created () {
this.list = this.environmentList
},
mounted () {
},
components: { }
}
</script>

163
dolphinscheduler-ui/src/js/conf/home/pages/security/pages/environment/index.vue

@ -0,0 +1,163 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<template>
<m-list-construction :title="$t('Environment manage')">
<template slot="conditions">
<m-conditions @on-conditions="_onConditions">
<template slot="button-group" v-if="isADMIN">
<el-button size="mini" @click="_create()">{{$t('Create environment')}}</el-button>
<el-dialog
:title="item && item.name ? $t('Edit environment') : $t('Create environment')"
:v-if="createEnvironmentDialog"
:visible.sync="createEnvironmentDialog"
width="auto">
<m-create-environment :item="item" @onUpdate="onUpdate" @close="close"></m-create-environment>
</el-dialog>
</template>
</m-conditions>
</template>
<template slot="content">
<template v-if="environmentList.length || total>0">
<m-list @on-edit="_onEdit"
:environment-list="environmentList"
:page-no="searchParams.pageNo"
:page-size="searchParams.pageSize">
</m-list>
<div class="page-box">
<el-pagination
background
@current-change="_page"
@size-change="_pageSize"
:page-size="searchParams.pageSize"
:current-page.sync="searchParams.pageNo"
:page-sizes="[10, 30, 50]"
layout="sizes, prev, pager, next, jumper"
:total="total">
</el-pagination>
</div>
</template>
<template v-if="!environmentList.length && total<=0">
<m-no-data></m-no-data>
</template>
<m-spin :is-spin="isLoading" :is-left="isLeft"></m-spin>
</template>
</m-list-construction>
</template>
<script>
import _ from 'lodash'
import { mapActions } from 'vuex'
import mList from './_source/list'
import store from '@/conf/home/store'
import mSpin from '@/module/components/spin/spin'
import mCreateEnvironment from './_source/createEnvironment'
import mNoData from '@/module/components/noData/noData'
import listUrlParamHandle from '@/module/mixin/listUrlParamHandle'
import mConditions from '@/module/components/conditions/conditions'
import mListConstruction from '@/module/components/listConstruction/listConstruction'
export default {
name: 'environment-index',
data () {
return {
total: null,
isLoading: true,
environmentList: [],
workerGroupList: [],
environmentWorkerGroupRelationList: [],
searchParams: {
pageSize: 10,
pageNo: 1,
searchVal: ''
},
isLeft: true,
isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER',
item: {},
createEnvironmentDialog: false
}
},
mixins: [listUrlParamHandle],
props: {},
methods: {
...mapActions('security', ['getEnvironmentListPaging', 'getWorkerGroupsAll']),
/**
* Query
*/
_onConditions (o) {
this.searchParams = _.assign(this.searchParams, o)
this.searchParams.pageNo = 1
},
_page (val) {
this.searchParams.pageNo = val
},
_pageSize (val) {
this.searchParams.pageSize = val
},
_onEdit (item) {
this.item = item
this.createEnvironmentDialog = true
},
_create () {
this.item = { workerGroupOptions: this.workerGroupList }
this.createEnvironmentDialog = true
},
onUpdate () {
this._debounceGET('false')
this.createEnvironmentDialog = false
},
close () {
this.createEnvironmentDialog = false
},
_getList (flag) {
if (sessionStorage.getItem('isLeft') === 0) {
this.isLeft = false
} else {
this.isLeft = true
}
this.isLoading = !flag
Promise.all([this.getEnvironmentListPaging(this.searchParams), this.getWorkerGroupsAll()]).then((values) => {
if (this.searchParams.pageNo > 1 && values[0].totalList.length === 0) {
this.searchParams.pageNo = this.searchParams.pageNo - 1
} else {
this.environmentList = []
this.environmentList = values[0].totalList
this.total = values[0].total
this.isLoading = false
}
this.workerGroupList = values[1]
this.environmentList.forEach(item => {
item.workerGroupOptions = this.workerGroupList
})
}).catch(e => {
this.isLoading = false
})
}
},
watch: {
// router
'$route' (a) {
// url no params get instance list
this.searchParams.pageNo = _.isEmpty(a.query) ? 1 : a.query.pageNo
}
},
beforeDestroy () {
sessionStorage.setItem('isLeft', 1)
},
components: { mList, mListConstruction, mConditions, mSpin, mNoData, mCreateEnvironment }
}
</script>

8
dolphinscheduler-ui/src/js/conf/home/router/index.js

@ -444,6 +444,14 @@ const router = new Router({
title: `${i18n.$t('Worker group manage')}`
}
},
{
path: '/security/environments',
name: 'environment-manage',
component: resolve => require(['../pages/security/pages/environment/index'], resolve),
meta: {
title: `${i18n.$t('Environment manage')}`
}
},
{
path: '/security/token',
name: 'token-manage',

71
dolphinscheduler-ui/src/js/conf/home/store/security/actions.js

@ -639,5 +639,76 @@ export default {
reject(e)
})
})
},
/**
* get environment list pages
*/
getEnvironmentListPaging ({ state }, payload) {
return new Promise((resolve, reject) => {
io.get('environment/list-paging', payload, res => {
resolve(res.data)
}).catch(e => {
reject(e)
})
})
},
/**
* create environment
*/
createEnvironment ({ state }, payload) {
return new Promise((resolve, reject) => {
io.post('environment/create', payload, res => {
resolve(res)
}).catch(e => {
reject(e)
})
})
},
/**
* update environment
*/
updateEnvironment ({ state }, payload) {
return new Promise((resolve, reject) => {
io.post('environment/update', payload, res => {
resolve(res)
}).catch(e => {
reject(e)
})
})
},
/**
* delete environment
*/
deleteEnvironment ({ state }, payload) {
return new Promise((resolve, reject) => {
io.post('environment/delete', payload, res => {
resolve(res)
}).catch(e => {
reject(e)
})
})
},
verifyEnvironment ({ state }, payload) {
return new Promise((resolve, reject) => {
io.post('environment/verify-environment', payload, res => {
resolve(res)
}).catch(e => {
reject(e)
})
})
},
/**
* get all environment
*/
getEnvironmentAll ({ state }, payload) {
return new Promise((resolve, reject) => {
io.get('environment/query-environment-list', payload, res => {
let list = res.data
state.environmentListAll = list
resolve(list)
}).catch(e => {
reject(e)
})
})
}
}

1
dolphinscheduler-ui/src/js/conf/home/store/security/state.js

@ -16,6 +16,7 @@
*/
export default {
workerGroupsListAll: [],
environmentListAll: [],
alarmGroupsListAll: [],
tenantAllList: []
}

9
dolphinscheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js

@ -136,6 +136,15 @@ const menu = {
icon: 'el-icon-s-grid',
children: []
},
{
name: `${i18n.$t('Environment manage')}`,
id: 3,
path: 'environment-manage',
isOpen: true,
enabled: true,
icon: 'el-icon-setting',
children: []
},
{
name: `${i18n.$t('Token manage')}`,
id: 2,

12
dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@ -696,6 +696,18 @@ export default {
Info: 'Info',
'Datasource userName': 'owner',
'Resource userName': 'owner',
'Environment manage': 'Environment manage',
'Create environment': 'Create environment',
'Edit environment': 'Edit environment',
'Environment value': 'Environment value',
'Environment Name': 'Environment Name',
'Environment Code': 'Environment Code',
'Environment Config': 'Environment Config',
'Environment Desc': 'Environment Desc',
'Environment Worker Group': 'Worker Groups',
'Please enter environment config': 'Please enter environment config',
'Please enter environment desc': 'Please enter environment desc',
'Please select worker groups': 'Please select worker groups',
condition: 'condition',
'The condition content cannot be empty': 'The condition content cannot be empty'
}

12
dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@ -695,6 +695,18 @@ export default {
Info: '提示',
'Datasource userName': '所属用户',
'Resource userName': '所属用户',
'Environment manage': '环境管理',
'Create environment': '创建环境',
'Edit environment': '编辑',
'Environment value': 'Environment value',
'Environment Name': '环境名称',
'Environment Code': '环境编码',
'Environment Config': '环境配置',
'Environment Desc': '详细描述',
'Environment Worker Group': 'Worker组',
'Please enter environment config': '请输入环境配置信息',
'Please enter environment desc': '请输入详细描述',
'Please select worker groups': '请选择Worker分组',
condition: '条件',
'The condition content cannot be empty': '条件内容不能为空'
}

3
pom.xml

@ -890,9 +890,12 @@
<include>**/api/service/UsersServiceTest.java</include>
<include>**/api/service/WorkerGroupServiceTest.java</include>
<include>**/api/service/WorkFlowLineageServiceTest.java</include>
<include>**/api/service/EnvironmentServiceTest.java</include>
<include>**/api/service/EnvironmentWorkerGroupRelationServiceTest.java</include>
<include>**/api/controller/ProcessDefinitionControllerTest.java</include>
<include>**/api/controller/TaskInstanceControllerTest.java</include>
<include>**/api/controller/WorkFlowLineageControllerTest.java</include>
<include>**/api/controller/EnvironmentControllerTest.java</include>
<include>**/api/utils/exportprocess/DataSourceParamTest.java</include>
<include>**/api/utils/exportprocess/DependentParamTest.java</include>
<include>**/api/utils/CheckUtilsTest.java</include>

43
sql/dolphinscheduler_h2.sql

@ -326,6 +326,7 @@ CREATE TABLE t_ds_command
update_time datetime DEFAULT NULL,
process_instance_priority int(11) DEFAULT NULL,
worker_group varchar(64),
environment_code bigint(20) DEFAULT NULL,
PRIMARY KEY (id)
);
@ -375,6 +376,7 @@ CREATE TABLE t_ds_error_command
update_time datetime DEFAULT NULL,
process_instance_priority int(11) DEFAULT NULL,
worker_group varchar(64),
environment_code bigint(20) DEFAULT NULL,
message text,
PRIMARY KEY (id)
);
@ -461,6 +463,7 @@ CREATE TABLE t_ds_task_definition
flag tinyint(2) DEFAULT NULL,
task_priority tinyint(4) DEFAULT NULL,
worker_group varchar(200) DEFAULT NULL,
environment_code bigint(20) DEFAULT NULL,
fail_retry_times int(11) DEFAULT NULL,
fail_retry_interval int(11) DEFAULT NULL,
timeout_flag tinyint(2) DEFAULT '0',
@ -492,6 +495,7 @@ CREATE TABLE t_ds_task_definition_log
flag tinyint(2) DEFAULT NULL,
task_priority tinyint(4) DEFAULT NULL,
worker_group varchar(200) DEFAULT NULL,
environment_code bigint(20) DEFAULT NULL,
fail_retry_times int(11) DEFAULT NULL,
fail_retry_interval int(11) DEFAULT NULL,
timeout_flag tinyint(2) DEFAULT '0',
@ -585,6 +589,7 @@ CREATE TABLE t_ds_process_instance
history_cmd text,
process_instance_priority int(11) DEFAULT NULL,
worker_group varchar(64) DEFAULT NULL,
environment_code bigint(20) DEFAULT NULL,
timeout int(11) DEFAULT '0',
tenant_id int(11) NOT NULL DEFAULT '-1',
var_pool longtext,
@ -770,6 +775,7 @@ CREATE TABLE t_ds_schedules
warning_group_id int(11) DEFAULT NULL,
process_instance_priority int(11) DEFAULT NULL,
worker_group varchar(64) DEFAULT '',
environment_code bigint(20) DEFAULT NULL,
create_time datetime NOT NULL,
update_time datetime NOT NULL,
PRIMARY KEY (id)
@ -825,6 +831,8 @@ CREATE TABLE t_ds_task_instance
max_retry_times int(2) DEFAULT NULL,
task_instance_priority int(11) DEFAULT NULL,
worker_group varchar(64) DEFAULT NULL,
environment_code bigint(20) DEFAULT NULL,
environment_config text DEFAULT '',
executor_id int(11) DEFAULT NULL,
first_submit_time datetime DEFAULT NULL,
delay_time int(4) DEFAULT '0',
@ -988,3 +996,38 @@ CREATE TABLE t_ds_alert_plugin_instance
instance_name varchar(200) DEFAULT NULL,
PRIMARY KEY (id)
);
--
-- Table structure for table t_ds_environment
--
DROP TABLE IF EXISTS t_ds_environment;
CREATE TABLE t_ds_environment
(
id int NOT NULL AUTO_INCREMENT,
code bigint(20) NOT NULL,
name varchar(100) DEFAULT NULL,
config text DEFAULT NULL,
description text,
operator int DEFAULT NULL,
create_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
update_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
UNIQUE KEY environment_name_unique (name),
UNIQUE KEY environment_code_unique (code)
);
--
-- Table structure for table t_ds_environment_worker_group_relation
--
DROP TABLE IF EXISTS t_ds_environment_worker_group_relation;
CREATE TABLE t_ds_environment_worker_group_relation
(
id int NOT NULL AUTO_INCREMENT,
environment_code bigint(20) NOT NULL,
worker_group varchar(255) NOT NULL,
operator int DEFAULT NULL,
create_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
update_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id) ,
UNIQUE KEY environment_worker_group_unique (environment_code,worker_group)
);

40
sql/dolphinscheduler_mysql.sql

@ -331,6 +331,7 @@ CREATE TABLE `t_ds_command` (
`update_time` datetime DEFAULT NULL COMMENT 'update time',
`process_instance_priority` int(11) DEFAULT NULL COMMENT 'process instance priority: 0 Highest,1 High,2 Medium,3 Low,4 Lowest',
`worker_group` varchar(64) COMMENT 'worker group',
`environment_code` bigint(20) NOT NULL COMMENT 'environment code',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
@ -378,6 +379,7 @@ CREATE TABLE `t_ds_error_command` (
`update_time` datetime DEFAULT NULL COMMENT 'update time',
`process_instance_priority` int(11) DEFAULT NULL COMMENT 'process instance priority, 0 Highest,1 High,2 Medium,3 Low,4 Lowest',
`worker_group` varchar(64) COMMENT 'worker group',
`environment_code` bigint(20) NOT NULL COMMENT 'environment code',
`message` text COMMENT 'message',
PRIMARY KEY (`id`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC;
@ -461,6 +463,7 @@ CREATE TABLE `t_ds_task_definition` (
`flag` tinyint(2) DEFAULT NULL COMMENT '0 not available, 1 available',
`task_priority` tinyint(4) DEFAULT NULL COMMENT 'job priority',
`worker_group` varchar(200) DEFAULT NULL COMMENT 'worker grouping',
`environment_code` bigint(20) NOT NULL COMMENT 'environment code',
`fail_retry_times` int(11) DEFAULT NULL COMMENT 'number of failed retries',
`fail_retry_interval` int(11) DEFAULT NULL COMMENT 'failed retry interval',
`timeout_flag` tinyint(2) DEFAULT '0' COMMENT 'timeout flag:0 close, 1 open',
@ -491,6 +494,7 @@ CREATE TABLE `t_ds_task_definition_log` (
`flag` tinyint(2) DEFAULT NULL COMMENT '0 not available, 1 available',
`task_priority` tinyint(4) DEFAULT NULL COMMENT 'job priority',
`worker_group` varchar(200) DEFAULT NULL COMMENT 'worker grouping',
`environment_code` bigint(20) NOT NULL COMMENT 'environment code',
`fail_retry_times` int(11) DEFAULT NULL COMMENT 'number of failed retries',
`fail_retry_interval` int(11) DEFAULT NULL COMMENT 'failed retry interval',
`timeout_flag` tinyint(2) DEFAULT '0' COMMENT 'timeout flag:0 close, 1 open',
@ -760,6 +764,7 @@ CREATE TABLE `t_ds_schedules` (
`warning_group_id` int(11) DEFAULT NULL COMMENT 'alert group id',
`process_instance_priority` int(11) DEFAULT NULL COMMENT 'process instance priority:0 Highest,1 High,2 Medium,3 Low,4 Lowest',
`worker_group` varchar(64) DEFAULT '' COMMENT 'worker group id',
`environment_code` bigint(20) NOT NULL COMMENT 'environment code',
`create_time` datetime NOT NULL COMMENT 'create time',
`update_time` datetime NOT NULL COMMENT 'update time',
PRIMARY KEY (`id`)
@ -813,6 +818,8 @@ CREATE TABLE `t_ds_task_instance` (
`max_retry_times` int(2) DEFAULT NULL COMMENT 'max retry times',
`task_instance_priority` int(11) DEFAULT NULL COMMENT 'task instance priority:0 Highest,1 High,2 Medium,3 Low,4 Lowest',
`worker_group` varchar(64) DEFAULT NULL COMMENT 'worker group id',
`environment_code` bigint(20) NOT NULL COMMENT 'environment code',
`environment_config` text DEFAULT '' COMMENT 'this config contains many environment variables config',
`executor_id` int(11) DEFAULT NULL,
`first_submit_time` datetime DEFAULT NULL COMMENT 'task first submit time',
`delay_time` int(4) DEFAULT '0' COMMENT 'task delay execution time',
@ -968,3 +975,36 @@ CREATE TABLE `t_ds_alert_plugin_instance` (
`instance_name` varchar(200) DEFAULT NULL COMMENT 'alert instance name',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- ----------------------------
-- Table structure for t_ds_environment
-- ----------------------------
DROP TABLE IF EXISTS `t_ds_environment`;
CREATE TABLE `t_ds_environment` (
`id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'id',
`code` bigint(20) DEFAULT NULL COMMENT 'encoding',
`name` varchar(100) NOT NULL COMMENT 'environment name',
`config` text NULL DEFAULT NULL COMMENT 'this config contains many environment variables config',
`description` text NULL DEFAULT NULL COMMENT 'the details',
`operator` int(11) DEFAULT NULL COMMENT 'operator user id',
`create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
`update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `environment_name_unique` (`name`),
UNIQUE KEY `environment_code_unique` (`code`),
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
-- ----------------------------
-- Table structure for t_ds_environment_worker_group_relation
-- ----------------------------
DROP TABLE IF EXISTS `t_ds_environment_worker_group_relation`;
CREATE TABLE `t_ds_environment_worker_group_relation` (
`id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'id',
`environment_code` bigint(20) NOT NULL COMMENT 'environment code',
`worker_group` varchar(255) NOT NULL COMMENT 'worker group id',
`operator` int(11) DEFAULT NULL COMMENT 'operator user id',
`create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
`update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `environment_worker_group_unique` (`environment_code`,`worker_group`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

41
sql/dolphinscheduler_postgre.sql

@ -232,6 +232,7 @@ CREATE TABLE t_ds_command (
update_time timestamp DEFAULT NULL ,
process_instance_priority int DEFAULT NULL ,
worker_group varchar(64),
environment_code bigint DEFAULT NULL,
PRIMARY KEY (id)
) ;
@ -273,6 +274,7 @@ CREATE TABLE t_ds_error_command (
update_time timestamp DEFAULT NULL ,
process_instance_priority int DEFAULT NULL ,
worker_group varchar(64),
environment_code bigint DEFAULT NULL,
message text ,
PRIMARY KEY (id)
);
@ -348,6 +350,7 @@ CREATE TABLE t_ds_task_definition (
flag int DEFAULT NULL ,
task_priority int DEFAULT NULL ,
worker_group varchar(255) DEFAULT NULL ,
environment_code bigint DEFAULT NULL,
fail_retry_times int DEFAULT NULL ,
fail_retry_interval int DEFAULT NULL ,
timeout_flag int DEFAULT NULL ,
@ -377,6 +380,7 @@ CREATE TABLE t_ds_task_definition_log (
flag int DEFAULT NULL ,
task_priority int DEFAULT NULL ,
worker_group varchar(255) DEFAULT NULL ,
environment_code bigint DEFAULT NULL,
fail_retry_times int DEFAULT NULL ,
fail_retry_interval int DEFAULT NULL ,
timeout_flag int DEFAULT NULL ,
@ -464,6 +468,7 @@ CREATE TABLE t_ds_process_instance (
dependence_schedule_times text ,
process_instance_priority int DEFAULT NULL ,
worker_group varchar(64) ,
environment_code bigint DEFAULT NULL,
timeout int DEFAULT '0' ,
tenant_id int NOT NULL DEFAULT '-1' ,
var_pool text ,
@ -624,6 +629,7 @@ CREATE TABLE t_ds_schedules (
warning_group_id int DEFAULT NULL ,
process_instance_priority int DEFAULT NULL ,
worker_group varchar(64),
environment_code bigint DEFAULT NULL,
create_time timestamp NOT NULL ,
update_time timestamp NOT NULL ,
PRIMARY KEY (id)
@ -671,6 +677,8 @@ CREATE TABLE t_ds_task_instance (
max_retry_times int DEFAULT NULL ,
task_instance_priority int DEFAULT NULL ,
worker_group varchar(64),
environment_code bigint DEFAULT NULL,
environment_config text DEFAULT '',
executor_id int DEFAULT NULL ,
first_submit_time timestamp DEFAULT NULL ,
delay_time int DEFAULT '0' ,
@ -914,3 +922,36 @@ CREATE TABLE t_ds_alert_plugin_instance (
instance_name varchar(200) NULL,
CONSTRAINT t_ds_alert_plugin_instance_pk PRIMARY KEY (id)
);
--
-- Table structure for table t_ds_environment
--
DROP TABLE IF EXISTS t_ds_environment;
CREATE TABLE t_ds_environment (
id serial NOT NULL,
code bigint NOT NULL,
name varchar(100) DEFAULT NULL,
config text DEFAULT NULL,
description text,
operator int DEFAULT NULL,
create_time timestamp DEFAULT NULL,
update_time timestamp DEFAULT NULL,
PRIMARY KEY (id),
CONSTRAINT environment_name_unique UNIQUE (name),
CONSTRAINT environment_code_unique UNIQUE (code)
);
--
-- Table structure for table t_ds_environment_worker_group_relation
--
DROP TABLE IF EXISTS t_ds_environment_worker_group_relation;
CREATE TABLE t_ds_environment_worker_group_relation (
id serial NOT NULL,
environment_code bigint NOT NULL,
worker_group varchar(255) NOT NULL,
operator int DEFAULT NULL,
create_time timestamp DEFAULT NULL,
update_time timestamp DEFAULT NULL,
PRIMARY KEY (id) ,
CONSTRAINT environment_worker_group_unique UNIQUE (environment_code,worker_group)
);

44
sql/upgrade/1.4.0_schema/mysql/dolphinscheduler_ddl.sql

@ -632,6 +632,50 @@ d//
delimiter ;
CALL uc_dolphin_T_t_ds_schedules_A_add_timezone();
DROP PROCEDURE uc_dolphin_T_t_ds_schedules_A_add_timezone;
-- ----------------------------
-- Table structure for t_ds_environment
-- ----------------------------
DROP TABLE IF EXISTS `t_ds_environment`;
CREATE TABLE `t_ds_environment` (
`id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'id',
`code` bigint(20) DEFAULT NULL COMMENT 'encoding',
`name` varchar(100) NOT NULL COMMENT 'environment config name',
`config` text NULL DEFAULT NULL COMMENT 'this config contains many environment variables config',
`description` text NULL DEFAULT NULL COMMENT 'the details',
`operator` int(11) DEFAULT NULL COMMENT 'operator user id',
`create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
`update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `environment_name_unique` (`name`),
UNIQUE KEY `environment_code_unique` (`code`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
ALTER TABLE t_ds_task_definition ADD COLUMN `environment_code` bigint(20) default NULL COMMENT 'environment code' AFTER `worker_group`;
ALTER TABLE t_ds_task_definition_log ADD COLUMN `environment_code` bigint(20) default NULL COMMENT 'environment code' AFTER `worker_group`;
ALTER TABLE t_ds_command ADD COLUMN `environment_code` bigint(20) default NULL COMMENT 'environment code' AFTER `worker_group`;
ALTER TABLE t_ds_error_command ADD COLUMN `environment_code` bigint(20) default NULL COMMENT 'environment code' AFTER `worker_group`;
ALTER TABLE t_ds_schedules ADD COLUMN `environment_code` bigint(20) default NULL COMMENT 'environment code' AFTER `worker_group`;
ALTER TABLE t_ds_process_instance ADD COLUMN `environment_code` bigint(20) default NULL COMMENT 'environment code' AFTER `worker_group`;
ALTER TABLE t_ds_task_instance ADD COLUMN `environment_code` bigint(20) default NULL COMMENT 'environment code' AFTER `worker_group`;
ALTER TABLE t_ds_task_instance ADD COLUMN `environment_config` text default '' COMMENT 'environment config' AFTER `environment_code`;
-- ----------------------------
-- Table structure for t_ds_environment_worker_group_relation
-- ----------------------------
DROP TABLE IF EXISTS `t_ds_environment_worker_group_relation`;
CREATE TABLE `t_ds_environment_worker_group_relation` (
`id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'id',
`environment_code` bigint(20) NOT NULL COMMENT 'environment code',
`worker_group` varchar(255) NOT NULL COMMENT 'worker group id',
`operator` int(11) DEFAULT NULL COMMENT 'operator user id',
`create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
`update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `environment_worker_group_unique` (`environment_code`,`worker_group`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
-- ----------------------------
-- These columns will not be used in the new version,if you determine that the historical data is useless, you can delete it using the sql below
-- ----------------------------

58
sql/upgrade/1.4.0_schema/postgresql/dolphinscheduler_ddl.sql

@ -658,6 +658,64 @@ d//
delimiter ;
SELECT uc_dolphin_T_t_ds_schedules_A_add_timezone();
DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_schedules_A_add_timezone();
--
-- Table structure for table t_ds_environment
--
DROP TABLE IF EXISTS t_ds_environment;
CREATE TABLE t_ds_environment (
id serial NOT NULL ,
code bigint NOT NULL,
name varchar(100) DEFAULT NULL ,
config text DEFAULT NULL ,
description text ,
operator int DEFAULT NULL ,
create_time timestamp DEFAULT NULL ,
update_time timestamp DEFAULT NULL ,
PRIMARY KEY (id) ,
CONSTRAINT environment_name_unique UNIQUE (name),
CONSTRAINT environment_code_unique UNIQUE (code)
);
ALTER TABLE t_ds_task_definition ADD COLUMN environment_code bigint DEFAULT NULL;
comment on column t_ds_task_definition.environment_code is 'environment code';
ALTER TABLE t_ds_task_definition_log ADD COLUMN environment_code bigint DEFAULT NULL;
comment on column t_ds_task_definition_log.environment_code is 'environment code';
ALTER TABLE t_ds_command ADD COLUMN environment_code bigint DEFAULT NULL;
comment on column t_ds_command.environment_code is 'environment code';
ALTER TABLE t_ds_error_command ADD COLUMN environment_code bigint DEFAULT NULL;
comment on column t_ds_error_command.environment_code is 'environment code';
ALTER TABLE t_ds_schedules ADD COLUMN environment_code bigint DEFAULT NULL;
comment on column t_ds_schedules.environment_code is 'environment code';
ALTER TABLE t_ds_process_instance ADD COLUMN environment_code bigint DEFAULT NULL;
comment on column t_ds_process_instance.environment_code is 'environment code';
ALTER TABLE t_ds_task_instance ADD COLUMN environment_code bigint DEFAULT NULL;
comment on column t_ds_task_instance.environment_code is 'environment code';
ALTER TABLE t_ds_task_instance ADD COLUMN environment_config text DEFAULT '';
comment on column t_ds_task_instance.environment_config is 'environment config';
--
-- Table structure for table t_ds_environment_worker_group_relation
--
DROP TABLE IF EXISTS t_ds_environment_worker_group_relation;
CREATE TABLE t_ds_environment_worker_group_relation (
id serial NOT NULL,
environment_code bigint NOT NULL,
worker_group varchar(255) NOT NULL,
operator int DEFAULT NULL,
create_time timestamp DEFAULT NULL,
update_time timestamp DEFAULT NULL,
PRIMARY KEY (id) ,
CONSTRAINT environment_worker_group_unique UNIQUE (environment_code,worker_group)
);
-- ----------------------------
-- These columns will not be used in the new version,if you determine that the historical data is useless, you can delete it using the sql below
-- ----------------------------

Loading…
Cancel
Save