Browse Source

[Feature-10495][Resource Center] Resource Center Refactor (#12076)

* resource center refactor - S3 services connection

Co-authored-by: caishunfeng <caishunfeng2021@gmail.com>
labbomb
HanayoZz 2 years ago committed by GitHub
parent
commit
489e7fe4e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 151
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
  2. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java
  3. 43
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
  4. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java
  5. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java
  6. 44
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
  7. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java
  8. 1507
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
  9. 55
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
  10. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
  11. 63
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
  12. 12
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java
  13. 58
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java
  14. 582
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
  15. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
  16. 34
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
  17. 56
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesTask.java
  18. 1
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
  19. 45
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.java
  20. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.java
  21. 11
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java
  22. 30
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java
  23. 4
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml
  24. 58
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.xml
  25. 4
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml
  26. 14
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml
  27. 18
      dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
  28. 17
      dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
  29. 12
      dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql
  30. 27
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
  31. 101
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
  32. 66
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageEntity.java
  33. 34
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageOperate.java
  34. 245
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtils.java
  35. 33
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/OssOperator.java
  36. 252
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java
  37. 21
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
  38. 4
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java
  39. 4
      dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java
  40. 5
      dolphinscheduler-ui/src/locales/en_US/resource.ts
  41. 10
      dolphinscheduler-ui/src/router/modules/resources.ts
  42. 64
      dolphinscheduler-ui/src/service/modules/resources/index.ts
  43. 12
      dolphinscheduler-ui/src/service/modules/resources/types.ts
  44. 2
      dolphinscheduler-ui/src/utils/tree-format.ts
  45. 4
      dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-resources.ts
  46. 11
      dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
  47. 12
      dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
  48. 1
      dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-modal.ts
  49. 4
      dolphinscheduler-ui/src/views/resource/file/create/index.tsx
  50. 7
      dolphinscheduler-ui/src/views/resource/file/create/use-create.ts
  51. 8
      dolphinscheduler-ui/src/views/resource/file/edit/index.tsx
  52. 15
      dolphinscheduler-ui/src/views/resource/file/edit/use-edit.ts
  53. 104
      dolphinscheduler-ui/src/views/resource/file/index.tsx
  54. 20
      dolphinscheduler-ui/src/views/resource/file/rename/index.tsx
  55. 11
      dolphinscheduler-ui/src/views/resource/file/rename/use-form.ts
  56. 8
      dolphinscheduler-ui/src/views/resource/file/rename/use-rename.ts
  57. 34
      dolphinscheduler-ui/src/views/resource/file/table/table-action.tsx
  58. 19
      dolphinscheduler-ui/src/views/resource/file/table/use-table.ts
  59. 7
      dolphinscheduler-ui/src/views/resource/file/types.ts
  60. 6
      dolphinscheduler-ui/src/views/resource/file/upload/use-upload.ts
  61. 14
      dolphinscheduler-ui/src/views/resource/file/use-file.ts
  62. 13
      dolphinscheduler-ui/src/views/resource/udf/function/components/function-modal.tsx
  63. 14
      dolphinscheduler-ui/src/views/resource/udf/function/components/use-form.ts
  64. 9
      dolphinscheduler-ui/src/views/resource/udf/function/components/use-modal.ts
  65. 1
      dolphinscheduler-ui/src/views/resource/udf/function/index.tsx
  66. 1
      dolphinscheduler-ui/src/views/resource/udf/function/types.ts
  67. 9
      dolphinscheduler-ui/src/views/resource/udf/function/use-table.ts
  68. 8
      dolphinscheduler-ui/src/views/resource/udf/resource/components/folder-modal.tsx
  69. 17
      dolphinscheduler-ui/src/views/resource/udf/resource/components/use-modal.ts
  70. 6
      dolphinscheduler-ui/src/views/resource/udf/resource/index.tsx
  71. 2
      dolphinscheduler-ui/src/views/resource/udf/resource/types.ts
  72. 74
      dolphinscheduler-ui/src/views/resource/udf/resource/use-table.ts
  73. 12
      dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java

151
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java

@ -55,6 +55,7 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
import java.util.Map;
import org.slf4j.Logger;
@ -139,7 +140,6 @@ public class ResourcesController extends BaseController {
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)),
@Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class)),
@Parameter(name = "pid", description = "RESOURCE_PID", required = true, schema = @Schema(implementation = int.class, example = "10")),
@Parameter(name = "currentDir", description = "RESOURCE_CURRENT_DIR", required = true, schema = @Schema(implementation = String.class))
})
@PostMapping()
@ -150,10 +150,9 @@ public class ResourcesController extends BaseController {
@RequestParam(value = "name") String alias,
@RequestParam(value = "description", required = false) String description,
@RequestParam("file") MultipartFile file,
@RequestParam(value = "pid") int pid,
@RequestParam(value = "currentDir") String currentDir) {
// todo verify the file name
return resourceService.createResource(loginUser, alias, description, type, file, pid, currentDir);
return resourceService.createResource(loginUser, alias, description, type, file, currentDir);
}
/**
@ -161,7 +160,6 @@ public class ResourcesController extends BaseController {
*
* @param loginUser login user
* @param alias alias
* @param resourceId resource id
* @param type resource type
* @param description description
* @param file resource file
@ -169,23 +167,24 @@ public class ResourcesController extends BaseController {
*/
@Operation(summary = "updateResource", description = "UPDATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)),
@Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class))
})
@PutMapping(value = "/{id}")
@PutMapping()
@ApiException(UPDATE_RESOURCE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result<Object> updateResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@PathVariable(value = "id") int resourceId,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode", required = false) String tenantCode,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String alias,
@RequestParam(value = "description", required = false) String description,
@RequestParam(value = "file", required = false) MultipartFile file) {
// todo verify the resource name
return resourceService.updateResource(loginUser, resourceId, alias, description, type, file);
return resourceService.updateResource(loginUser, fullName, tenantCode, alias, description, type, file);
}
/**
@ -197,15 +196,17 @@ public class ResourcesController extends BaseController {
*/
@Operation(summary = "queryResourceList", description = "QUERY_RESOURCE_LIST_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class))
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class))
})
@GetMapping(value = "/list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result<Object> queryResourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type) {
Map<String, Object> result = resourceService.queryResourceList(loginUser, type);
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "fullName") String fullName) {
Map<String, Object> result = resourceService.queryResourceList(loginUser, type, fullName);
return returnDataList(result);
}
@ -222,7 +223,7 @@ public class ResourcesController extends BaseController {
@Operation(summary = "queryResourceListPaging", description = "QUERY_RESOURCE_LIST_PAGING_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "10")),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "bucket_name/tenant_name/type/ds")),
@Parameter(name = "searchVal", description = "SEARCH_VAL", schema = @Schema(implementation = String.class)),
@Parameter(name = "pageNo", description = "PAGE_NO", required = true, schema = @Schema(implementation = int.class, example = "1")),
@Parameter(name = "pageSize", description = "PAGE_SIZE", required = true, schema = @Schema(implementation = int.class, example = "20"))
@ -232,8 +233,9 @@ public class ResourcesController extends BaseController {
@ApiException(QUERY_RESOURCES_LIST_PAGING)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result<Object> queryResourceListPaging(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode") String tenantCode,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "id") int id,
@RequestParam("pageNo") Integer pageNo,
@RequestParam(value = "searchVal", required = false) String searchVal,
@RequestParam("pageSize") Integer pageSize) {
@ -243,7 +245,8 @@ public class ResourcesController extends BaseController {
}
searchVal = ParameterUtils.handleEscapes(searchVal);
result = resourceService.queryResourceListPaging(loginUser, id, type, searchVal, pageNo, pageSize);
result = resourceService.queryResourceListPaging(loginUser, fullName, tenantCode, type, searchVal, pageNo,
pageSize);
return result;
}
@ -251,20 +254,20 @@ public class ResourcesController extends BaseController {
* delete resource
*
* @param loginUser login user
* @param resourceId resource id
* @return delete result code
*/
@Operation(summary = "deleteResource", description = "DELETE_RESOURCE_BY_ID_NOTES")
@Parameters({
@Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100"))
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/"))
})
@DeleteMapping(value = "/{id}")
@DeleteMapping()
@ResponseStatus(HttpStatus.OK)
@ApiException(DELETE_RESOURCE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result<Object> deleteResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@PathVariable(value = "id") int resourceId) throws Exception {
return resourceService.delete(loginUser, resourceId);
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode", required = false) String tenantCode) throws Exception {
return resourceService.delete(loginUser, fullName, tenantCode);
}
/**
@ -312,55 +315,56 @@ public class ResourcesController extends BaseController {
}
/**
* query resource by full name and type
* query resource by file name and type
*
* @param loginUser login user
* @param fullName resource full name
* @param fileName resource full name
* @param tenantCode tenantcode of the owner of the resource
* @param type resource type
* @param id resource id
* @return true if the resource name not exists, otherwise return false
*/
@Operation(summary = "queryResource", description = "QUERY_BY_RESOURCE_NAME")
@Operation(summary = "queryResourceByFileName", description = "QUERY_BY_RESOURCE_FILE_NAME")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "id", description = "RESOURCE_ID", required = false, schema = @Schema(implementation = int.class, example = "10"))
@Parameter(name = "fileName", description = "RESOURCE_FILE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
})
@GetMapping(value = "/{id}")
@GetMapping(value = "/query-file-name")
@ResponseStatus(HttpStatus.OK)
@ApiException(RESOURCE_NOT_EXIST)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result<Object> queryResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName", required = false) String fullName,
@PathVariable(value = "id", required = false) Integer id,
@RequestParam(value = "type") ResourceType type) {
public Result<Object> queryResourceByFileName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fileName", required = false) String fileName,
@RequestParam(value = "tenantCode", required = false) String tenantCode,
@RequestParam(value = "type") ResourceType type) {
return resourceService.queryResource(loginUser, fullName, id, type);
return resourceService.queryResourceByFileName(loginUser, fileName, type, tenantCode);
}
/**
* view resource file online
*
* @param loginUser login user
* @param resourceId resource id
* @param skipLineNum skip line number
* @param limit limit
* @return resource content
*/
@Operation(summary = "viewResource", description = "VIEW_RESOURCE_BY_ID_NOTES")
@Parameters({
@Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")),
@Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class, example = "tenant/1.png")),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "skipLineNum", description = "SKIP_LINE_NUM", required = true, schema = @Schema(implementation = int.class, example = "100")),
@Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100"))
})
@GetMapping(value = "/{id}/view")
@GetMapping(value = "/view")
@ApiException(VIEW_RESOURCE_FILE_ON_LINE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result viewResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@PathVariable(value = "id") int resourceId,
@RequestParam(value = "skipLineNum") int skipLineNum,
@RequestParam(value = "limit") int limit) {
return resourceService.readResource(loginUser, resourceId, skipLineNum, limit);
@RequestParam(value = "limit") int limit,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode") String tenantCode) {
return resourceService.readResource(loginUser, fullName, tenantCode, skipLineNum, limit);
}
/**
@ -375,7 +379,6 @@ public class ResourcesController extends BaseController {
@Parameter(name = "suffix", description = "SUFFIX", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)),
@Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "pid", description = "RESOURCE_PID", required = true, schema = @Schema(implementation = int.class, example = "10")),
@Parameter(name = "currentDir", description = "RESOURCE_CURRENTDIR", required = true, schema = @Schema(implementation = String.class))
})
@PostMapping(value = "/online-create")
@ -387,13 +390,12 @@ public class ResourcesController extends BaseController {
@RequestParam(value = "suffix") String fileSuffix,
@RequestParam(value = "description", required = false) String description,
@RequestParam(value = "content") String content,
@RequestParam(value = "pid") int pid,
@RequestParam(value = "currentDir") String currentDir) {
if (StringUtils.isEmpty(content)) {
logger.error("resource file contents are not allowed to be empty");
return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg());
}
return resourceService.onlineCreateResource(loginUser, type, fileName, fileSuffix, description, content, pid,
return resourceService.onlineCreateResource(loginUser, type, fileName, fileSuffix, description, content,
currentDir);
}
@ -401,46 +403,46 @@ public class ResourcesController extends BaseController {
* edit resource file online
*
* @param loginUser login user
* @param resourceId resource id
* @param content content
* @return update result code
*/
@Operation(summary = "updateResourceContent", description = "UPDATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")),
@Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class))
@Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "fullName", description = "FULL_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class))
})
@PutMapping(value = "/{id}/update-content")
@PutMapping(value = "/update-content")
@ApiException(EDIT_RESOURCE_FILE_ON_LINE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result updateResourceContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@PathVariable(value = "id") int resourceId,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode") String tenantCode,
@RequestParam(value = "content") String content) {
if (StringUtils.isEmpty(content)) {
logger.error("The resource file contents are not allowed to be empty");
return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg());
}
return resourceService.updateResourceContent(loginUser, resourceId, content);
return resourceService.updateResourceContent(loginUser, fullName, tenantCode, content);
}
/**
* download resource file
*
* @param loginUser login user
* @param resourceId resource id
* @return resource content
*/
@Operation(summary = "downloadResource", description = "DOWNLOAD_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100"))
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/"))
})
@GetMapping(value = "/{id}/download")
@GetMapping(value = "/download")
@ResponseBody
@ApiException(DOWNLOAD_RESOURCE_FILE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public ResponseEntity downloadResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@PathVariable(value = "id") int resourceId) throws Exception {
Resource file = resourceService.downloadResource(loginUser, resourceId);
@RequestParam(value = "fullName") String fullName) throws Exception {
Resource file = resourceService.downloadResource(loginUser, fullName);
if (file == null) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(RESOURCE_NOT_EXIST.getMsg());
}
@ -460,7 +462,6 @@ public class ResourcesController extends BaseController {
* @param database database
* @param description description
* @param className class name
* @param resourceId resource id
* @return create result code
*/
@Operation(summary = "createUdfFunc", description = "CREATE_UDF_FUNCTION_NOTES")
@ -474,7 +475,7 @@ public class ResourcesController extends BaseController {
@Parameter(name = "resourceId", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100"))
})
@PostMapping(value = "/{resourceId}/udf-func")
@PostMapping(value = "/udf-func")
@ResponseStatus(HttpStatus.CREATED)
@ApiException(CREATE_UDF_FUNCTION_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
@ -482,20 +483,20 @@ public class ResourcesController extends BaseController {
@RequestParam(value = "type") UdfType type,
@RequestParam(value = "funcName") String funcName,
@RequestParam(value = "className") String className,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "argTypes", required = false) String argTypes,
@RequestParam(value = "database", required = false) String database,
@RequestParam(value = "description", required = false) String description,
@PathVariable(value = "resourceId") int resourceId) {
@RequestParam(value = "description", required = false) String description) {
// todo verify the sourceName
return udfFuncService.createUdfFunction(loginUser, funcName, className, argTypes, database, description, type,
resourceId);
return udfFuncService.createUdfFunction(loginUser, funcName, className, fullName,
argTypes, database, description, type);
}
/**
* view udf function
*
* @param loginUser login user
* @param id resource id
* @param id udf function id
* @return udf function detail
*/
@Operation(summary = "viewUIUdfFunction", description = "VIEW_UDF_FUNCTION_NOTES")
@ -521,7 +522,6 @@ public class ResourcesController extends BaseController {
* @param argTypes argument types
* @param database data base
* @param description description
* @param resourceId resource id
* @param className class name
* @param udfFuncId udf function id
* @return update result code
@ -534,11 +534,9 @@ public class ResourcesController extends BaseController {
@Parameter(name = "className", description = "CLASS_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "argTypes", description = "ARG_TYPES", schema = @Schema(implementation = String.class)),
@Parameter(name = "database", description = "DATABASE_NAME", schema = @Schema(implementation = String.class)),
@Parameter(name = "description", description = "UDF_DESC", schema = @Schema(implementation = String.class)),
@Parameter(name = "resourceId", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100"))
@Parameter(name = "description", description = "UDF_DESC", schema = @Schema(implementation = String.class))
})
@PutMapping(value = "/{resourceId}/udf-func/{id}")
@PutMapping(value = "/udf-func/{id}")
@ApiException(UPDATE_UDF_FUNCTION_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result updateUdfFunc(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@ -549,9 +547,9 @@ public class ResourcesController extends BaseController {
@RequestParam(value = "argTypes", required = false) String argTypes,
@RequestParam(value = "database", required = false) String database,
@RequestParam(value = "description", required = false) String description,
@PathVariable(value = "resourceId") int resourceId) {
return udfFuncService.updateUdfFunc(loginUser, udfFuncId, funcName, className, argTypes, database, description,
type, resourceId);
@RequestParam(value = "fullName") String fullName) {
return udfFuncService.updateUdfFunc(loginUser, udfFuncId, funcName, className,
argTypes, database, description, type, fullName);
}
/**
@ -731,23 +729,26 @@ public class ResourcesController extends BaseController {
}
/**
* query resource by resource id
* query a resource by resource full name
*
* @param loginUser login user
* @param id resource id
* @param fullName resource full name
* @return resource
*/
@Operation(summary = "queryResourceById", description = "QUERY_BY_RESOURCE_NAME")
@Operation(summary = "queryResourceByFullName", description = "QUERY_BY_RESOURCE_FULL_NAME")
@Parameters({
@Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "10"))
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class)),
})
@GetMapping(value = "/{id}/query")
@GetMapping(value = "/query-full-name")
@ResponseStatus(HttpStatus.OK)
@ApiException(RESOURCE_NOT_EXIST)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result queryResourceById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@PathVariable(value = "id", required = true) Integer id) {
public Result queryResourceByFullName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode") String tenantCode) throws IOException {
return resourceService.queryResourceById(loginUser, id);
return resourceService.queryResourceByFullName(loginUser, fullName, tenantCode, type);
}
}

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java

@ -35,7 +35,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder;
@JsonPropertyOrder({"id", "pid", "name", "fullName", "description", "isDirctory", "children", "type"})
public abstract class ResourceComponent {
public ResourceComponent(int id, int pid, String name, String fullName, String description, boolean isDirctory) {
public ResourceComponent(int id, String pid, String name, String fullName, String description, boolean isDirctory) {
this.id = id;
this.pid = pid;
this.name = name;
@ -53,7 +53,7 @@ public abstract class ResourceComponent {
/**
* parent id
*/
protected int pid;
protected String pid;
/**
* name
*/

43
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java

@ -19,10 +19,11 @@ package org.apache.dolphinscheduler.api.dto.resources.visitor;
import org.apache.dolphinscheduler.api.dto.resources.Directory;
import org.apache.dolphinscheduler.api.dto.resources.FileLeaf;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.service.storage.StorageEntity;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* resource tree visitor
@ -32,7 +33,7 @@ public class ResourceTreeVisitor implements Visitor {
/**
* resource list
*/
private List<Resource> resourceList;
private List<StorageEntity> resourceList;
public ResourceTreeVisitor() {
}
@ -41,7 +42,7 @@ public class ResourceTreeVisitor implements Visitor {
* constructor
* @param resourceList resource list
*/
public ResourceTreeVisitor(List<Resource> resourceList) {
public ResourceTreeVisitor(List<StorageEntity> resourceList) {
this.resourceList = resourceList;
}
@ -50,14 +51,15 @@ public class ResourceTreeVisitor implements Visitor {
* @return resoruce component
*/
@Override
public ResourceComponent visit() {
public ResourceComponent visit(String rootPath) {
ResourceComponent rootDirectory = new Directory();
for (Resource resource : resourceList) {
for (StorageEntity resource : resourceList) {
// judge whether is root node
if (rootNode(resource)) {
if (rootNode(resource, rootPath)) {
// if it is a root node.
ResourceComponent tempResourceComponent = getResourceComponent(resource);
rootDirectory.add(tempResourceComponent);
tempResourceComponent.setChildren(setChildren(tempResourceComponent.getId(), resourceList));
tempResourceComponent.setChildren(setChildren(tempResourceComponent.getFullName(), resourceList));
}
}
return rootDirectory;
@ -65,20 +67,21 @@ public class ResourceTreeVisitor implements Visitor {
/**
* set children
* @param id id
* @param fullName unique path
* @param list resource list
* @return resource component list
*/
public static List<ResourceComponent> setChildren(int id, List<Resource> list) {
public static List<ResourceComponent> setChildren(String fullName, List<StorageEntity> list) {
// id is the unique value,
List<ResourceComponent> childList = new ArrayList<>();
for (Resource resource : list) {
if (id == resource.getPid()) {
for (StorageEntity resource : list) {
if (Objects.equals(fullName, resource.getPfullName())) {
ResourceComponent tempResourceComponent = getResourceComponent(resource);
childList.add(tempResourceComponent);
}
}
for (ResourceComponent resourceComponent : childList) {
resourceComponent.setChildren(setChildren(resourceComponent.getId(), list));
resourceComponent.setChildren(setChildren(resourceComponent.getFullName(), list));
}
if (childList.size() == 0) {
return new ArrayList<>();
@ -91,17 +94,18 @@ public class ResourceTreeVisitor implements Visitor {
* @param resource resource
* @return true if it is the root node
*/
public boolean rootNode(Resource resource) {
public boolean rootNode(StorageEntity resource, String rootPath) {
boolean isRootNode = true;
if (resource.getPid() != -1) {
for (Resource parent : resourceList) {
if (resource.getPid() == parent.getId()) {
if (!Objects.equals(resource.getPfullName(), rootPath)) {
for (StorageEntity parent : resourceList) {
if (Objects.equals(resource.getPfullName(), parent.getFullName())) {
isRootNode = false;
break;
}
}
}
return isRootNode;
}
@ -110,7 +114,7 @@ public class ResourceTreeVisitor implements Visitor {
* @param resource resource
* @return resource component
*/
private static ResourceComponent getResourceComponent(Resource resource) {
private static ResourceComponent getResourceComponent(StorageEntity resource) {
ResourceComponent tempResourceComponent;
if (resource.isDirectory()) {
tempResourceComponent = new Directory();
@ -119,9 +123,10 @@ public class ResourceTreeVisitor implements Visitor {
}
tempResourceComponent.setName(resource.getAlias());
tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/", ""));
// tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/",""));
tempResourceComponent.setFullName(resource.getFullName());
tempResourceComponent.setId(resource.getId());
tempResourceComponent.setPid(resource.getPid());
tempResourceComponent.setPid(resource.getPfullName());
tempResourceComponent.setIdValue(resource.getId(), resource.isDirectory());
tempResourceComponent.setDescription(resource.getDescription());
tempResourceComponent.setType(resource.getType());

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java

@ -28,5 +28,5 @@ public interface Visitor {
* visit
* @return resource component
*/
ResourceComponent visit();
ResourceComponent visit(String rootPath);
}

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java

@ -38,7 +38,6 @@ public interface ResourcePermissionCheckService<T> {
* @param authorizationType
* @param userId
* @param logger
* @param <T>
* @return
*/
Set<T> userOwnedResourceIdsAcquisition(Object authorizationType, Integer userId, Logger logger);

44
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@ -57,9 +57,8 @@ public interface ResourcesService {
* @param loginUser login user
* @param name alias
* @param desc description
* @param file file
* @param type type
* @param pid parent id
* @param file file
* @param currentDir current directory
* @return create result code
*/
@ -68,13 +67,11 @@ public interface ResourcesService {
String desc,
ResourceType type,
MultipartFile file,
int pid,
String currentDir);
/**
* update resource
* @param loginUser login user
* @param resourceId resource id
* @param name name
* @param desc description
* @param type resource type
@ -82,7 +79,8 @@ public interface ResourcesService {
* @return update result code
*/
Result<Object> updateResource(User loginUser,
int resourceId,
String fullName,
String tenantCode,
String name,
String desc,
ResourceType type,
@ -98,8 +96,8 @@ public interface ResourcesService {
* @param pageSize page size
* @return resource list page
*/
Result queryResourceListPaging(User loginUser, int directoryId, ResourceType type, String searchVal, Integer pageNo,
Integer pageSize);
Result queryResourceListPaging(User loginUser, String fullName, String resTenantCode,
ResourceType type, String searchVal, Integer pageNo, Integer pageSize);
/**
* query resource list
@ -108,7 +106,7 @@ public interface ResourcesService {
* @param type resource type
* @return resource list
*/
Map<String, Object> queryResourceList(User loginUser, ResourceType type);
Map<String, Object> queryResourceList(User loginUser, ResourceType type, String fullName);
/**
* query resource list by program type
@ -123,11 +121,10 @@ public interface ResourcesService {
* delete resource
*
* @param loginUser login user
* @param resourceId resource id
* @return delete result code
* @throws IOException exception
*/
Result<Object> delete(User loginUser, int resourceId) throws IOException;
Result<Object> delete(User loginUser, String fullName, String tenantCode) throws IOException;
/**
* verify resource by name and type
@ -139,23 +136,22 @@ public interface ResourcesService {
Result<Object> verifyResourceName(String fullName, ResourceType type, User loginUser);
/**
* verify resource by full name or pid and type
* @param fullName resource full name
* @param id resource id
* verify resource by file name
* @param fileName resource file name
* @param type resource type
* @return true if the resource full name or pid not exists, otherwise return false
* @return true if the resource file name, otherwise return false
*/
Result<Object> queryResource(User loginUser, String fullName, Integer id, ResourceType type);
Result<Object> queryResourceByFileName(User loginUser, String fileName, ResourceType type, String resTenantCode);
/**
* view resource file online
*
* @param resourceId resource id
* @param skipLineNum skip line number
* @param limit limit
* @param fullName fullName
* @return resource content
*/
Result<Object> readResource(User loginUser, int resourceId, int skipLineNum, int limit);
Result<Object> readResource(User loginUser, String fullName, String tenantCode, int skipLineNum, int limit);
/**
* create resource file online
@ -169,7 +165,7 @@ public interface ResourcesService {
* @return create result code
*/
Result<Object> onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix,
String desc, String content, int pid, String currentDirectory);
String desc, String content, String currentDirectory);
/**
* create or update resource.
@ -203,16 +199,16 @@ public interface ResourcesService {
* @param content content
* @return update result cod
*/
Result<Object> updateResourceContent(User loginUser, int resourceId, String content);
Result<Object> updateResourceContent(User loginUser, String fullName, String tenantCode,
String content);
/**
* download file
*
* @param resourceId resource id
* @return resource content
* @throws IOException exception
*/
org.springframework.core.io.Resource downloadResource(User loginUser, int resourceId) throws IOException;
org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException;
/**
* list all file
@ -270,9 +266,11 @@ public interface ResourcesService {
/**
* get resource by id
* @param resourceId resource id
* @param fullName resource full name
* @param tenantCode owner's tenant code of resource
* @return resource
*/
Result<Object> queryResourceById(User loginUser, Integer resourceId);
Result<Object> queryResourceByFullName(User loginUser, String fullName, String tenantCode,
ResourceType type) throws IOException;
}

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java

@ -35,18 +35,17 @@ public interface UdfFuncService {
* @param argTypes argument types
* @param database database
* @param desc description
* @param resourceId resource id
* @param className class name
* @return create result code
*/
Result<Object> createUdfFunction(User loginUser,
String funcName,
String className,
String fullName,
String argTypes,
String database,
String desc,
UdfType type,
int resourceId);
UdfType type);
/**
* query udf function
@ -66,6 +65,7 @@ public interface UdfFuncService {
* @param database data base
* @param desc description
* @param resourceId resource id
* @param fullName resource full name
* @param className class name
* @return update result code
*/
@ -77,7 +77,7 @@ public interface UdfFuncService {
String database,
String desc,
UdfType type,
int resourceId);
String fullName);
/**
* query udf function list paging

1507
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java

File diff suppressed because it is too large Load Diff

55
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java

@ -25,15 +25,16 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@ -66,6 +67,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
@Autowired
private UDFUserMapper udfUserMapper;
@Autowired(required = false)
private StorageOperate storageOperate;
/**
* create udf function
*
@ -75,7 +79,6 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
* @param argTypes argument types
* @param database database
* @param desc description
* @param resourceId resource id
* @param className class name
* @return create result code
*/
@ -84,11 +87,11 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
public Result<Object> createUdfFunction(User loginUser,
String funcName,
String className,
String fullName,
String argTypes,
String database,
String desc,
UdfType type,
int resourceId) {
UdfType type) {
Result<Object> result = new Result<>();
boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF,
@ -117,9 +120,15 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
return result;
}
Resource resource = resourceMapper.selectById(resourceId);
if (resource == null) {
logger.error("Resource does not exist, resourceId:{}.", resourceId);
Boolean existResource = false;
try {
existResource = storageOperate.exists(fullName);
} catch (IOException e) {
logger.error("Check resource error: {}", fullName, e);
}
if (!existResource) {
logger.error("resource full name {} is not exist", fullName);
putMsg(result, Status.RESOURCE_NOT_EXIST);
return result;
}
@ -137,8 +146,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
udf.setDatabase(database);
}
udf.setDescription(desc);
udf.setResourceId(resourceId);
udf.setResourceName(resource.getFullName());
// set resourceId to -1 because we do not store resource to db anymore, instead we use fullName
udf.setResourceId(-1);
udf.setResourceName(fullName);
udf.setType(type);
udf.setCreateTime(now);
@ -178,7 +188,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
}
UdfFunc udfFunc = udfFuncMapper.selectById(id);
if (udfFunc == null) {
logger.error("Resource does not exist, resourceId:{}.", id);
logger.error("Resource does not exist, udf func id:{}.", id);
putMsg(result, Status.RESOURCE_NOT_EXIST);
return result;
}
@ -196,7 +206,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
* @param argTypes argument types
* @param database data base
* @param desc description
* @param resourceId resource id
* @param fullName resource full name
* @param className class name
* @return update result code
*/
@ -209,7 +219,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
String database,
String desc,
UdfType type,
int resourceId) {
String fullName) {
Result<Object> result = new Result<>();
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{udfFuncId},
@ -251,13 +261,23 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
}
}
Resource resource = resourceMapper.selectById(resourceId);
if (resource == null) {
logger.error("Resource does not exist, resourceId:{}.", resourceId);
Boolean doesResExist = false;
try {
doesResExist = storageOperate.exists(fullName);
} catch (Exception e) {
logger.error("udf resource checking error", fullName);
result.setCode(Status.RESOURCE_NOT_EXIST.getCode());
result.setMsg(Status.RESOURCE_NOT_EXIST.getMsg());
return result;
}
if (!doesResExist) {
logger.error("resource full name {} is not exist", fullName);
result.setCode(Status.RESOURCE_NOT_EXIST.getCode());
result.setMsg(Status.RESOURCE_NOT_EXIST.getMsg());
return result;
}
Date now = new Date();
udf.setFuncName(funcName);
udf.setClassName(className);
@ -266,8 +286,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
udf.setDatabase(database);
}
udf.setDescription(desc);
udf.setResourceId(resourceId);
udf.setResourceName(resource.getFullName());
// set resourceId to -1 because we do not store resource to db anymore, instead we use fullName
udf.setResourceId(-1);
udf.setResourceName(fullName);
udf.setType(type);
udf.setUpdateTime(now);

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@ -1173,7 +1173,7 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
if (CollectionUtils.isNotEmpty(components)) {
for (ResourceComponent component : components) {
// verify whether exist
if (!storageOperate.exists(oldTenantCode,
if (!storageOperate.exists(
String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()))) {
logger.error("Resource file: {} does not exist, copy error.", component.getFullName());
throw new ServiceException(Status.RESOURCE_NOT_EXIST);
@ -1188,8 +1188,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
if (CollectionUtils.isEmpty(component.getChildren())) {
// if not exist,need create it
if (!storageOperate.exists(oldTenantCode,
String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()))) {
if (!storageOperate
.exists(String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()))) {
storageOperate.mkdir(newTenantCode,
String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()));
}

63
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java

@ -65,11 +65,15 @@ public class ResourcesControllerTest extends AbstractControllerTest {
public void testQuerytResourceList() throws Exception {
Map<String, Object> mockResult = new HashMap<>();
mockResult.put(Constants.STATUS, Status.SUCCESS);
Mockito.when(resourcesService.queryResourceList(Mockito.any(), Mockito.any())).thenReturn(mockResult);
Mockito.when(resourcesService.queryResourceList(Mockito.any(), Mockito.any(), Mockito.anyString()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
paramsMap.add("type", ResourceType.FILE.name());
MvcResult mvcResult = mockMvc.perform(get("/resources/list")
.header(SESSION_ID, sessionId)
.param("type", ResourceType.FILE.name()))
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
@ -85,8 +89,8 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result mockResult = new Result<>();
mockResult.setCode(Status.SUCCESS.getCode());
Mockito.when(resourcesService.queryResourceListPaging(
Mockito.any(), Mockito.anyInt(), Mockito.any(), Mockito.anyString(), Mockito.anyInt(),
Mockito.anyInt()))
Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any(),
Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
@ -95,6 +99,8 @@ public class ResourcesControllerTest extends AbstractControllerTest {
paramsMap.add("pageNo", "1");
paramsMap.add("searchVal", "test");
paramsMap.add("pageSize", "1");
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
paramsMap.add("tenantCode", "123");
MvcResult mvcResult = mockMvc.perform(get("/resources")
.header(SESSION_ID, sessionId)
@ -137,14 +143,17 @@ public class ResourcesControllerTest extends AbstractControllerTest {
public void testViewResource() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.HDFS_NOT_STARTUP.getCode());
Mockito.when(resourcesService.readResource(Mockito.any(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()))
Mockito.when(resourcesService.readResource(Mockito.any(),
Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("skipLineNum", "2");
paramsMap.add("limit", "100");
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
paramsMap.add("tenantCode", "123");
MvcResult mvcResult = mockMvc.perform(get("/resources/{id}/view", "5")
MvcResult mvcResult = mockMvc.perform(get("/resources/view")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
@ -163,7 +172,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
Mockito.when(resourcesService
.onlineCreateResource(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.anyString(),
Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyString()))
Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
@ -192,14 +201,17 @@ public class ResourcesControllerTest extends AbstractControllerTest {
public void testUpdateResourceContent() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
Mockito.when(resourcesService.updateResourceContent(Mockito.any(), Mockito.anyInt(), Mockito.anyString()))
Mockito.when(resourcesService.updateResourceContent(Mockito.any(), Mockito.anyString(),
Mockito.anyString(), Mockito.anyString()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("id", "1");
paramsMap.add("content", "echo test_1111");
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
paramsMap.add("tenantCode", "123");
MvcResult mvcResult = mockMvc.perform(put("/resources/1/update-content")
MvcResult mvcResult = mockMvc.perform(put("/resources/update-content")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
@ -215,9 +227,14 @@ public class ResourcesControllerTest extends AbstractControllerTest {
@Test
public void testDownloadResource() throws Exception {
Mockito.when(resourcesService.downloadResource(Mockito.any(), Mockito.anyInt())).thenReturn(null);
Mockito.when(resourcesService.downloadResource(Mockito.any(), Mockito.anyString()))
.thenReturn(null);
MvcResult mvcResult = mockMvc.perform(get("/resources/{id}/download", 5)
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
MvcResult mvcResult = mockMvc.perform(get("/resources/download")
.params(paramsMap)
.header(SESSION_ID, sessionId))
.andExpect(status().is(HttpStatus.BAD_REQUEST.value()))
.andReturn();
@ -231,7 +248,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
Mockito.when(udfFuncService
.createUdfFunction(Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(),
Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.anyInt()))
Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
@ -242,8 +259,9 @@ public class ResourcesControllerTest extends AbstractControllerTest {
paramsMap.add("database", "database");
paramsMap.add("description", "description");
paramsMap.add("resourceId", "1");
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
MvcResult mvcResult = mockMvc.perform(post("/resources/{resourceId}/udf-func", "123")
MvcResult mvcResult = mockMvc.perform(post("/resources/udf-func")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isCreated())
@ -282,7 +300,8 @@ public class ResourcesControllerTest extends AbstractControllerTest {
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
Mockito.when(udfFuncService
.updateUdfFunc(Mockito.any(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyString(),
Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.anyInt()))
Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any(),
Mockito.anyString()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
@ -294,8 +313,9 @@ public class ResourcesControllerTest extends AbstractControllerTest {
paramsMap.add("database", "database");
paramsMap.add("description", "description");
paramsMap.add("resourceId", "1");
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
MvcResult mvcResult = mockMvc.perform(put("/resources/{resourceId}/udf-func/{id}", "123", "456")
MvcResult mvcResult = mockMvc.perform(put("/resources/udf-func/{id}", "456")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
@ -465,10 +485,15 @@ public class ResourcesControllerTest extends AbstractControllerTest {
public void testDeleteResource() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.SUCCESS.getCode());
Mockito.when(resourcesService.delete(Mockito.any(), Mockito.anyInt())).thenReturn(mockResult);
MvcResult mvcResult = mockMvc.perform(delete("/resources/{id}", "123")
.header(SESSION_ID, sessionId))
Mockito.when(resourcesService.delete(Mockito.any(), Mockito.anyString(),
Mockito.anyString()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
paramsMap.add("tenantCode", "123");
MvcResult mvcResult = mockMvc.perform(delete("/resources")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();

12
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java

@ -27,9 +27,12 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
@ -41,17 +44,21 @@ public class TenantControllerTest extends AbstractControllerTest {
private static final Logger logger = LoggerFactory.getLogger(TenantControllerTest.class);
private MockedStatic<PropertyUtils> mockedStaticPropertyUtils;
@Test
public void testCreateTenant() throws Exception {
mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("tenantCode", "hayden");
paramsMap.add("queueId", "1");
paramsMap.add("description", "tenant description");
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
MvcResult mvcResult = mockMvc.perform(post("/tenants/")
MvcResult mvcResult = mockMvc.perform(post("/tenants")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isCreated())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
@ -59,6 +66,7 @@ public class TenantControllerTest extends AbstractControllerTest {
Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
mockedStaticPropertyUtils.close();
}
@Test

58
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.api.dto.resources.visitor;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.service.storage.StorageEntity;
import java.util.ArrayList;
import java.util.List;
@ -31,53 +31,43 @@ import org.junit.jupiter.api.Test;
public class ResourceTreeVisitorTest {
@Test
public void visit() throws Exception {
List<Resource> resourceList = new ArrayList<>();
public void visit() {
List<StorageEntity> resourceList = new ArrayList<>();
StorageEntity resource1 = new StorageEntity();
resource1.setFullName("/default/a");
resource1.setPfullName("/default");
StorageEntity resource2 = new StorageEntity();
resource1.setFullName("/default/a/a1.txt");
resource1.setPfullName("/default/a");
Resource resource1 = new Resource(3, -1, "b", "/b", true);
Resource resource2 = new Resource(4, 2, "a1.txt", "/a/a1.txt", false);
Resource resource3 = new Resource(5, 3, "b1.txt", "/b/b1.txt", false);
Resource resource4 = new Resource(6, 3, "b2.jar", "/b/b2.jar", false);
Resource resource5 = new Resource(7, -1, "b2", "/b2", true);
Resource resource6 = new Resource(8, -1, "b2", "/b/b2", true);
Resource resource7 = new Resource(9, 8, "c2.jar", "/b/b2/c2.jar", false);
resourceList.add(resource1);
resourceList.add(resource2);
resourceList.add(resource3);
resourceList.add(resource4);
resourceList.add(resource5);
resourceList.add(resource6);
resourceList.add(resource7);
ResourceTreeVisitor resourceTreeVisitor = new ResourceTreeVisitor(resourceList);
ResourceComponent resourceComponent = resourceTreeVisitor.visit();
ResourceComponent resourceComponent = resourceTreeVisitor.visit("/default");
Assertions.assertNotNull(resourceComponent.getChildren());
}
@Test
public void rootNode() throws Exception {
List<Resource> resourceList = new ArrayList<>();
public void rootNode() {
List<StorageEntity> resourceList = new ArrayList<>();
StorageEntity resource1 = new StorageEntity();
resource1.setFullName("/default/a");
resource1.setPfullName("/default");
StorageEntity resource2 = new StorageEntity();
resource1.setFullName("/default/a/a1.txt");
resource1.setPfullName("/default/a");
Resource resource1 = new Resource(3, -1, "b", "/b", true);
Resource resource2 = new Resource(4, 2, "a1.txt", "/a/a1.txt", false);
Resource resource3 = new Resource(5, 3, "b1.txt", "/b/b1.txt", false);
Resource resource4 = new Resource(6, 3, "b2.jar", "/b/b2.jar", false);
Resource resource5 = new Resource(7, -1, "b2", "/b2", true);
Resource resource6 = new Resource(8, -1, "b2", "/b/b2", true);
Resource resource7 = new Resource(9, 8, "c2.jar", "/b/b2/c2.jar", false);
resourceList.add(resource1);
resourceList.add(resource2);
resourceList.add(resource3);
resourceList.add(resource4);
resourceList.add(resource5);
resourceList.add(resource6);
resourceList.add(resource7);
ResourceTreeVisitor resourceTreeVisitor = new ResourceTreeVisitor(resourceList);
Assertions.assertTrue(resourceTreeVisitor.rootNode(resource1));
Assertions.assertTrue(resourceTreeVisitor.rootNode(resource2));
Assertions.assertFalse(resourceTreeVisitor.rootNode(resource3));
Assertions.assertTrue(resourceTreeVisitor.rootNode(resource1, "/default"));
Assertions.assertFalse(resourceTreeVisitor.rootNode(resource2, "/default"));
}
}

582
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service;
import static org.mockito.ArgumentMatchers.eq;
import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService;
import org.apache.dolphinscheduler.api.service.impl.BaseServiceImpl;
@ -27,7 +26,6 @@ import org.apache.dolphinscheduler.api.service.impl.ResourcesServiceImpl;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
@ -41,6 +39,7 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.storage.StorageEntity;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
@ -145,11 +144,6 @@ public class ResourcesServiceTest {
@Test
public void testCreateResource() {
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, ApiFuncIdentificationConstant.FILE_UPLOAD, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = new User();
user.setId(1);
@ -158,10 +152,10 @@ public class ResourcesServiceTest {
// CURRENT_LOGIN_USER_TENANT_NOT_EXIST
MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes());
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(1)).thenReturn(null);
Result result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest",
ResourceType.FILE, mockMultipartFile, -1, "/");
ResourceType.FILE, mockMultipartFile, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg());
// set tenant for user
@ -171,7 +165,7 @@ public class ResourcesServiceTest {
// HDFS_NOT_STARTUP
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest",
ResourceType.FILE, null, -1, "/");
ResourceType.FILE, null, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
@ -179,7 +173,7 @@ public class ResourcesServiceTest {
mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes());
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest",
ResourceType.FILE, mockMultipartFile, -1, "/");
ResourceType.FILE, mockMultipartFile, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg());
@ -188,47 +182,33 @@ public class ResourcesServiceTest {
Mockito.when(Files.getFileExtension("test.pdf")).thenReturn("pdf");
Mockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.createResource(user, "ResourcesServiceTest.jar", "ResourcesServiceTest",
ResourceType.FILE, mockMultipartFile, -1, "/");
ResourceType.FILE, mockMultipartFile, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg());
// UDF_RESOURCE_SUFFIX_NOT_JAR
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, ApiFuncIdentificationConstant.UDF_UPLOAD, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, serviceLogger)).thenReturn(true);
mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf",
"pdf", "test".getBytes());
Mockito.when(Files.getFileExtension("ResourcesServiceTest.pdf")).thenReturn("pdf");
result = resourcesService.createResource(user, "ResourcesServiceTest.pdf", "ResourcesServiceTest",
ResourceType.UDF, mockMultipartFile, -1, "/");
ResourceType.UDF, mockMultipartFile, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg());
// FULL_FILE_NAME_TOO_LONG
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, ApiFuncIdentificationConstant.FILE_UPLOAD, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, serviceLogger)).thenReturn(true);
String tooLongFileName = getRandomStringWithLength(Constants.RESOURCE_FULL_NAME_MAX_LENGTH) + ".pdf";
mockMultipartFile = new MockMultipartFile(tooLongFileName, tooLongFileName, "pdf", "test".getBytes());
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(Files.getFileExtension(tooLongFileName)).thenReturn("pdf");
// '/databasePath/tenantCode/RESOURCE/'
Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
result = resourcesService.createResource(user, tooLongFileName, tooLongFileName, ResourceType.FILE,
mockMultipartFile, -1, "/");
mockMultipartFile, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR.getMsg(), result.getMsg());
}
@Test
public void testCreateDirecotry() {
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = new User();
user.setId(1);
@ -239,23 +219,18 @@ public class ResourcesServiceTest {
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
// PARENT_RESOURCE_NOT_EXIST
// RESOURCE_EXIST
user.setId(1);
user.setTenantId(1);
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser());
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcesMapper.selectById(Mockito.anyInt())).thenReturn(null);
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)).thenReturn(true);
result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, 1, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// RESOURCE_EXIST
Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
try {
Mockito.when(storageOperate.exists("/dolphinscheduler/123/resources/directoryTest")).thenReturn(true);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcesMapper.existResource("/directoryTest", 0)).thenReturn(true);
result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/");
@ -272,131 +247,126 @@ public class ResourcesServiceTest {
@Test
public void testUpdateResource() {
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{1}, 1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = new User();
user.setId(1);
user.setUserType(UserType.GENERAL_USER);
user.setTenantId(1);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
// HDFS_NOT_STARTUP
Result result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest",
ResourceType.FILE, null);
Result result = resourcesService.updateResource(user, "ResourcesServiceTest",
"123", "ResourcesServiceTest", "", ResourceType.FILE, null);
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
// RESOURCE_NOT_EXIST
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{0}, 1, serviceLogger)).thenReturn(true);
Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = resourcesService.updateResource(user, 0, "ResourcesServiceTest", "ResourcesServiceTest",
ResourceType.FILE, null);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// USER_NO_OPERATION_PERM
user.setId(2);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
user.setUserType(UserType.GENERAL_USER);
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
null, 2, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{1}, 2, serviceLogger)).thenReturn(false);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest",
ResourceType.FILE, null);
// tenant who have access to resource is 123,
Tenant tenantWNoPermission = new Tenant();
tenantWNoPermission.setTenantCode("321");
Mockito.when(tenantMapper.queryById(1)).thenReturn(tenantWNoPermission);
result = resourcesService.updateResource(user,
"/dolphinscheduler/123/resources/ResourcesServiceTest",
"123",
"ResourcesServiceTest", "", ResourceType.FILE, null);
logger.info(result.toString());
Assertions.assertEquals(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(), result.getMsg());
// RESOURCE_NOT_EXIST
user.setId(1);
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
// SUCCESS
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(storageOperate.getFileName(Mockito.any(), Mockito.any(), Mockito.anyString())).thenReturn("test1");
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF_FILE, null,
1, ApiFuncIdentificationConstant.UDF_UPDATE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.UDF_FILE, new Object[]{1},
1, serviceLogger)).thenReturn(true);
try {
Mockito.when(storageOperate.exists(Mockito.any(), Mockito.any())).thenReturn(false);
Mockito.when(storageOperate.exists(Mockito.any())).thenReturn(false);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest",
ResourceType.UDF, null);
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// SUCCESS
user.setId(1);
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
try {
Mockito.when(storageOperate.exists(Mockito.any(), Mockito.any())).thenReturn(true);
} catch (IOException e) {
logger.error(e.getMessage(), e);
Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest",
"/dolphinscheduler/123/resources/",
"123", ResourceType.FILE)).thenReturn(getStorageEntityResource());
result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest",
"123",
"ResourcesServiceTest", "", ResourceType.FILE, null);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
} catch (Exception e) {
logger.error(e.getMessage() + " Resource path: {}", "/dolphinscheduler/123/resources/ResourcesServiceTest",
e);
}
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{1},
1, serviceLogger)).thenReturn(true);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest.jar", "ResourcesServiceTest",
ResourceType.FILE, null);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
// Tests for udf resources.
// RESOURCE_EXIST
Mockito.when(resourcesMapper.existResource("/ResourcesServiceTest1.jar", 0)).thenReturn(true);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest",
ResourceType.FILE, null);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
// USER_NOT_EXIST
try {
Mockito.when(storageOperate.exists("/dolphinscheduler/123/resources/ResourcesServiceTest2.jar"))
.thenReturn(true);
} catch (IOException e) {
logger.error("error occurred when checking resource: "
+ "/dolphinscheduler/123/resources/ResourcesServiceTest2.jar");
}
Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(null);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest",
ResourceType.UDF, null);
try {
Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest1.jar",
"/dolphinscheduler/123/resources/",
"123", ResourceType.UDF)).thenReturn(getStorageEntityUdfResource());
} catch (Exception e) {
logger.error(e.getMessage() + " Resource path: {}",
"/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", e);
}
result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar",
"123", "ResourcesServiceTest2.jar", "", ResourceType.UDF, null);
logger.info(result.toString());
Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
Assertions.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
// TENANT_NOT_EXIST
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest",
ResourceType.UDF, null);
result = resourcesService.updateResource(user, "ResourcesServiceTest1.jar",
"", "ResourcesServiceTest", "", ResourceType.UDF, null);
logger.info(result.toString());
Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg());
// SUCCESS
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest1.jar",
ResourceType.UDF, null);
result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar",
"123", "ResourcesServiceTest1.jar", "", ResourceType.UDF, null);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testQueryResourceListPaging() {
User loginUser = new User();
loginUser.setId(1);
loginUser.setTenantId(1);
loginUser.setTenantCode("tenant1");
loginUser.setUserType(UserType.ADMIN_USER);
IPage<Resource> resourcePage = new Page<>(1, 10);
resourcePage.setTotal(1);
resourcePage.setRecords(getResourceList());
Mockito.when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.RESOURCE_FILE_ID,
1, resourceLogger)).thenReturn(getSetIds());
List<StorageEntity> mockResList = new ArrayList<StorageEntity>();
mockResList.add(getStorageEntityResource());
List<User> mockUserList = new ArrayList<User>();
mockUserList.add(getUser());
Mockito.when(userMapper.selectList(null)).thenReturn(mockUserList);
Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
Mockito.when(
resourcesMapper.queryResourcePaging(Mockito.any(Page.class), eq(-1), eq(0), eq("test"), Mockito.any()))
.thenReturn(resourcePage);
Result result = resourcesService.queryResourceListPaging(loginUser, -1, ResourceType.FILE, "test", 1, 10);
try {
Mockito.when(storageOperate.listFilesStatus("/dolphinscheduler/123/resources/",
"/dolphinscheduler/123/resources/",
"123", ResourceType.FILE)).thenReturn(mockResList);
} catch (Exception e) {
logger.error("QueryResourceListPaging Error");
}
Result result = resourcesService.queryResourceListPaging(loginUser, "", "",
ResourceType.FILE, "Test", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode());
PageInfo pageInfo = (PageInfo) result.getData();
@ -410,30 +380,33 @@ public class ResourcesServiceTest {
loginUser.setId(0);
loginUser.setUserType(UserType.ADMIN_USER);
Mockito.when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.RESOURCE_FILE_ID,
0, resourceLogger)).thenReturn(getSetIds());
Mockito.when(resourcesMapper.selectBatchIds(Mockito.anySet())).thenReturn(getResourceList());
Map<String, Object> result = resourcesService.queryResourceList(loginUser, ResourceType.FILE);
Mockito.when(userMapper.selectList(null)).thenReturn(Arrays.asList(loginUser));
Mockito.when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser);
Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant());
Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
Mockito.when(storageOperate.listFilesStatusRecursively("/dolphinscheduler/123/resources/",
"/dolphinscheduler/123/resources/",
"123",
ResourceType.FILE)).thenReturn(Arrays.asList(getStorageEntityResource()));
Map<String, Object> result = resourcesService.queryResourceList(loginUser, ResourceType.FILE, "");
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<Resource> resourceList = (List<Resource>) result.get(Constants.DATA_LIST);
Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList));
// test udf
Mockito.when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF_FILE, 0,
resourceLogger)).thenReturn(getSetIds());
Mockito.when(resourcesMapper.selectBatchIds(Mockito.anySet()))
.thenReturn(Arrays.asList(getResource(11, ResourceType.UDF),
getResource(10, ResourceType.UDF), getResource(9, ResourceType.UDF),
getResource(8, ResourceType.UDF)));
Mockito.when(storageOperate.getUdfDir("123")).thenReturn("/dolphinscheduler/123/udfs/");
Mockito.when(storageOperate.listFilesStatusRecursively("/dolphinscheduler/123/udfs/",
"/dolphinscheduler/123/udfs/",
"123",
ResourceType.UDF))
.thenReturn(Arrays.asList(getStorageEntityUdfResource()));
loginUser.setUserType(UserType.GENERAL_USER);
result = resourcesService.queryResourceList(loginUser, ResourceType.UDF);
result = resourcesService.queryResourceList(loginUser, ResourceType.UDF, "");
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
resourceList = (List<Resource>) result.get(Constants.DATA_LIST);
Assertions.assertTrue(resourceList.size() == 4);
Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList));
}
@Test
@ -443,59 +416,34 @@ public class ResourcesServiceTest {
loginUser.setId(0);
loginUser.setUserType(UserType.GENERAL_USER);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
0, ApiFuncIdentificationConstant.FILE_DELETE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{1},
0, serviceLogger)).thenReturn(true);
try {
// HDFS_NOT_STARTUP
Result result = resourcesService.delete(loginUser, 1);
Result result = resourcesService.delete(loginUser, "", "");
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
// RESOURCE_NOT_EXIST
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
Mockito.when(
resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
0, ApiFuncIdentificationConstant.FILE_DELETE, serviceLogger))
.thenReturn(true);
result = resourcesService.delete(loginUser, 2);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// USER_NO_OPERATION_PERM
result = resourcesService.delete(loginUser, 2);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// TENANT_NOT_EXIST
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
loginUser.setUserType(UserType.ADMIN_USER);
loginUser.setTenantId(2);
Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(loginUser);
Mockito.when(
resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
0, ApiFuncIdentificationConstant.FILE_DELETE, serviceLogger))
.thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{1},
0, serviceLogger)).thenReturn(true);
result = resourcesService.delete(loginUser, 1);
Mockito.when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser);
result = resourcesService.delete(loginUser, "", "");
logger.info(result.toString());
Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg());
// RESOURCE_NOT_EXIST
Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant());
Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest",
null, "123", null))
.thenReturn(getStorageEntityResource());
result = resourcesService.delete(loginUser, "/dolphinscheduler/123/resources/ResNotExist", "123");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// SUCCESS
loginUser.setTenantId(1);
Mockito.when(processDefinitionMapper.listResources()).thenReturn(getResources());
Mockito.when(resourcesMapper.deleteIds(Mockito.any())).thenReturn(1);
Mockito.when(resourceUserMapper.deleteResourceUserArray(Mockito.anyInt(), Mockito.any())).thenReturn(1);
result = resourcesService.delete(loginUser, 1);
result = resourcesService.delete(loginUser, "/dolphinscheduler/123/resources/ResourcesServiceTest",
"123");
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
@ -508,29 +456,19 @@ public class ResourcesServiceTest {
@Test
public void testVerifyResourceName() {
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, serviceLogger)).thenReturn(true);
User user = new User();
user.setId(1);
user.setUserType(UserType.GENERAL_USER);
Mockito.when(resourcesMapper.existResource("/ResourcesServiceTest.jar", 0)).thenReturn(true);
try {
Mockito.when(storageOperate.exists("/ResourcesServiceTest.jar")).thenReturn(true);
} catch (IOException e) {
logger.error("error occurred when checking resource: /ResourcesServiceTest.jar\"");
}
Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
// TENANT_NOT_EXIST
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
String unExistFullName = "/test.jar";
result = resourcesService.verifyResourceName("/test.jar", ResourceType.FILE, user);
logger.info(result.toString());
Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg());
// RESOURCE_FILE_EXIST
user.setTenantId(1);
result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user);
logger.info(result.toString());
Assertions.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode());
@ -544,123 +482,89 @@ public class ResourcesServiceTest {
@Test
public void testReadResource() {
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_VIEW, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{1},
1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
// HDFS_NOT_STARTUP
Result result = resourcesService.readResource(getUser(), 1, 1, 10);
Result result = resourcesService.readResource(getUser(), "", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
// RESOURCE_NOT_EXIST
Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_VIEW, serviceLogger)).thenReturn(true);
result = resourcesService.readResource(getUser(), 2, 1, 10);
Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
result = resourcesService.readResource(getUser(), "", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
Assertions.assertEquals(Status.RESOURCE_FILE_NOT_EXIST.getCode(), (int) result.getCode());
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_VIEW, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{1},
1, serviceLogger)).thenReturn(true);
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = resourcesService.readResource(getUser(), 1, 1, 10);
result = resourcesService.readResource(getUser(), "", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
// USER_NOT_EXIST
Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(null);
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
Mockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.readResource(getUser(), 1, 1, 10);
result = resourcesService.readResource(getUser(), "", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.USER_NOT_EXIST.getCode(), (int) result.getCode());
// TENANT_NOT_EXIST
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
result = resourcesService.readResource(getUser(), 1, 1, 10);
Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(null);
result = resourcesService.readResource(getUser(), "", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg());
// RESOURCE_FILE_NOT_EXIST
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
result = resourcesService.readResource(getUser(), 1, 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_FILE_NOT_EXIST.getCode(), (int) result.getCode());
// SUCCESS
Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
try {
Mockito.when(storageOperate.exists(Mockito.any(), Mockito.any())).thenReturn(true);
Mockito.when(storageOperate.exists(Mockito.any())).thenReturn(true);
Mockito.when(storageOperate.vimFile(Mockito.any(), Mockito.any(), eq(1), eq(10))).thenReturn(getContent());
} catch (IOException e) {
logger.error("storage error", e);
}
result = resourcesService.readResource(getUser(), 1, 1, 10);
Mockito.when(Files.getFileExtension("test.jar")).thenReturn("jar");
result = resourcesService.readResource(getUser(), "test.jar", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testOnlineCreateResource() {
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_ONLINE_CREATE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = getUser();
user.setId(1);
Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
// HDFS_NOT_STARTUP
Result result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",
-1, "/");
"/");
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1,
result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",
"/");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
// RuntimeException
try {
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",
-1, "/");
} catch (RuntimeException ex) {
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), ex.getMessage());
}
// SUCCESS
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, serviceLogger)).thenReturn(true);
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1,
result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",
"/");
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
@ -684,141 +588,138 @@ public class ResourcesServiceTest {
Resource dir2 = new Resource();
dir2.setFullName(resourceDir);
dir2.setUserId(user.getId());
Mockito.when(resourcesMapper.queryResource(dir1.getFullName(), ResourceType.FILE.ordinal()))
.thenReturn(Collections.singletonList(dir1));
Mockito.when(resourcesMapper.queryResource(resourceDir, ResourceType.FILE.ordinal())).thenReturn(null);
Tenant tenant = getTenant();
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(
AuthorizationType.RESOURCE_FILE_ID, null, 1, ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE,
serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(
AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true);
Mockito.when(storageOperate.getDir(ResourceType.FILE, "123")).thenReturn("/dolphinscheduler/123/resources/");
Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
try {
Mockito.when(storageOperate.mkdir(tenant.getTenantCode(), null)).thenReturn(true);
Mockito.when(storageOperate.mkdir("123", "/dolphinscheduler/123/resources" + dir1Path)).thenReturn(true);
Mockito.when(storageOperate.mkdir("123", "/dolphinscheduler/123/resources" + dir2Path)).thenReturn(true);
} catch (IOException e) {
logger.error("storage error", e);
logger.error("create resource directory {} failed", fullName);
}
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(
AuthorizationType.RESOURCE_FILE_ID, null, 1, ApiFuncIdentificationConstant.FILE_ONLINE_CREATE,
serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(
AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(
AuthorizationType.RESOURCE_FILE_ID, null, 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger))
.thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(
AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true);
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant());
Result<Object> result = resourcesService.onlineCreateOrUpdateResourceWithDir(user, fullName, desc, content);
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testQueryResourcesFileInfo() {
User user = getUser();
String userName = "test-user";
Mockito.when(userMapper.queryByUserNameAccurately(userName)).thenReturn(user);
Resource file = new Resource();
file.setFullName("/dir/file1.py");
file.setId(1);
Mockito.when(resourcesMapper.queryResource(file.getFullName(), ResourceType.FILE.ordinal()))
.thenReturn(Collections.singletonList(file));
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(
AuthorizationType.RESOURCE_FILE_ID, null, user.getId(), ApiFuncIdentificationConstant.FILE_VIEW,
serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(
AuthorizationType.RESOURCE_FILE_ID, new Object[]{file.getId()}, user.getId(), serviceLogger))
.thenReturn(true);
Resource result = resourcesService.queryResourcesFileInfo(userName, file.getFullName());
Assertions.assertEquals(file.getFullName(), result.getFullName());
}
// TODO: revise this testcase after modifying PythonGateway.java
// @Test
// public void testQueryResourcesFileInfo() {
// User user = getUser();
// String userName = "test-user";
// Mockito.when(userMapper.queryByUserNameAccurately(userName)).thenReturn(user);
// Resource file = new Resource();
// file.setFullName("/dir/file1.py");
// file.setId(1);
// Mockito.when(resourcesMapper.queryResource(file.getFullName(), ResourceType.FILE.ordinal()))
// .thenReturn(Collections.singletonList(file));
// Mockito.when(resourcePermissionCheckService.operationPermissionCheck(
// AuthorizationType.RESOURCE_FILE_ID, null, user.getId(), ApiFuncIdentificationConstant.FILE_VIEW,
// serviceLogger)).thenReturn(true);
// Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(
// AuthorizationType.RESOURCE_FILE_ID, new Object[]{file.getId()}, user.getId(), serviceLogger))
// .thenReturn(true);
// Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
// Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
// Resource result = resourcesService.queryResourcesFileInfo(userName, file.getFullName());
// Assertions.assertEquals(file.getFullName(), result.getFullName());
// }
@Test
public void testUpdateResourceContent() {
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
// HDFS_NOT_STARTUP
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{1},
1, serviceLogger)).thenReturn(true);
Result result = resourcesService.updateResourceContent(getUser(), 1, "content");
Result result = resourcesService.updateResourceContent(getUser(), "", "", "content");
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
// RESOURCE_NOT_EXIST
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
result = resourcesService.updateResourceContent(getUser(), 2, "content");
Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
try {
Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest.jar",
"",
"123", ResourceType.FILE)).thenReturn(null);
} catch (Exception e) {
logger.error(e.getMessage() + " Resource path: {}", "", e);
}
result = resourcesService.updateResourceContent(getUser(),
"/dolphinscheduler/123/resources/ResourcesServiceTest.jar",
"123", "content");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{1},
1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
result = resourcesService.updateResourceContent(getUser(), 1, "content");
try {
Mockito.when(storageOperate.getFileStatus("", "", "123", ResourceType.FILE))
.thenReturn(getStorageEntityResource());
} catch (Exception e) {
logger.error(e.getMessage() + " Resource path: {}", "", e);
}
result = resourcesService.updateResourceContent(getUser(), "", "123", "content");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
// USER_NOT_EXIST
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
Mockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.updateResourceContent(getUser(), 1, "content");
Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(null);
result = resourcesService.updateResourceContent(getUser(), "", "123", "content");
logger.info(result.toString());
Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
// TENANT_NOT_EXIST
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
result = resourcesService.updateResourceContent(getUser(), 1, "content");
Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(1)).thenReturn(null);
result = resourcesService.updateResourceContent(getUser(), "", "123", "content");
logger.info(result.toString());
Assertions.assertTrue(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getCode() == result.getCode());
// SUCCESS
try {
Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest.jar",
"",
"123", ResourceType.FILE)).thenReturn(getStorageEntityResource());
} catch (Exception e) {
logger.error(e.getMessage() + " Resource path: {}", "", e);
}
Mockito.when(Files.getFileExtension(Mockito.anyString())).thenReturn("jar");
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
result = resourcesService.updateResourceContent(getUser(), 1, "content");
result = resourcesService.updateResourceContent(getUser(),
"/dolphinscheduler/123/resources/ResourcesServiceTest.jar",
"123", "content");
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testDownloadResource() {
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
1, ApiFuncIdentificationConstant.FILE_DOWNLOAD, serviceLogger)).thenReturn(true);
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
new Object[]{1},
1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class);
try {
// resource null
org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), 1);
org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), "");
Assertions.assertNull(resource);
Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
Mockito.when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any()))
.thenReturn(resourceMock);
resource = resourcesService.downloadResource(getUser(), 1);
resource = resourcesService.downloadResource(getUser(), "");
Assertions.assertNotNull(resource);
} catch (Exception e) {
logger.error("DownloadResource error", e);
@ -1016,7 +917,6 @@ public class ResourcesServiceTest {
}
private Resource getResource() {
Resource resource = new Resource();
resource.setPid(-1);
resource.setUserId(1);
@ -1027,6 +927,19 @@ public class ResourcesServiceTest {
return resource;
}
private StorageEntity getStorageEntityResource() {
StorageEntity entity = new StorageEntity();
entity.setAlias("ResourcesServiceTest");
entity.setFileName("ResourcesServiceTest");
entity.setDirectory(false);
entity.setDescription("");
entity.setUserName("123");
entity.setType(ResourceType.FILE);
entity.setFullName("/dolphinscheduler/123/resources/ResourcesServiceTest");
return entity;
}
private Resource getResource(int resourceId) {
Resource resource = new Resource();
@ -1064,6 +977,19 @@ public class ResourcesServiceTest {
return resource;
}
private StorageEntity getStorageEntityUdfResource() {
StorageEntity entity = new StorageEntity();
entity.setAlias("ResourcesServiceTest1.jar");
entity.setFileName("ResourcesServiceTest1.jar");
entity.setDirectory(false);
entity.setDescription("");
entity.setUserName("123");
entity.setType(ResourceType.UDF);
entity.setFullName("/dolphinscheduler/123/resources/ResourcesServiceTest1.jar");
return entity;
}
private UdfFunc getUdfFunc() {
UdfFunc udfFunc = new UdfFunc();

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java

@ -40,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.commons.collections.CollectionUtils;
@ -96,6 +97,9 @@ public class TenantServiceTest {
@Mock
private ResourcePermissionCheckService resourcePermissionCheckService;
@Mock
private StorageOperate storageOperate;
private static final String tenantCode = "hayden";
private static final String tenantDesc = "This is the tenant desc";
private static final String queue = "queue";

34
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java

@ -34,9 +34,11 @@ import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.commons.collections.CollectionUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@ -85,6 +87,9 @@ public class UdfFuncServiceTest {
@Mock
private UDFUserMapper udfUserMapper;
@Mock
private StorageOperate storageOperate;
@BeforeEach
public void setUp() {
mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class);
@ -108,21 +113,27 @@ public class UdfFuncServiceTest {
// hdfs not start
Result result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE);
"UdfFuncServiceTest", "UdfFuncServiceTest", "", UdfType.HIVE);
logger.info(result.toString());
Assertions.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg());
// resource not exist
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE);
"UdfFuncServiceTest", "UdfFuncServiceTest", "", UdfType.HIVE);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// success
Mockito.when(resourceMapper.selectById(1)).thenReturn(getResource());
try {
Mockito.when(storageOperate.exists("String")).thenReturn(true);
} catch (IOException e) {
logger.error("AmazonServiceException when checking resource: String");
}
result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1);
"UdfFuncServiceTest", "UdfFuncServiceTest", "", UdfType.HIVE);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@ -154,7 +165,6 @@ public class UdfFuncServiceTest {
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
Mockito.when(udfFuncMapper.selectUdfById(1)).thenReturn(getUdfFunc());
Mockito.when(resourceMapper.selectById(1)).thenReturn(getResource());
// UDF_FUNCTION_NOT_EXIST
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF, null, 1,
@ -163,7 +173,7 @@ public class UdfFuncServiceTest {
serviceLogger)).thenReturn(true);
Result<Object> result = udfFuncService.updateUdfFunc(getLoginUser(), 12, "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1);
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, "");
logger.info(result.toString());
Assertions.assertTrue(Status.UDF_FUNCTION_NOT_EXIST.getCode() == result.getCode());
@ -172,7 +182,7 @@ public class UdfFuncServiceTest {
serviceLogger)).thenReturn(true);
result = udfFuncService.updateUdfFunc(getLoginUser(), 1, "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1);
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, "");
logger.info(result.toString());
Assertions.assertTrue(Status.HDFS_NOT_STARTUP.getCode() == result.getCode());
@ -185,16 +195,22 @@ public class UdfFuncServiceTest {
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = udfFuncService.updateUdfFunc(getLoginUser(), 11, "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 12);
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, "");
logger.info(result.toString());
Assertions.assertTrue(Status.RESOURCE_NOT_EXIST.getCode() == result.getCode());
// success
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF, null, 1,
ApiFuncIdentificationConstant.UDF_FUNCTION_UPDATE, serviceLogger)).thenReturn(true);
try {
Mockito.when(storageOperate.exists("")).thenReturn(true);
} catch (IOException e) {
logger.error("AmazonServiceException when checking resource: ");
}
result = udfFuncService.updateUdfFunc(getLoginUser(), 11, "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1);
"UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, "");
logger.info(result.toString());
Assertions.assertTrue(Status.SUCCESS.getCode() == result.getCode());

56
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesTask.java

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package org.apache.dolphinscheduler.dao.entity;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.Data;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
@Data
@TableName("t_ds_relation_resources_task")
public class ResourcesTask {
@TableId(value = "id", type = IdType.AUTO)
private Integer id;
private String fullName;
private int taskId;
private ResourceType type;
public ResourcesTask(int id, String fullName, int taskId, ResourceType type) {
this.id = id;
this.fullName = fullName;
this.taskId = taskId;
this.type = type;
}
public ResourcesTask(int taskId, String fullName, ResourceType type) {
this.taskId = taskId;
this.fullName = fullName;
this.type = type;
}
}

1
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java

@ -54,7 +54,6 @@ public interface ResourceMapper extends BaseMapper<Resource> {
/**
* resource page
* @param page page
* @param userId userId
* @param id id
* @param type type
* @param searchVal searchVal

45
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.java

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.ResourcesTask;
import org.apache.ibatis.annotations.Param;
import java.util.List;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* resource task relation mapper interface
*/
public interface ResourceTaskMapper extends BaseMapper<ResourcesTask> {
Integer existResourceByTaskIdNFullName(@Param("taskId") int task_id, @Param("fullName") String fullName);
int deleteIds(@Param("resIds") Integer[] resIds);
int updateResource(@Param("id") int id, @Param("fullName") String fullName);
List<ResourcesTask> selectBatchFullNames(@Param("fullNameArr") String[] fullNameArr);
List<ResourcesTask> selectSubfoldersFullNames(@Param("folderPath") String folderPath);
}

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.java

@ -75,7 +75,7 @@ public interface TaskDefinitionMapper extends BaseMapper<TaskDefinition> {
List<DefinitionGroupByUser> countDefinitionGroupByUser(@Param("projectCodes") Long[] projectCodes);
/**
* list all resource ids
* list all resource ids and task_params containing resourceList
*
* @return task ids list
*/

11
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java

@ -49,7 +49,7 @@ public interface UdfFuncMapper extends BaseMapper<UdfFunc> {
/**
* udf function page
* @param page page
* @param userId userId
* @param ids userId
* @param searchVal searchVal
* @return udf function IPage
*/
@ -59,7 +59,7 @@ public interface UdfFuncMapper extends BaseMapper<UdfFunc> {
/**
* query udf function by type
* @param userId userId
* @param ids userId
* @param type type
* @return udf function list
*/
@ -95,6 +95,13 @@ public interface UdfFuncMapper extends BaseMapper<UdfFunc> {
*/
List<UdfFunc> listUdfByResourceId(@Param("resourceIds") Integer[] resourceIds);
/**
* list UDF by resource fullName
* @param resourceFullNames resource fullName array
* @return UDF function list
*/
List<UdfFunc> listUdfByResourceFullName(@Param("resourceFullNames") String[] resourceFullNames);
/**
* list authorized UDF by resource id
* @param resourceIds resource id array

30
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java

@ -66,4 +66,34 @@ public class ResourceProcessDefinitionUtils {
return resourceResult;
}
public static <T> Map<Integer, Set<T>> getResourceObjectMap(List<Map<String, Object>> resourceList,
String objectName, Class<T> clazz) {
// resourceId -> task ids or code depends on the objectName
Map<Integer, Set<T>> resourceResult = new HashMap<>();
if (CollectionUtils.isNotEmpty(resourceList)) {
for (Map<String, Object> resourceMap : resourceList) {
// get resName and resource_id_news, td_id
T taskId = (T) resourceMap.get(objectName);
String[] resourceIds = ((String) resourceMap.get("resource_ids"))
.split(",");
Set<Integer> resourceIdSet =
Arrays.stream(resourceIds).map(Integer::parseInt).collect(Collectors.toSet());
for (Integer resourceId : resourceIdSet) {
Set<T> codeSet;
if (resourceResult.containsKey(resourceId)) {
codeSet = resourceResult.get(resourceId);
} else {
codeSet = new HashSet<>();
}
codeSet.add(taskId);
resourceResult.put(resourceId, codeSet);
}
}
}
return resourceResult;
}
}

4
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml

@ -168,7 +168,7 @@
</select>
<select id="listResources" resultType="java.util.HashMap">
SELECT distinct pd.code,td.resource_ids
SELECT distinct pd.code, td.id as td_id, td.resource_ids
FROM t_ds_process_task_relation ptr
join t_ds_process_definition pd
on ptr.process_definition_code=pd.code and ptr.process_definition_version = pd.version
@ -180,7 +180,7 @@
</select>
<select id="listResourcesByUser" resultType="java.util.HashMap">
SELECT distinct pd.code,td.resource_ids
SELECT distinct pd.code, td.id as td_id, td.resource_ids
FROM t_ds_process_task_relation ptr
join t_ds_process_definition pd
on ptr.process_definition_code=pd.code and ptr.process_definition_version = pd.version

58
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.xml

@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ResourceTaskMapper">
<sql id="baseSqlV2">
${alias}.id, ${alias}.full_name, ${alias}.task_id, ${alias}.type
</sql>
<select id="existResourceByTaskIdNFullName" resultType="java.lang.Integer">
select
id
from t_ds_relation_resources_task
where full_name = #{fullName} and task_id = #{taskId}
</select>
<select id="selectBatchFullNames" resultType="org.apache.dolphinscheduler.dao.entity.ResourcesTask">
select
id, full_name, task_id, type
from t_ds_relation_resources_task
where full_name in
<foreach collection="fullNameArr" item="i" open="(" close=")" separator=",">
#{i}
</foreach>
</select>
<update id="updateResource" >
UPDATE t_ds_relation_resources_task SET full_name=#{fullName} WHERE id=#{id}
</update>
<delete id="deleteIds" parameterType="java.lang.Integer">
delete from t_ds_relation_resources_task where id in
<foreach collection="resIds" item="i" open="(" close=")" separator=",">
#{i}
</foreach>
</delete>
<select id="selectSubfoldersFullNames" resultType="org.apache.dolphinscheduler.dao.entity.ResourcesTask">
select
id, full_name, task_id, type
from t_ds_relation_resources_task
where full_name like concat(#{folderPath}, '%')
</select>
</mapper>

4
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml

@ -96,8 +96,8 @@
#{taskDefinition.projectCode},#{taskDefinition.userId},#{taskDefinition.taskType},#{taskDefinition.taskParams},#{taskDefinition.flag},
#{taskDefinition.taskPriority},#{taskDefinition.workerGroup},#{taskDefinition.environmentCode},#{taskDefinition.failRetryTimes},
#{taskDefinition.failRetryInterval},#{taskDefinition.timeoutFlag},#{taskDefinition.timeoutNotifyStrategy},#{taskDefinition.timeout},
#{taskDefinition.delayTime},#{taskDefinition.resourceIds},#{taskDefinition.createTime},#{taskDefinition.updateTime}, #{taskDefinition.taskGroupId},
#{taskDefinition.taskExecuteType})
#{taskDefinition.delayTime},#{taskDefinition.resourceIds},#{taskDefinition.createTime},#{taskDefinition.updateTime},
#{taskDefinition.taskGroupId}, #{taskDefinition.taskExecuteType})
</foreach>
</insert>
<select id="queryDefineListPaging" resultType="org.apache.dolphinscheduler.dao.entity.TaskMainInfo">

14
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml

@ -141,6 +141,20 @@
</foreach>
</if>
</select>
<select id="listUdfByResourceFullName" resultType="org.apache.dolphinscheduler.dao.entity.UdfFunc">
select
<include refid="baseSql">
<property name="alias" value="udf"/>
</include>
from t_ds_udfs udf
where 1=1
<if test="resourceFullNames != null and resourceFullNames.length > 0">
and udf.resource_name in
<foreach collection="resourceFullNames" item="i" open="(" close=")" separator=",">
#{i}
</foreach>
</if>
</select>
<select id="listAuthorizedUdfByResourceId" resultType="org.apache.dolphinscheduler.dao.entity.UdfFunc">
select
<include refid="baseSql">

18
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql

@ -793,6 +793,24 @@ CREATE TABLE t_ds_resources
-- Records of t_ds_resources
-- ----------------------------
-- ----------------------------
-- Table structure for t_ds_relation_resources_task
-- ----------------------------
DROP TABLE IF EXISTS t_ds_relation_resources_task CASCADE;
CREATE TABLE t_ds_relation_resources_task
(
id int(11) NOT NULL AUTO_INCREMENT,
task_id int(11) DEFAULT NULL,
full_name varchar(255) DEFAULT NULL,
type tinyint(4) DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY t_ds_relation_resources_task_un (task_id, full_name)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
-- ----------------------------
-- Records of t_ds_relation_resources_task
-- ----------------------------
-- ----------------------------
-- Table structure for t_ds_schedules
-- ----------------------------

17
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql

@ -791,6 +791,23 @@ CREATE TABLE `t_ds_resources` (
-- Records of t_ds_resources
-- ----------------------------
-- ----------------------------
-- Table structure for t_ds_relation_resources_task
-- ----------------------------
DROP TABLE IF EXISTS `t_ds_relation_resources_task`;
CREATE TABLE `t_ds_relation_resources_task` (
`id` int NOT NULL AUTO_INCREMENT COMMENT 'key',
`task_id` int(11) DEFAULT NULL COMMENT 'task id',
`full_name` varchar(255) DEFAULT NULL,
`type` tinyint DEFAULT NULL COMMENT 'resource type,0:FILE,1:UDF',
PRIMARY KEY (`id`),
UNIQUE KEY `t_ds_relation_resources_task_un` (`task_id`, `full_name`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
-- ----------------------------
-- Records of t_ds_relation_resources_task
-- ----------------------------
-- ----------------------------
-- Table structure for t_ds_schedules
-- ----------------------------

12
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql

@ -700,6 +700,18 @@ CREATE TABLE t_ds_resources (
CONSTRAINT t_ds_resources_un UNIQUE (full_name, type)
) ;
--
-- Table structure for table t_ds_relation_resources_task
--
DROP TABLE IF EXISTS t_ds_relation_resources_task;
CREATE TABLE t_ds_relation_resources_task (
id SERIAL NOT NULL,
task_id int DEFAULT NULL,
full_name varchar(255) DEFAULT NULL,
type int DEFAULT NULL,
PRIMARY KEY (id),
CONSTRAINT t_ds_relation_resources_task_un UNIQUE (task_id, full_name)
);
--
-- Table structure for table t_ds_schedules

27
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java

@ -44,7 +44,6 @@ import org.apache.dolphinscheduler.dao.entity.DqRule;
import org.apache.dolphinscheduler.dao.entity.DqRuleExecuteSql;
import org.apache.dolphinscheduler.dao.entity.DqRuleInputEntry;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
@ -92,9 +91,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.NonNull;
@ -624,26 +620,9 @@ public abstract class BaseTaskProcessor implements ITaskProcessor {
if (baseParam != null) {
List<ResourceInfo> projectResourceFiles = baseParam.getResourceFilesList();
if (CollectionUtils.isNotEmpty(projectResourceFiles)) {
// filter the resources that the resource id equals 0
Set<ResourceInfo> oldVersionResources =
projectResourceFiles.stream().filter(t -> t.getId() == null).collect(Collectors.toSet());
if (CollectionUtils.isNotEmpty(oldVersionResources)) {
oldVersionResources.forEach(t -> resourcesMap.put(t.getRes(),
processService.queryTenantCodeByResName(t.getRes(), ResourceType.FILE)));
}
// get the resource id in order to get the resource names in batch
Stream<Integer> resourceIdStream = projectResourceFiles.stream().map(ResourceInfo::getId);
Set<Integer> resourceIdsSet = resourceIdStream.collect(Collectors.toSet());
if (CollectionUtils.isNotEmpty(resourceIdsSet)) {
Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]);
List<Resource> resources = processService.listResourceByIds(resourceIds);
resources.forEach(t -> resourcesMap.put(t.getFullName(),
processService.queryTenantCodeByResName(t.getFullName(), ResourceType.FILE)));
}
// TODO: Modify this part to accomodate(migrate) oldversionresources in the future.
projectResourceFiles.forEach(file -> resourcesMap.put(file.getResourceName(),
processService.queryTenantCodeByResName(file.getResourceName(), ResourceType.FILE)));
}
}

101
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java

@ -69,6 +69,7 @@ import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.ResourcesTask;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
@ -97,6 +98,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceTaskMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
@ -142,9 +144,11 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@ -229,6 +233,9 @@ public class ProcessServiceImpl implements ProcessService {
@Autowired
private ResourceMapper resourceMapper;
@Autowired
private ResourceTaskMapper resourceTaskMapper;
@Autowired
private ResourceUserMapper resourceUserMapper;
@ -1376,7 +1383,7 @@ public class ProcessServiceImpl implements ProcessService {
ResourceInfo mainJar = JSONUtils.parseObject(
JSONUtils.toJsonString(mainJarObj),
ResourceInfo.class);
ResourceInfo resourceInfo = updateResourceInfo(mainJar);
ResourceInfo resourceInfo = updateResourceInfo(taskDefinition.getId(), mainJar);
if (resourceInfo != null) {
taskParameters.put("mainJar", resourceInfo);
}
@ -1387,7 +1394,7 @@ public class ProcessServiceImpl implements ProcessService {
List<ResourceInfo> resourceInfos = JSONUtils.toList(resourceListStr, ResourceInfo.class);
List<ResourceInfo> updatedResourceInfos = resourceInfos
.stream()
.map(this::updateResourceInfo)
.map(resourceInfo -> updateResourceInfo(taskDefinition.getId(), resourceInfo))
.filter(Objects::nonNull)
.collect(Collectors.toList());
taskParameters.put("resourceList", updatedResourceInfos);
@ -1403,21 +1410,23 @@ public class ProcessServiceImpl implements ProcessService {
* @param res origin resource info
* @return {@link ResourceInfo}
*/
protected ResourceInfo updateResourceInfo(ResourceInfo res) {
protected ResourceInfo updateResourceInfo(int task_id, ResourceInfo res) {
ResourceInfo resourceInfo = null;
// only if mainJar is not null and does not contains "resourceName" field
// only if mainJar is not null and does not contain "resourceName" field
if (res != null) {
Integer resourceId = res.getId();
if (resourceId == null) {
logger.error("invalid resourceId, {}", resourceId);
return null;
String resourceFullName = res.getResourceName();
if (StringUtils.isBlank(resourceFullName)) {
logger.error("invalid resource full name, {}", resourceFullName);
return new ResourceInfo();
}
resourceInfo = new ResourceInfo();
// get resource from database, only one resource should be returned
Resource resource = getResourceById(resourceId);
resourceInfo.setId(resourceId);
resourceInfo.setRes(resource.getFileName());
resourceInfo.setResourceName(resource.getFullName());
Integer resultList = resourceTaskMapper.existResourceByTaskIdNFullName(task_id, resourceFullName);
if (resultList != null) {
resourceInfo.setId(resultList);
resourceInfo.setRes(res.getRes());
resourceInfo.setResourceName(resourceFullName);
}
if (logger.isInfoEnabled()) {
logger.info("updated resource info {}",
JSONUtils.toJsonString(resourceInfo));
@ -2033,7 +2042,6 @@ public class ProcessServiceImpl implements ProcessService {
taskDefinitionLog.setUpdateTime(now);
taskDefinitionLog.setOperateTime(now);
taskDefinitionLog.setOperator(operator.getId());
taskDefinitionLog.setResourceIds(getResourceIds(taskDefinitionLog));
if (taskDefinitionLog.getCode() == 0) {
taskDefinitionLog.setCode(CodeGenerateUtils.getInstance().genCode());
}
@ -2074,6 +2082,8 @@ public class ProcessServiceImpl implements ProcessService {
TaskDefinition task = taskDefinitionMap.get(taskDefinitionToUpdate.getCode());
if (task == null) {
newTaskDefinitionLogs.add(taskDefinitionToUpdate);
} else {
taskDefinitionToUpdate.setId(task.getId());
}
}
}
@ -2091,9 +2101,42 @@ public class ProcessServiceImpl implements ProcessService {
if (CollectionUtils.isNotEmpty(newTaskDefinitionLogs) && Boolean.TRUE.equals(syncDefine)) {
updateResult += taskDefinitionMapper.batchInsert(newTaskDefinitionLogs);
for (TaskDefinitionLog newTaskDefinitionLog : newTaskDefinitionLogs) {
Set<String> resourceFullNameSet = getResourceFullNames(newTaskDefinitionLog);
for (String resourceFullName : resourceFullNameSet) {
List<TaskDefinition> taskDefinitionList = taskDefinitionMapper.selectByMap(
Collections.singletonMap("code", newTaskDefinitionLog.getCode()));
if (taskDefinitionList.size() > 0) {
createRelationTaskResourcesIfNotExist(
taskDefinitionList.get(0).getId(), resourceFullName);
}
}
}
}
if (CollectionUtils.isNotEmpty(updateTaskDefinitionLogs) && Boolean.TRUE.equals(syncDefine)) {
for (TaskDefinitionLog taskDefinitionLog : updateTaskDefinitionLogs) {
Set<String> resourceFullNameSet = getResourceFullNames(taskDefinitionLog);
// remove resources that user deselected.
for (ResourcesTask resourcesTask : resourceTaskMapper.selectByMap(
Collections.singletonMap("task_id",
taskDefinitionMapper.queryByCode(taskDefinitionLog.getCode()).getId()))) {
if (!resourceFullNameSet.contains(resourcesTask.getFullName())) {
resourceTaskMapper.deleteById(resourcesTask.getId());
}
}
for (String resourceFullName : resourceFullNameSet) {
List<TaskDefinition> taskDefinitionList = taskDefinitionMapper.selectByMap(
Collections.singletonMap("code", taskDefinitionLog.getCode()));
if (taskDefinitionList.size() > 0) {
createRelationTaskResourcesIfNotExist(
taskDefinitionList.get(0).getId(), resourceFullName);
}
}
updateResult += taskDefinitionMapper.updateById(taskDefinitionLog);
}
}
@ -2682,4 +2725,36 @@ public class ProcessServiceImpl implements ProcessService {
return testDataSourceId;
return null;
}
private Set<String> getResourceFullNames(TaskDefinition taskDefinition) {
Set<String> resourceFullNames = null;
AbstractParameters params = taskPluginManager.getParameters(ParametersNode.builder()
.taskType(taskDefinition.getTaskType()).taskParams(taskDefinition.getTaskParams()).build());
if (params != null && CollectionUtils.isNotEmpty(params.getResourceFilesList())) {
resourceFullNames = params.getResourceFilesList().stream()
.filter(t -> !StringUtils.isBlank(t.getResourceName()))
.map(ResourceInfo::getResourceName)
.collect(toSet());
}
if (CollectionUtils.isEmpty(resourceFullNames)) {
return new HashSet<String>();
}
return resourceFullNames;
}
private Integer createRelationTaskResourcesIfNotExist(int taskId, String resourceFullName) {
Integer resourceId = resourceTaskMapper.existResourceByTaskIdNFullName(taskId, resourceFullName);
if (null == resourceId) {
// create the relation if not exist
ResourcesTask resourcesTask = new ResourcesTask(taskId, resourceFullName, ResourceType.FILE);
resourceTaskMapper.insert(resourcesTask);
return resourcesTask.getId();
}
return resourceId;
}
}

66
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageEntity.java

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package org.apache.dolphinscheduler.service.storage;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import java.util.Date;
import lombok.Data;
// StorageEneity is an entity representing a resource in the third-part storage service.
// It is only stored in t_ds_relation_resources_task if the resource is used by a task.
// It is not put in the model module because it has more attributes than corresponding objects stored
// in table t_ds_relation_resources_task.
@Data
public class StorageEntity {
/**
* exist only if it is stored in t_ds_relation_resources_task.
*
*/
private int id;
/**
* fullname is in a format of basepath + tenantCode + res/udf + filename
*/
private String fullName;
/**
* filename is in a format of possible parent folders + alias
*/
private String fileName;
/**
* the name of the file
*/
private String alias;
/**
* parent folder time
*/
private String pfullName;
private boolean isDirectory;
private String description;
private int userId;
private String userName;
private ResourceType type;
private long size;
private Date createTime;
private Date updateTime;
}

34
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageOperate.java

@ -68,6 +68,13 @@ public interface StorageOperate {
*/
String getResourceFileName(String tenantCode, String fullName);
/**
* get the path of the resource file excluding the base path.
* @param fullName
* @return
*/
String getResourceFileName(String fullName);
/**
* get the path of the file
* @param resourceType
@ -79,23 +86,23 @@ public interface StorageOperate {
/**
* predicate if the resource of tenant exists
* @param tenantCode
* @param fileName
* @param fullName
* @return
* @throws IOException
*/
boolean exists(String tenantCode, String fileName) throws IOException;
boolean exists(String fullName) throws IOException;
/**
* delete the resource of filePath
* todo if the filePath is the type of directory ,the files in the filePath need to be deleted at all
* @param tenantCode
* @param filePath
* @param recursive
* @return
* @throws IOException
*/
boolean delete(String tenantCode, String filePath, boolean recursive) throws IOException;
boolean delete(String filePath, boolean recursive) throws IOException;
boolean delete(String filePath, List<String> childrenPathArray, boolean recursive) throws IOException;
/**
* copy the file from srcPath to dstPath
@ -167,4 +174,21 @@ public interface StorageOperate {
*/
ResUploadType returnStorageType();
/**
* return files and folders in the current directory and subdirectories
* */
public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
ResourceType type);
/**
* return files and folders in the current directory
* */
public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception;
/**
* return a file status
* */
public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception;
}

245
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtils.java

@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.KerberosHttpClient;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.service.storage.StorageEntity;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.dolphinscheduler.service.utils.CommonUtils;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
@ -48,12 +49,17 @@ import org.apache.hadoop.security.UserGroupInformation;
import java.io.BufferedReader;
import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@ -64,6 +70,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Joiner;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@ -71,6 +78,7 @@ import com.google.common.cache.LoadingCache;
/**
* hadoop utils
* single instance
* By default, directory path does NOT end with '/'
*/
public class HadoopUtils implements Closeable, StorageOperate {
@ -185,7 +193,11 @@ public class HadoopUtils implements Closeable, StorageOperate {
* @return DefaultFS
*/
public String getDefaultFS() {
return getConfiguration().get(Constants.FS_DEFAULT_FS);
String defaultFS = getConfiguration().get(Constants.FS_DEFAULT_FS);
if (StringUtils.isBlank(defaultFS)) {
defaultFS = PropertyUtils.getString(Constants.FS_DEFAULT_FS);
}
return defaultFS;
}
/**
@ -271,12 +283,12 @@ public class HadoopUtils implements Closeable, StorageOperate {
@Override
public String getResDir(String tenantCode) {
return getHdfsResDir(tenantCode);
return getHdfsResDir(tenantCode) + FOLDER_SEPARATOR;
}
@Override
public String getUdfDir(String tenantCode) {
return getHdfsUdfDir(tenantCode);
return getHdfsUdfDir(tenantCode) + FOLDER_SEPARATOR;
}
/**
@ -290,7 +302,7 @@ public class HadoopUtils implements Closeable, StorageOperate {
*/
@Override
public boolean mkdir(String tenantCode, String hdfsPath) throws IOException {
return fs.mkdirs(new Path(hdfsPath));
return fs.mkdirs(new Path(addFolderSeparatorIfNotExisted(hdfsPath)));
}
@Override
@ -298,6 +310,26 @@ public class HadoopUtils implements Closeable, StorageOperate {
return getHdfsResourceFileName(tenantCode, fullName);
}
@Override
public String getResourceFileName(String fullName) {
// here is a quick fix here to get fileName. We get the resource upload path and
// get the index of the first appearance of resource upload path. The index is put
// in the start index of the substring function and get the result substring containing
// tenantcode and "resource" directory and the fileName.
// Then we split the result substring
// with "/" and join all elements except the first two elements because they are
// tenantCode and "resource" directory.
String resourceUploadPath =
RESOURCE_UPLOAD_PATH.endsWith(FOLDER_SEPARATOR) ? StringUtils.chop(RESOURCE_UPLOAD_PATH)
: RESOURCE_UPLOAD_PATH;
// +1 because we want to skip the "/" after resource upload path as well.
String pathContainingTenantNResource = fullName.substring(
fullName.indexOf(resourceUploadPath)
+ resourceUploadPath.length() + 1);
String[] fileNameArr = pathContainingTenantNResource.split(FOLDER_SEPARATOR);
return Joiner.on(FOLDER_SEPARATOR).join(Arrays.stream(fileNameArr).skip(2).collect(Collectors.toList()));
}
@Override
public String getFileName(ResourceType resourceType, String tenantCode, String fileName) {
return getHdfsFileName(resourceType, tenantCode, fileName);
@ -351,7 +383,7 @@ public class HadoopUtils implements Closeable, StorageOperate {
return copyLocalToHdfs(srcFile, dstPath, deleteSource, overwrite);
}
/*
/**
* copy hdfs file to local
*
* @param srcHdfsFilePath source hdfs file path
@ -368,6 +400,7 @@ public class HadoopUtils implements Closeable, StorageOperate {
*/
public boolean copyHdfsToLocal(String srcHdfsFilePath, String dstFile, boolean deleteSource,
boolean overwrite) throws IOException {
Path srcPath = new Path(srcHdfsFilePath);
File dstPath = new File(dstFile);
@ -399,10 +432,29 @@ public class HadoopUtils implements Closeable, StorageOperate {
* @throws IOException errors
*/
@Override
public boolean delete(String tenantCode, String hdfsFilePath, boolean recursive) throws IOException {
public boolean delete(String hdfsFilePath, boolean recursive) throws IOException {
return fs.delete(new Path(hdfsFilePath), recursive);
}
/**
* delete a list of files
*
* @param filePath the path to delete, usually it is a directory.
* @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception. In
* case of a file the recursive can be set to either true or false.
* @return true if delete is successful else false.
* @throws IOException errors
*/
@Override
public boolean delete(String filePath, List<String> childrenPathArray, boolean recursive) throws IOException {
if (filePath.endsWith("/")) {
return fs.delete(new Path(filePath), true);
}
return fs.delete(new Path(filePath), recursive);
}
/**
* check if exists
*
@ -411,23 +463,119 @@ public class HadoopUtils implements Closeable, StorageOperate {
* @throws IOException errors
*/
@Override
public boolean exists(String tenantCode, String hdfsFilePath) throws IOException {
public boolean exists(String hdfsFilePath) throws IOException {
return fs.exists(new Path(hdfsFilePath));
}
/**
* Gets a list of files in the directory
*
* @param filePath file path
* @param path file fullName path
* @return {@link FileStatus} file status
* @throws IOException errors
*/
public FileStatus[] listFileStatus(String filePath) throws IOException {
@Override
public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws IOException {
// TODO: Does listStatus truncate resultList if its size goes above certain threshold (like a 1000 in S3)
// TODO: add hdfs prefix getFile
List<StorageEntity> storageEntityList = new ArrayList<>();
try {
FileStatus[] fileStatuses = fs.listStatus(new Path(path));
// transform FileStatusArray into the StorageEntity List
for (FileStatus fileStatus : fileStatuses) {
if (fileStatus.isDirectory()) {
// the path is a directory
String fullName = fileStatus.getPath().toString();
fullName = addFolderSeparatorIfNotExisted(fullName);
String suffix = StringUtils.difference(path, fullName);
String fileName = StringUtils.difference(defaultPath, fullName);
StorageEntity entity = new StorageEntity();
entity.setAlias(suffix);
entity.setFileName(fileName);
entity.setFullName(fullName);
entity.setDirectory(true);
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(fileStatus.getLen());
entity.setCreateTime(new Date(fileStatus.getModificationTime()));
entity.setUpdateTime(new Date(fileStatus.getModificationTime()));
entity.setPfullName(path);
storageEntityList.add(entity);
} else {
// the path is a file
String fullName = fileStatus.getPath().toString();
String[] aliasArr = fullName.split("/");
String alias = aliasArr[aliasArr.length - 1];
String fileName = StringUtils.difference(defaultPath, fullName);
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(fullName);
entity.setDirectory(false);
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(fileStatus.getLen());
entity.setCreateTime(new Date(fileStatus.getModificationTime()));
entity.setUpdateTime(new Date(fileStatus.getModificationTime()));
entity.setPfullName(path);
storageEntityList.add(entity);
}
}
} catch (FileNotFoundException e) {
throw new FileNotFoundException("The path does not exist.");
} catch (IOException e) {
throw new IOException("Get file list exception.", e);
}
return storageEntityList;
}
@Override
public StorageEntity getFileStatus(String path, String prefix, String tenantCode,
ResourceType type) throws IOException {
try {
return fs.listStatus(new Path(filePath));
FileStatus fileStatus = fs.getFileStatus(new Path(path));
String alias = "";
String fileName = "";
String fullName = fileStatus.getPath().toString();
if (fileStatus.isDirectory()) {
fullName = addFolderSeparatorIfNotExisted(fullName);
alias = findDirAlias(fullName);
fileName = StringUtils.difference(prefix, fullName);
} else {
String[] aliasArr = fileStatus.getPath().toString().split("/");
alias = aliasArr[aliasArr.length - 1];
fileName = StringUtils.difference(prefix, fileStatus.getPath().toString());
}
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(fullName);
entity.setDirectory(fileStatus.isDirectory());
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(fileStatus.getLen());
entity.setCreateTime(new Date(fileStatus.getModificationTime()));
entity.setUpdateTime(new Date(fileStatus.getModificationTime()));
entity.setPfullName(path);
return entity;
} catch (FileNotFoundException e) {
throw new FileNotFoundException("The path does not exist.");
} catch (IOException e) {
logger.error("Get file list exception", e);
throw new IOException("Get file list exception", e);
throw new IOException("Get file exception.", e);
}
}
@ -528,10 +676,12 @@ public class HadoopUtils implements Closeable, StorageOperate {
* @return data hdfs path
*/
public static String getHdfsDataBasePath() {
String defaultFS = PropertyUtils.getString(Constants.FS_DEFAULT_FS);
defaultFS = defaultFS.endsWith("/") ? StringUtils.chop(defaultFS) : defaultFS;
if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) {
return "";
return defaultFS + "";
} else {
return RESOURCE_UPLOAD_PATH;
return defaultFS + RESOURCE_UPLOAD_PATH;
}
}
@ -739,8 +889,8 @@ public class HadoopUtils implements Closeable, StorageOperate {
public void deleteTenant(String tenantCode) throws Exception {
String tenantPath = getHdfsDataBasePath() + FOLDER_SEPARATOR + tenantCode;
if (exists(tenantCode, tenantPath)) {
delete(tenantCode, tenantPath, true);
if (exists(tenantPath)) {
delete(tenantPath, true);
}
}
@ -749,4 +899,67 @@ public class HadoopUtils implements Closeable, StorageOperate {
public ResUploadType returnStorageType() {
return ResUploadType.HDFS;
}
@Override
public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
ResourceType type) {
List<StorageEntity> storageEntityList = new ArrayList<>();
LinkedList<StorageEntity> foldersToFetch = new LinkedList<>();
do {
String pathToExplore = "";
if (foldersToFetch.size() == 0) {
pathToExplore = path;
} else {
pathToExplore = foldersToFetch.pop().getFullName();
}
try {
List<StorageEntity> tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type);
for (StorageEntity temp : tempList) {
if (temp.isDirectory()) {
foldersToFetch.add(temp);
}
}
storageEntityList.addAll(tempList);
} catch (FileNotFoundException e) {
logger.error("Resource path: {}", pathToExplore, e);
// return the resources fetched before error occurs.
return storageEntityList;
} catch (IOException e) {
logger.error("Resource path: {}", pathToExplore, e);
// return the resources fetched before error occurs.
return storageEntityList;
}
} while (foldersToFetch.size() != 0);
return storageEntityList;
}
/**
* find alias for directories, NOT for files
* a directory is a path ending with "/"
*/
private String findDirAlias(String myStr) {
if (!myStr.endsWith("/")) {
// Make sure system won't crush down if someone accidentally misuse the function.
return myStr;
}
int lastIndex = myStr.lastIndexOf("/");
String subbedString = myStr.substring(0, lastIndex);
int secondLastIndex = subbedString.lastIndexOf("/");
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(myStr, secondLastIndex + 1, lastIndex + 1);
return stringBuilder.toString();
}
private String addFolderSeparatorIfNotExisted(String fullName) {
return fullName.endsWith(FOLDER_SEPARATOR) ? fullName : fullName + FOLDER_SEPARATOR;
}
}

33
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/OssOperator.java

@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.common.factory.OssClientFactory;
import org.apache.dolphinscheduler.common.model.OssConnection;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.service.storage.StorageEntity;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
@ -175,6 +176,11 @@ public class OssOperator implements Closeable, StorageOperate {
return String.format(FORMAT_S_S, getOssResDir(tenantCode), fileName);
}
@Override
public String getResourceFileName(String fullName) {
return null;
}
@Override
public String getFileName(ResourceType resourceType, String tenantCode, String fileName) {
if (fileName.startsWith(FOLDER_SEPARATOR)) {
@ -183,6 +189,11 @@ public class OssOperator implements Closeable, StorageOperate {
return getDir(resourceType, tenantCode) + fileName;
}
@Override
public boolean delete(String filePath, List<String> childrenPathArray, boolean recursive) throws IOException {
return false;
}
@Override
public void download(String tenantCode, String srcFilePath, String dstFilePath, boolean deleteSource,
boolean overwrite) throws IOException {
@ -210,12 +221,12 @@ public class OssOperator implements Closeable, StorageOperate {
}
@Override
public boolean exists(String tenantCode, String fileName) throws IOException {
public boolean exists(String fileName) throws IOException {
return ossClient.doesObjectExist(bucketName, fileName);
}
@Override
public boolean delete(String tenantCode, String filePath, boolean recursive) throws IOException {
public boolean delete(String filePath, boolean recursive) throws IOException {
try {
ossClient.deleteObject(bucketName, filePath);
return true;
@ -274,6 +285,24 @@ public class OssOperator implements Closeable, StorageOperate {
return ResUploadType.OSS;
}
@Override
public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
ResourceType type) {
return null;
}
@Override
public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception {
return null;
}
@Override
public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception {
return null;
}
@Override
public void deleteTenant(String tenantCode) throws Exception {
deleteTenantCode(tenantCode);

252
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java

@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.service.storage.StorageEntity;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
@ -44,7 +45,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -61,14 +65,22 @@ import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.MultipleFileDownload;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.google.common.base.Joiner;
/**
* By default, directory path does end with '/'
*/
public class S3Utils implements Closeable, StorageOperate {
private static final Logger logger = LoggerFactory.getLogger(S3Utils.class);
@ -171,6 +183,26 @@ public class S3Utils implements Closeable, StorageOperate {
return String.format(FORMAT_S_S, getS3ResDir(tenantCode), fileName);
}
@Override
public String getResourceFileName(String fullName) {
// here is a quick fix here to get fileName. We get the resource upload path and
// get the index of the first appearance of resource upload path. The index is put
// in the start index of the substring function and get the result substring containing
// tenantcode and "resource" directory and the fileName.
// Then we split the result substring
// with "/" and join all elements except the first two elements because they are
// tenantCode and "resource" directory.
String resourceUploadPath =
RESOURCE_UPLOAD_PATH.endsWith(FOLDER_SEPARATOR) ? StringUtils.chop(RESOURCE_UPLOAD_PATH)
: RESOURCE_UPLOAD_PATH;
// +1 because we want to skip the "/" after resource upload path as well.
String pathContainingTenantNResource = fullName.substring(
fullName.indexOf(resourceUploadPath)
+ resourceUploadPath.length() + 1);
String[] fileNameArr = pathContainingTenantNResource.split(FOLDER_SEPARATOR);
return Joiner.on(FOLDER_SEPARATOR).join(Arrays.stream(fileNameArr).skip(2).collect(Collectors.toList()));
}
@Override
public String getFileName(ResourceType resourceType, String tenantCode, String fileName) {
if (fileName.startsWith(FOLDER_SEPARATOR)) {
@ -206,21 +238,38 @@ public class S3Utils implements Closeable, StorageOperate {
}
@Override
public boolean exists(String tenantCode, String fileName) throws IOException {
return s3Client.doesObjectExist(BUCKET_NAME, fileName);
public boolean exists(String fullName) throws IOException {
return s3Client.doesObjectExist(BUCKET_NAME, fullName);
}
@Override
public boolean delete(String tenantCode, String filePath, boolean recursive) throws IOException {
public boolean delete(String fullName, boolean recursive) throws IOException {
try {
s3Client.deleteObject(BUCKET_NAME, filePath);
s3Client.deleteObject(BUCKET_NAME, fullName);
return true;
} catch (AmazonServiceException e) {
logger.error("delete the object error,the resource path is {}", filePath);
logger.error("delete the object error,the resource path is {}", fullName);
return false;
}
}
@Override
public boolean delete(String fullName, List<String> childrenPathList, boolean recursive) throws IOException {
// append the resource fullName to the list for deletion.
childrenPathList.add(fullName);
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(BUCKET_NAME)
.withKeys(childrenPathList.stream().toArray(String[]::new));
try {
s3Client.deleteObjects(deleteObjectsRequest);
} catch (AmazonServiceException e) {
logger.error("delete objects error", e);
return false;
}
return true;
}
@Override
public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException {
s3Client.copyObject(BUCKET_NAME, srcPath, BUCKET_NAME, dstPath);
@ -384,4 +433,197 @@ public class S3Utils implements Closeable, StorageOperate {
public ResUploadType returnStorageType() {
return ResUploadType.S3;
}
@Override
public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
ResourceType type) {
List<StorageEntity> storageEntityList = new ArrayList<>();
LinkedList<StorageEntity> foldersToFetch = new LinkedList<>();
do {
String pathToExplore = "";
if (foldersToFetch.size() == 0) {
pathToExplore = path;
} else {
pathToExplore = foldersToFetch.pop().getFullName();
}
try {
List<StorageEntity> tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type);
for (StorageEntity temp : tempList) {
if (temp.isDirectory()) {
foldersToFetch.add(temp);
}
}
storageEntityList.addAll(tempList);
} catch (AmazonServiceException e) {
logger.error("Resource path: {}", pathToExplore, e);
// return the resources fetched before error occurs.
return storageEntityList;
}
} while (foldersToFetch.size() != 0);
return storageEntityList;
}
@Override
public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws AmazonServiceException {
List<StorageEntity> storageEntityList = new ArrayList<>();
// TODO: optimize pagination
ListObjectsV2Request request = new ListObjectsV2Request();
request.setBucketName(BUCKET_NAME);
request.setPrefix(path);
request.setDelimiter("/");
ListObjectsV2Result v2Result;
do {
try {
v2Result = s3Client.listObjectsV2(request);
} catch (AmazonServiceException e) {
throw new AmazonServiceException("Get S3 file list exception, error type:" + e.getErrorType(), e);
}
List<S3ObjectSummary> summaries = v2Result.getObjectSummaries();
for (S3ObjectSummary summary : summaries) {
if (!summary.getKey().endsWith("/")) {
// the path is a file
String[] aliasArr = summary.getKey().split("/");
String alias = aliasArr[aliasArr.length - 1];
String fileName = StringUtils.difference(defaultPath, summary.getKey());
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(summary.getKey());
entity.setDirectory(false);
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(summary.getSize());
entity.setCreateTime(summary.getLastModified());
entity.setUpdateTime(summary.getLastModified());
entity.setPfullName(path);
storageEntityList.add(entity);
}
}
for (String commonPrefix : v2Result.getCommonPrefixes()) {
// the paths in commonPrefix are directories
String suffix = StringUtils.difference(path, commonPrefix);
String fileName = StringUtils.difference(defaultPath, commonPrefix);
StorageEntity entity = new StorageEntity();
entity.setAlias(suffix);
entity.setFileName(fileName);
entity.setFullName(commonPrefix);
entity.setDirectory(true);
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(0);
entity.setCreateTime(null);
entity.setUpdateTime(null);
entity.setPfullName(path);
storageEntityList.add(entity);
}
request.setContinuationToken(v2Result.getContinuationToken());
} while (v2Result.isTruncated());
return storageEntityList;
}
@Override
public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws AmazonServiceException, FileNotFoundException {
// Notice: we do not use getObject here because intermediate directories
// may not exist in S3, which can cause getObject to throw exception.
// Since we still want to access it on frontend, this is a workaround using listObjects.
ListObjectsV2Request request = new ListObjectsV2Request();
request.setBucketName(BUCKET_NAME);
request.setPrefix(path);
request.setDelimiter("/");
ListObjectsV2Result v2Result;
try {
v2Result = s3Client.listObjectsV2(request);
} catch (AmazonServiceException e) {
throw new AmazonServiceException("Get S3 file list exception, error type:" + e.getErrorType(), e);
}
List<S3ObjectSummary> summaries = v2Result.getObjectSummaries();
if (path.endsWith("/")) {
// the path is a directory that may or may not exist in S3
String alias = findDirAlias(path);
String fileName = StringUtils.difference(defaultPath, path);
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(path);
entity.setDirectory(true);
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(0);
return entity;
} else {
// the path is a file
if (summaries.size() > 0) {
S3ObjectSummary summary = summaries.get(0);
String[] aliasArr = summary.getKey().split("/");
String alias = aliasArr[aliasArr.length - 1];
String fileName = StringUtils.difference(defaultPath, summary.getKey());
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(summary.getKey());
entity.setDirectory(false);
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(summary.getSize());
entity.setCreateTime(summary.getLastModified());
entity.setUpdateTime(summary.getLastModified());
return entity;
}
}
throw new FileNotFoundException("Object is not found in S3 Bucket: " + BUCKET_NAME);
}
/**
* find alias for directories, NOT for files
* a directory is a path ending with "/"
*/
private String findDirAlias(String myStr) {
if (!myStr.endsWith("/")) {
// Make sure system won't crush down if someone accidentally misuse the function.
return myStr;
}
int lastIndex = myStr.lastIndexOf("/");
String subbedString = myStr.substring(0, lastIndex);
int secondLastIndex = subbedString.lastIndexOf("/");
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(myStr, secondLastIndex + 1, lastIndex + 1);
return stringBuilder.toString();
}
}

21
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java

@ -40,7 +40,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@ -59,6 +58,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceTaskMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper;
@ -149,6 +149,8 @@ public class ProcessServiceTest {
@Mock
private ResourceMapper resourceMapper;
@Mock
private ResourceTaskMapper resourceTaskMapper;
@Mock
private TaskGroupMapper taskGroupMapper;
@Mock
private DataSourceMapper dataSourceMapper;
@ -616,6 +618,7 @@ public class ProcessServiceTest {
Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(taskDefinition.getCode())).thenReturn(1);
Mockito.when(taskDefinitionMapper.queryByCodeList(Collections.singletonList(taskDefinition.getCode())))
.thenReturn(Collections.singletonList(taskDefinition));
Mockito.when(taskDefinitionMapper.queryByCode(Mockito.anyLong())).thenReturn(taskDefinition);
int result = processService.saveTaskDefine(operator, projectCode, taskDefinitionLogs, Boolean.TRUE);
Assertions.assertEquals(0, result);
}
@ -693,24 +696,22 @@ public class ProcessServiceTest {
public void testUpdateResourceInfo() throws Exception {
// test if input is null
ResourceInfo resourceInfoNull = null;
ResourceInfo updatedResourceInfo1 = processService.updateResourceInfo(resourceInfoNull);
ResourceInfo updatedResourceInfo1 = processService.updateResourceInfo(0, resourceInfoNull);
Assertions.assertNull(updatedResourceInfo1);
// test if resource id less than 1
ResourceInfo resourceInfoVoid = new ResourceInfo();
ResourceInfo updatedResourceInfo2 = processService.updateResourceInfo(resourceInfoVoid);
Assertions.assertNull(updatedResourceInfo2);
ResourceInfo updatedResourceInfo2 = processService.updateResourceInfo(0, resourceInfoVoid);
Assertions.assertNull(updatedResourceInfo2.getResourceName());
// test normal situation
ResourceInfo resourceInfoNormal = new ResourceInfo();
resourceInfoNormal.setId(1);
Resource resource = new Resource();
resource.setId(1);
resource.setFileName("test.txt");
resource.setFullName("/test.txt");
Mockito.when(resourceMapper.selectById(1)).thenReturn(resource);
resourceInfoNormal.setRes("test.txt");
resourceInfoNormal.setResourceName("/test.txt");
Mockito.when(resourceTaskMapper.existResourceByTaskIdNFullName(0, "/test.txt")).thenReturn(1);
ResourceInfo updatedResourceInfo3 = processService.updateResourceInfo(resourceInfoNormal);
ResourceInfo updatedResourceInfo3 = processService.updateResourceInfo(0, resourceInfoNormal);
Assertions.assertEquals(1, updatedResourceInfo3.getId().intValue());
Assertions.assertEquals("test.txt", updatedResourceInfo3.getRes());

4
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java

@ -169,7 +169,7 @@ public class OssOperatorTest {
boolean doesExist = false;
doReturn(true).when(ossClientMock).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK);
try {
doesExist = ossOperator.exists(TENANT_CODE_MOCK, FILE_NAME_MOCK);
doesExist = ossOperator.exists(FILE_NAME_MOCK);
} catch (IOException e) {
Assertions.fail("unexpected IO exception in unit test");
}
@ -183,7 +183,7 @@ public class OssOperatorTest {
boolean isDeleted = false;
doReturn(null).when(ossClientMock).deleteObject(anyString(), anyString());
try {
isDeleted = ossOperator.delete(TENANT_CODE_MOCK, FILE_NAME_MOCK, true);
isDeleted = ossOperator.delete(FILE_NAME_MOCK, true);
} catch (IOException e) {
Assertions.fail("unexpected IO exception in unit test");
}

4
dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java

@ -572,9 +572,7 @@ public class SqlTask extends AbstractTask {
String prefixPath = defaultFS.startsWith("file://") ? "file://" : defaultFS;
String uploadPath = CommonUtils.getHdfsUdfDir(value.getTenantCode());
String resourceFullName = value.getResourceName();
resourceFullName =
resourceFullName.startsWith("/") ? resourceFullName : String.format("/%s", resourceFullName);
return String.format("add jar %s%s%s", prefixPath, uploadPath, resourceFullName);
return String.format("add jar %s", resourceFullName);
}).collect(Collectors.toList());
}

5
dolphinscheduler-ui/src/locales/en_US/resource.ts

@ -24,6 +24,7 @@ export default {
enter_keyword_tips: 'Please enter keyword',
name: 'Name',
user_name: 'Resource userName',
tenant_name: 'Resource tenantName',
whether_directory: 'Whether directory',
file_name: 'File Name',
description: 'Description',
@ -58,6 +59,7 @@ export default {
upload_udf_resources: 'Upload UDF Resources',
udf_source_name: 'UDF Resource Name',
user_name: 'Resource userName',
tenant_name: 'Resource tenantName',
whether_directory: 'Whether directory',
file_name: 'File Name',
file_size: 'File Size',
@ -68,6 +70,7 @@ export default {
yes: 'Yes',
no: 'No',
edit: 'Edit',
rename: 'Rename',
download: 'Download',
delete: 'Delete',
delete_confirm: 'Delete?',
@ -83,7 +86,7 @@ export default {
function: {
udf_function: 'UDF Function',
create_udf_function: 'Create UDF Function',
edit_udf_function: 'Create UDF Function',
edit_udf_function: 'Edit UDF Function',
udf_function_name: 'UDF Function Name',
user_name: 'Resource userName',
class_name: 'Class Name',

10
dolphinscheduler-ui/src/router/modules/resources.ts

@ -53,7 +53,7 @@ export default {
}
},
{
path: '/resource/file/edit/:id',
path: '/resource/file/edit',
name: 'resource-file-edit',
component: components['resource-file-edit'],
meta: {
@ -65,7 +65,7 @@ export default {
}
},
{
path: '/resource/file/subdirectory/:id',
path: '/resource/file/subdirectory',
name: 'resource-file-subdirectory',
component: components['resource-file'],
meta: {
@ -77,7 +77,7 @@ export default {
}
},
{
path: '/resource/file/list/:id',
path: '/resource/file/list',
name: 'resource-file-list',
component: components['resource-file-edit'],
meta: {
@ -89,7 +89,7 @@ export default {
}
},
{
path: '/resource/file/create/:id',
path: '/resource/file/create',
name: 'resource-subfile-create',
component: components['resource-file-create'],
meta: {
@ -112,7 +112,7 @@ export default {
}
},
{
path: '/resource/resource-manage/:id',
path: '/resource/resource-manage',
name: 'resource-sub-manage',
component: components['resource-udf-resource'],
meta: {

64
dolphinscheduler-ui/src/service/modules/resources/index.ts

@ -22,6 +22,7 @@ import {
NameReq,
FileNameReq,
FullNameReq,
TenantCodeReq,
IdReq,
ContentReq,
DescriptionReq,
@ -31,12 +32,11 @@ import {
ProgramTypeReq,
ListReq,
ViewResourceReq,
ResourceIdReq,
UdfFuncReq
} from './types'
export function queryResourceListPaging(
params: ListReq & IdReq & ResourceTypeReq
params: ListReq & ResourceTypeReq & FullNameReq & TenantCodeReq
): any {
return axios({
url: '/resources',
@ -45,21 +45,23 @@ export function queryResourceListPaging(
})
}
export function queryResourceById(
params: ResourceTypeReq & FullNameReq & IdReq,
id: number
export function queryCurrentResourceByFileName(
params: ResourceTypeReq & FileNameReq & TenantCodeReq,
): any {
return axios({
url: `/resources/${id}`,
url: `/resources/query-file-name`,
method: 'get',
params
})
}
export function queryCurrentResourceById(id: number): any {
export function queryCurrentResourceByFullName(
params: ResourceTypeReq & FullNameReq & TenantCodeReq,
): any {
return axios({
url: `/resources/${id}/query`,
method: 'get'
url: `/resources/query-full-name`,
method: 'get',
params
})
}
@ -107,7 +109,7 @@ export function createDirectory(
})
}
export function queryResourceList(params: ResourceTypeReq): any {
export function queryResourceList(params: ResourceTypeReq & FullNameReq): any {
return axios({
url: '/resources/list',
method: 'get',
@ -159,10 +161,11 @@ export function verifyUdfFuncName(params: NameReq): any {
})
}
export function deleteUdfFunc(id: number): any {
export function deleteUdfFunc(id: number, params: FullNameReq): any {
return axios({
url: `/resources/udf-func/${id}`,
method: 'delete'
method: 'delete',
params
})
}
@ -182,37 +185,36 @@ export function verifyResourceName(params: FullNameReq & ResourceTypeReq): any {
})
}
export function queryResource(
export function doesResourceExist(
params: FullNameReq & ResourceTypeReq,
id: IdReq
): any {
return axios({
url: `/resources/verify-name/${id}`,
url: `/resources/verify-name`,
method: 'get',
params
})
}
export function updateResource(
data: NameReq & ResourceTypeReq & IdReq & DescriptionReq,
id: number
data: NameReq & ResourceTypeReq & DescriptionReq & FullNameReq & TenantCodeReq,
): any {
return axios({
url: `/resources/${id}`,
url: `/resources`,
method: 'put',
data
})
}
export function deleteResource(id: number): any {
export function deleteResource(params: FullNameReq & TenantCodeReq): any {
return axios({
url: `/resources/${id}`,
method: 'delete'
url: `/resources`,
method: 'delete',
params
})
}
export function downloadResource(id: number): void {
utils.downloadFile(`resources/${id}/download`)
export function downloadResource(params: FullNameReq): void {
utils.downloadFile(`resources/download`, params)
}
export function viewUIUdfFunction(id: IdReq): any {
@ -222,28 +224,27 @@ export function viewUIUdfFunction(id: IdReq): any {
})
}
export function updateResourceContent(data: ContentReq, id: number): any {
export function updateResourceContent(data: ContentReq & TenantCodeReq & FullNameReq): any {
return axios({
url: `/resources/${id}/update-content`,
url: `/resources/update-content`,
method: 'put',
data
})
}
export function viewResource(params: ViewResourceReq, id: number): any {
export function viewResource(params: ViewResourceReq & FullNameReq & TenantCodeReq): any {
return axios({
url: `/resources/${id}/view`,
url: `/resources/view`,
method: 'get',
params
})
}
export function createUdfFunc(
data: UdfFuncReq,
resourceId: ResourceIdReq
data: UdfFuncReq
): any {
return axios({
url: `/resources/${resourceId}/udf-func`,
url: `/resources/udf-func`,
method: 'post',
data
})
@ -251,11 +252,10 @@ export function createUdfFunc(
export function updateUdfFunc(
data: UdfFuncReq,
resourceId: ResourceIdReq,
id: number
): any {
return axios({
url: `/resources/${resourceId}/udf-func/${id}`,
url: `/resources/udf-func/${id}`,
method: 'put',
data
})

12
dolphinscheduler-ui/src/service/modules/resources/types.ts

@ -36,6 +36,10 @@ interface FileNameReq {
fileName: string
}
interface TenantCodeReq{
tenantCode: string
}
interface FullNameReq {
fullName: string
}
@ -80,11 +84,7 @@ interface ViewResourceReq {
skipLineNum: number
}
interface ResourceIdReq {
resourceId: number
}
interface UdfFuncReq extends UdfTypeReq, DescriptionReq, ResourceIdReq {
interface UdfFuncReq extends UdfTypeReq, DescriptionReq, FullNameReq {
className: string
funcName: string
argTypes?: string
@ -126,6 +126,7 @@ export {
NameReq,
FileNameReq,
FullNameReq,
TenantCodeReq,
IdReq,
ContentReq,
DescriptionReq,
@ -135,7 +136,6 @@ export {
ProgramTypeReq,
ListReq,
ViewResourceReq,
ResourceIdReq,
UdfFuncReq,
ResourceListRes,
ResourceViewRes,

2
dolphinscheduler-ui/src/utils/tree-format.ts

@ -16,7 +16,7 @@
*/
const removeUselessChildren = (
list: { children?: []; dirctory?: boolean; disabled?: boolean }[]
list: { children?: []; directory?: boolean; disabled?: boolean; dirctory?: boolean }[]
) => {
if (!list.length) return
list.forEach((item) => {

4
dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-resources.ts

@ -41,7 +41,7 @@ export function useResources(
}
if (resourcesLoading.value) return
resourcesLoading.value = true
const res = await queryResourceList({ type: 'FILE' })
const res = await queryResourceList({ type: 'FILE', fullName:"" })
utils.removeUselessChildren(res)
resourcesOptions.value = res || []
resourcesLoading.value = false
@ -65,7 +65,7 @@ export function useResources(
showPath: true,
checkStrategy: 'child',
placeholder: t('project.node.resources_tips'),
keyField: 'id',
keyField: 'fullName',
labelField: 'name',
loading: resourcesLoading
},

11
dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts

@ -42,7 +42,7 @@ export function formatParams(data: INodeData): {
taskParams.jvmArgs = data.jvmArgs
taskParams.isModulePath = data.isModulePath
if (data.runType === 'JAR' && data.mainJar) {
taskParams.mainJar = { id: data.mainJar }
taskParams.mainJar = { resourceName: data.mainJar }
}
}
@ -53,7 +53,7 @@ export function formatParams(data: INodeData): {
taskParams.programType = data.programType
taskParams.mainClass = data.mainClass
if (data.mainJar) {
taskParams.mainJar = { id: data.mainJar }
taskParams.mainJar = { resourceName: data.mainJar }
}
taskParams.deployMode = data.deployMode
taskParams.appName = data.appName
@ -488,7 +488,7 @@ export function formatParams(data: INodeData): {
initScript: data.initScript,
rawScript: data.rawScript,
resourceList: data.resourceList?.length
? data.resourceList.map((id: number) => ({ id }))
? data.resourceList.map((fullName: string) => ({ resourceName: `${fullName}` }))
: [],
...taskParams
},
@ -511,7 +511,6 @@ export function formatParams(data: INodeData): {
params.taskDefinitionJsonObj.timeout = 0
params.taskDefinitionJsonObj.timeoutNotifyStrategy = ''
}
return params
}
@ -537,11 +536,11 @@ export function formatModel(data: ITaskData) {
}
if (data.taskParams?.resourceList) {
params.resourceList = data.taskParams.resourceList.map(
(item: { id: number }) => item.id
(item: { resourceName: string }) => (`${item.resourceName}`)
)
}
if (data.taskParams?.mainJar) {
params.mainJar = data.taskParams?.mainJar.id
params.mainJar = data.taskParams?.mainJar.resourceName
}
if (data.taskParams?.method) {

12
dolphinscheduler-ui/src/views/projects/task/components/node/types.ts

@ -90,8 +90,14 @@ interface ISwitchResult {
nextNode?: number
}
/*
* resourceName: resource full name
* res: resource file name
*/
interface ISourceItem {
id: number
id?: number,
resourceName: string,
res?: string
}
interface ISqoopTargetData {
@ -429,8 +435,8 @@ interface INodeData
preTasks?: number[]
preTaskOptions?: []
postTaskOptions?: []
resourceList?: number[]
mainJar?: number
resourceList?: string[]
mainJar?: string
timeoutSetting?: boolean
isCustomTask?: boolean
method?: string

1
dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-modal.ts

@ -125,7 +125,6 @@ export function useModal(
params.startParams = !_.isEmpty(startParams)
? JSON.stringify(startParams)
: ''
await startProcessInstance(params, variables.projectCode)
window.$message.success(t('project.workflow.success'))
state.saving = false

4
dolphinscheduler-ui/src/views/resource/file/create/index.tsx

@ -49,9 +49,7 @@ export default defineComponent({
}
const handleReturn = () => {
const { id } = router.currentRoute.value.params
const name = id ? 'resource-file-subdirectory' : 'file'
router.push({ name, params: { id } })
router.go(-1)
}
const trim = getCurrentInstance()?.appContext.config.globalProperties.trim

7
dolphinscheduler-ui/src/views/resource/file/create/use-create.ts

@ -27,18 +27,17 @@ export function useCreate(state: any) {
const fileStore = useFileStore()
const handleCreateFile = () => {
const pid = router.currentRoute.value.params.id || -1
// no more pid, as currentDir acts as the pid or parent path right now.
const currentDir = fileStore.getCurrentDir || '/'
state.fileFormRef.validate(async (valid: any) => {
if (!valid) {
await onlineCreateResource({
...state.fileForm,
...{ pid, currentDir }
...{ currentDir }
})
window.$message.success(t('resource.file.success'))
const name = pid ? 'resource-file-subdirectory' : 'file'
router.push({ name, params: { id: pid } })
router.go(-1)
}
})
}

8
dolphinscheduler-ui/src/views/resource/file/edit/index.tsx

@ -32,21 +32,23 @@ export default defineComponent({
const router = useRouter()
const componentName = route.name
const fileId = Number(route.params.id)
// fullname is now the id of resources
const fullName = String(router.currentRoute.value.query.prefix || "")
const tenantCode = String(router.currentRoute.value.query.tenantCode || "")
const { state } = useForm()
const { getResourceView, handleUpdateContent } = useEdit(state)
const handleFileContent = () => {
state.fileForm.content = resourceViewRef.state.value.content
handleUpdateContent(fileId)
handleUpdateContent(fullName, tenantCode)
}
const handleReturn = () => {
router.go(-1)
}
const resourceViewRef = getResourceView(fileId)
const resourceViewRef = getResourceView(fullName, tenantCode)
watch(
() => resourceViewRef.state.value.content,
() => (state.fileForm.content = resourceViewRef.state.value.content)

15
dolphinscheduler-ui/src/views/resource/file/edit/use-edit.ts

@ -28,25 +28,28 @@ export function useEdit(state: any) {
const { t } = useI18n()
const router: Router = useRouter()
const getResourceView = (id: number) => {
const getResourceView = (fullName: string, tenantCode: string) => {
const params = {
skipLineNum: 0,
limit: 3000
limit: 3000,
fullName: fullName,
tenantCode: tenantCode
}
return useAsyncState(viewResource(params, id), {
return useAsyncState(viewResource(params), {
alias: '',
content: ''
})
}
const handleUpdateContent = (id: number) => {
const handleUpdateContent = (fullName: string, tenantCode: string) => {
state.fileFormRef.validate(async (valid: any) => {
if (!valid) {
await updateResourceContent(
{
...state.fileForm
...state.fileForm,
tenantCode: tenantCode,
fullName: fullName,
},
id
)
window.$message.success(t('resource.file.success'))

104
dolphinscheduler-ui/src/views/resource/file/index.tsx

@ -42,8 +42,8 @@ import { useFileState } from './use-file'
import { BreadcrumbItem, IRenameFile } from './types'
import { useFileStore } from '@/store/file/file'
import {
queryCurrentResourceById,
queryResourceById
queryCurrentResourceByFullName,
queryCurrentResourceByFileName
} from '@/service/modules/resources'
import Card from '@/components/card'
import ResourceFolderModal from './folder'
@ -57,8 +57,8 @@ export default defineComponent({
name: 'File',
setup() {
const router: Router = useRouter()
const fileId = ref(Number(router.currentRoute.value.params.id) || -1)
const fullName = ref(String(router.currentRoute.value.query.prefix || ""))
const tenantCode = ref(String(router.currentRoute.value.query.tenantCode || ""))
const resourceListRef = ref()
const folderShowRef = ref(false)
const uploadShowRef = ref(false)
@ -66,9 +66,10 @@ export default defineComponent({
const searchRef = ref()
const renameInfo = reactive({
id: -1,
name: '',
description: ''
description: '',
fullName: '',
user_name: ''
})
const paginationReactive = reactive({
@ -81,7 +82,8 @@ export default defineComponent({
const handleUpdatePage = (page: number) => {
paginationReactive.page = page
resourceListRef.value = getResourceListState(
fileId.value,
fullName.value,
tenantCode.value,
searchRef.value,
paginationReactive.page,
paginationReactive.pageSize
@ -92,7 +94,8 @@ export default defineComponent({
paginationReactive.page = 1
paginationReactive.pageSize = pageSize
resourceListRef.value = getResourceListState(
fileId.value,
fullName.value,
tenantCode.value,
searchRef.value,
paginationReactive.page,
paginationReactive.pageSize
@ -111,7 +114,8 @@ export default defineComponent({
const handleConditions = () => {
resourceListRef.value = getResourceListState(
fileId.value,
fullName.value,
tenantCode.value,
searchRef.value
)
}
@ -121,12 +125,12 @@ export default defineComponent({
}
const handleCreateFile = () => {
const name = fileId.value
const name = fullName.value
? 'resource-subfile-create'
: 'resource-file-create'
router.push({
name,
params: { id: fileId.value }
params: { id: fullName.value }
})
}
@ -134,10 +138,11 @@ export default defineComponent({
handleShowModal(uploadShowRef)
}
const handleRenameFile: IRenameFile = (id, name, description) => {
renameInfo.id = id
const handleRenameFile: IRenameFile = (name: string, description: string, fullName: string, user_name: string) => {
renameInfo.fullName = fullName
renameInfo.name = name
renameInfo.description = description
renameInfo.user_name = user_name
handleShowModal(renameShowRef)
}
@ -149,75 +154,75 @@ export default defineComponent({
const updateList = () => {
resourceListRef.value = getResourceListState(
fileId.value,
fullName.value,
tenantCode.value,
searchRef.value
)
}
const fileStore = useFileStore()
onMounted(() => {
resourceListRef.value = getResourceListState(fileId.value)
resourceListRef.value = getResourceListState(fullName.value, tenantCode.value,searchRef.value)
})
const breadcrumbItemsRef: Ref<Array<BreadcrumbItem> | undefined> = ref([
{
id: 1,
fullName: 'l1'
fullName: 'l1',
userName: 'u1'
},
{
id: 2,
fullName: 'l2'
fullName: 'l2',
userName: 'u2'
},
{
id: 4,
fullName: 'l3'
fullName: 'l3',
userName: 'u3'
}
])
const trim = getCurrentInstance()?.appContext.config.globalProperties.trim
onMounted(() => {
const currFileId = Number(router.currentRoute.value.params.id) || -1
if (currFileId === -1) {
const currfullName = String(router.currentRoute.value.query.prefix || "")
if (currfullName === "") {
fileStore.setCurrentDir('/')
} else {
queryCurrentResourceById(currFileId).then((res: ResourceFile) => {
if (res.fullName) {
fileStore.setCurrentDir(res.fullName)
}
})
fileStore.setCurrentDir(currfullName)
}
})
const initBreadcrumb = async (dirs: string[]) => {
let index = 0
for (const dir of dirs) {
for (let index = 0; index < dirs.length; index ++) {
const newDir = dirs.slice(0, index + 1).join('/')
if (newDir) {
const id = 0
const resource = await queryResourceById(
{
id,
type: 'FILE',
fullName: newDir
},
id
)
breadcrumbItemsRef.value?.push({ id: resource.id, fullName: dir })
} else {
breadcrumbItemsRef.value?.push({ id: 0, fullName: 'Root' })
const resource = await queryCurrentResourceByFileName(
{
type: 'FILE',
fileName: newDir+"/",
tenantCode: tenantCode.value
}
)
breadcrumbItemsRef.value?.push({ id: resource.fullName, fullName: resource.alias, userName: resource.userName })
}
index = index + 1
}
}
onMounted(() => {
breadcrumbItemsRef.value = []
if (fileId.value != -1) {
queryCurrentResourceById(fileId.value).then((res: ResourceFile) => {
if (res.fullName) {
const dirs = res.fullName.split('/')
if (fullName.value != "") {
breadcrumbItemsRef.value?.push({ id: 0, fullName: 'Root', userName: '' })
queryCurrentResourceByFullName(
{
type: 'FILE',
fullName: fullName.value,
tenantCode: tenantCode.value,
}
).then((res: ResourceFile) => {
if (res.fileName) {
const dirs = res.fileName.split('/')
if (dirs && dirs.length > 1) {
dirs.pop()
initBreadcrumb(dirs)
}
}
@ -226,7 +231,7 @@ export default defineComponent({
})
return {
fileId,
fullName,
searchRef,
folderShowRef,
uploadShowRef,
@ -307,7 +312,7 @@ export default defineComponent({
)
} else {
return (
<NBreadcrumbItem href={item.id.toString()}>
<NBreadcrumbItem href={"0?prefix=" + item.id.toString() + "&tenantCode=" + item.userName}>
{item.fullName}
</NBreadcrumbItem>
)
@ -353,9 +358,10 @@ export default defineComponent({
/>
<ResourceRenameModal
v-model:show={this.renameShowRef}
id={this.renameInfo.id}
name={this.renameInfo.name}
fullName={this.renameInfo.fullName}
description={this.renameInfo.description}
userName={this.renameInfo.user_name}
onUpdateList={this.updateList}
/>
</NSpace>

20
dolphinscheduler-ui/src/views/resource/file/rename/index.tsx

@ -32,10 +32,6 @@ const props = {
type: Boolean as PropType<boolean>,
default: false
},
id: {
type: Number as PropType<number>,
default: -1
},
name: {
type: String as PropType<string>,
default: ''
@ -43,7 +39,15 @@ const props = {
description: {
type: String as PropType<string>,
default: ''
}
},
fullName: {
type: String as PropType<string>,
default: ''
},
userName: {
type: String as PropType<string>,
default: ''
},
}
export default defineComponent({
@ -51,9 +55,8 @@ export default defineComponent({
props,
emits: ['updateList', 'update:show'],
setup(props, ctx) {
const { state, resetForm } = useForm(props.name, props.description)
const { state, resetForm } = useForm(props.fullName, props.name, props.description, props.userName)
const { handleRenameFile } = useRename(state)
const hideModal = () => {
ctx.emit('update:show', false)
}
@ -67,9 +70,10 @@ export default defineComponent({
watch(
() => props.show,
() => {
state.renameForm.id = props.id
state.renameForm.fullName = props.fullName
state.renameForm.name = props.name
state.renameForm.description = props.description
state.renameForm.user_name = props.userName
}
)

11
dolphinscheduler-ui/src/views/resource/file/rename/use-form.ts

@ -19,23 +19,24 @@ import { reactive, ref, unref } from 'vue'
import { useI18n } from 'vue-i18n'
import type { FormRules } from 'naive-ui'
const defaultValue = (name = '', description = '') => ({
const defaultValue = (fullName = '',name = '', description = '', user_name = '') => ({
id: -1,
fullName,
name,
type: 'FILE',
description
description,
user_name
})
export function useForm(name: string, description: string) {
export function useForm(fullName: string, name: string, description: string, user_name: string) {
const { t } = useI18n()
const resetForm = () => {
state.renameForm = Object.assign(unref(state.renameForm), defaultValue())
}
const state = reactive({
renameFormRef: ref(),
renameForm: defaultValue(name, description),
renameForm: defaultValue(fullName, name, description, user_name),
saving: false,
rules: {
name: {

8
dolphinscheduler-ui/src/views/resource/file/rename/use-rename.ts

@ -28,16 +28,14 @@ export function useRename(state: any) {
resetForm: () => void
) => {
await state.renameFormRef.validate()
if (state.saving) return
state.saving = true
try {
await updateResource(
{
...state.renameForm
},
state.renameForm.id
...state.renameForm,
tenantCode: state.renameForm.user_name,
}
)
window.$message.success(t('resource.file.success'))
state.saving = false

34
dolphinscheduler-ui/src/views/resource/file/table/table-action.tsx

@ -27,10 +27,9 @@ import {
} from '@vicons/antd'
import _ from 'lodash'
import { useI18n } from 'vue-i18n'
import { ResourceFileTableData } from '../types'
import { ResourceFileTableData, IRenameFile, IRtDisb } from '../types'
import { fileTypeArr } from '@/common/common'
import { downloadResource, deleteResource } from '@/service/modules/resources'
import { IRenameFile, IRtDisb } from '../types'
import type { Router } from 'vue-router'
const props = {
@ -41,9 +40,9 @@ const props = {
row: {
type: Object as PropType<ResourceFileTableData>,
default: {
id: -1,
name: '',
description: ''
description: '',
user_name: '',
}
}
}
@ -63,16 +62,16 @@ export default defineComponent({
return !(flag && size < 1000000)
}
const handleEditFile = (item: { id: number }) => {
router.push({ name: 'resource-file-edit', params: { id: item.id } })
const handleEditFile = (item: {fullName: string, user_name: string }) => {
router.push({ name: 'resource-file-edit', query: {prefix: item.fullName, tenantCode: item.user_name} })
}
const handleDeleteFile = (id: number) => {
deleteResource(id).then(() => emit('updateList'))
const handleDeleteFile = (fullNameObj: {fullName: string, tenantCode: string}) => {
deleteResource(fullNameObj).then(() => emit('updateList'))
}
const handleRenameFile: IRenameFile = (id, name, description) => {
emit('renameResource', id, name, description)
const handleRenameFile: IRenameFile = (name: string, description: string, fullName: string, user_name: string) => {
emit('renameResource', name, description, fullName, user_name)
}
return {
@ -98,7 +97,7 @@ export default defineComponent({
disabled={this.rtDisb(this.row.name, this.row.size)}
tag='div'
onClick={() => {
this.handleEditFile(this.row)
this.handleEditFile({fullName:this.row.fullName, user_name:this.row.user_name})
}}
style={{ marginRight: '-5px' }}
circle
@ -118,13 +117,16 @@ export default defineComponent({
<NButton
size='tiny'
type='info'
onClick={() =>
onClick={() => {
this.handleRenameFile(
this.row.id,
this.row.name,
this.row.description
this.row.description,
this.row.fullName,
this.row.user_name
)
}
}
style={{ marginRight: '-5px' }}
circle
class='btn-rename'
@ -147,7 +149,7 @@ export default defineComponent({
tag='div'
circle
style={{ marginRight: '-5px' }}
onClick={() => downloadResource(this.row.id)}
onClick={() => downloadResource({fullName: this.row.fullName})}
class='btn-download'
>
<NIcon>
@ -166,7 +168,7 @@ export default defineComponent({
positive-text={t('resource.file.confirm')}
negative-text={t('resource.file.cancel')}
onPositiveClick={() => {
this.handleDeleteFile(this.row.id)
this.handleDeleteFile({fullName: this.row.fullName, tenantCode: this.row.user_name})
}}
>
{{

19
dolphinscheduler-ui/src/views/resource/file/table/use-table.ts

@ -35,12 +35,11 @@ import type { TableColumns } from 'naive-ui/es/data-table/src/interface'
const goSubFolder = (router: Router, item: any) => {
const fileStore = useFileStore()
fileStore.setFileInfo(`${item.alias}|${item.size}`)
if (item.directory) {
fileStore.setCurrentDir(`${item.fullName}`)
router.push({ name: 'resource-file-subdirectory', params: { id: item.id } })
router.push({ name: 'resource-file-subdirectory', query: { prefix: item.fullName, tenantCode: item.user_name}})
} else {
router.push({ name: 'resource-file-list', params: { id: item.id } })
router.push({ name: 'resource-file-list', query: {prefix: item.fullName, tenantCode: item.user_name} })
}
}
@ -72,7 +71,7 @@ export function useTable(renameResource: IRenameFile, updateList: () => void) {
)
},
{
title: t('resource.file.user_name'),
title: t('resource.file.tenant_name'),
...COLUMN_WIDTH_CONFIG['userName'],
key: 'user_name'
},
@ -86,12 +85,7 @@ export function useTable(renameResource: IRenameFile, updateList: () => void) {
{
title: t('resource.file.file_name'),
...COLUMN_WIDTH_CONFIG['name'],
key: 'file_name'
},
{
title: t('resource.file.description'),
...COLUMN_WIDTH_CONFIG['note'],
key: 'description'
key: 'fullName'
},
{
title: t('resource.file.size'),
@ -110,8 +104,9 @@ export function useTable(renameResource: IRenameFile, updateList: () => void) {
render: (row) =>
h(TableAction, {
row,
onRenameResource: (id, name, description) =>
renameResource(id, name, description),
onRenameResource: ( name, description, fullName, user_name ) => {
renameResource(name, description, fullName, user_name)
},
onUpdateList: () => updateList()
}),
...COLUMN_WIDTH_CONFIG['operation'](4)

7
dolphinscheduler-ui/src/views/resource/file/types.ts

@ -16,8 +16,8 @@
*/
export interface ResourceFileTableData {
id: number
name: string
fullName: string
user_name: string
directory: string
file_name: string
@ -31,14 +31,14 @@ export interface IEmit {
}
export interface IRenameFile {
(id: number, name: string, description: string): void
(name: string, description: string, fullName: string, user_name: string): void
}
export interface IRtDisb {
(name: string, size: number): boolean
}
export interface IResourceListState {
(id?: number, searchVal?: string, pageNo?: number, pageSize?: number): any
(searchVal?: string, fullName?: string, tenantCode?: string, pageNo?: number, pageSize?: number): any
}
export interface BasicTableProps {
@ -68,4 +68,5 @@ export interface ISetPagination {
export interface BreadcrumbItem {
id: number
fullName: string
userName: string
}

6
dolphinscheduler-ui/src/views/resource/file/upload/use-upload.ts

@ -17,14 +17,11 @@
import { useI18n } from 'vue-i18n'
import { IEmit } from '../types'
import { useRouter } from 'vue-router'
import type { Router } from 'vue-router'
import { useFileStore } from '@/store/file/file'
import { createResource } from '@/service/modules/resources'
export function useUpload(state: any) {
const { t } = useI18n()
const router: Router = useRouter()
const fileStore = useFileStore()
const handleUploadFile = async (
@ -37,13 +34,12 @@ export function useUpload(state: any) {
if (state.saving) return
state.saving = true
try {
const pid = router.currentRoute.value.params.id || -1
// no more pid, as currentDir acts as the pid or parent path right now.
const currentDir = fileStore.getCurrentDir || '/'
const formData = new FormData()
formData.append('file', state.uploadForm.file)
formData.append('type', 'FILE')
formData.append('name', state.uploadForm.name)
formData.append('pid', String(pid))
formData.append('currentDir', currentDir)
formData.append('description', state.uploadForm.description)

14
dolphinscheduler-ui/src/views/resource/file/use-file.ts

@ -27,14 +27,16 @@ export function useFileState(
setPagination: ISetPagination = {} as ISetPagination
) {
const getResourceListState: IResourceListState = (
id = -1,
fullName = '',
tenantCode = '',
searchVal = '',
pageNo = 1,
pageSize = 10
) => {
const { state } = useAsyncState(
queryResourceListPaging({
id,
fullName,
tenantCode,
type: 'FILE',
searchVal,
pageNo,
@ -66,12 +68,14 @@ export function useFileState(
return state
}
const getResourceView = (id: number) => {
const getResourceView = (fullName: string, tenantCode: string) => {
const params = {
skipLineNum: 0,
limit: 3000
limit: 3000,
fullName: fullName,
tenantCode: tenantCode
}
const { state } = useAsyncState(viewResource(params, id), {})
const { state } = useAsyncState(viewResource(params), {})
return state
}

13
dolphinscheduler-ui/src/views/resource/udf/function/components/function-modal.tsx

@ -106,6 +106,7 @@ export default defineComponent({
state.functionForm.funcName = props.row.funcName
state.functionForm.className = props.row.className
state.functionForm.resourceId = props.row.resourceId || -1
state.functionForm.fullName = props.row.resourceName || ""
state.functionForm.description = props.row.description
}
)
@ -124,7 +125,6 @@ export default defineComponent({
},
render() {
const { t } = useI18n()
return (
<Modal
show={this.$props.show}
@ -173,18 +173,19 @@ export default defineComponent({
</NFormItem>
<NFormItem
label={t('resource.function.udf_resources')}
path='resourceId'
path='fullName'
>
<NInputGroup>
<NTreeSelect
options={this.udfResourceList}
label-field='fullName'
key-field='id'
v-model={[this.functionForm.resourceId, 'value']}
label-field='name'
key-field='fullName'
check-strategy='child'
v-model={[this.functionForm.fullName, 'value']}
placeholder={t(
'resource.function.enter_select_udf_resources_tips'
)}
defaultValue={this.functionForm.resourceId}
defaultValue={this.functionForm.fullName}
disabled={this.uploadShow}
showPath={false}
class='btn-udf-resource-dropdown'

14
dolphinscheduler-ui/src/views/resource/udf/function/components/use-form.ts

@ -31,7 +31,8 @@ export const useForm = () => {
argTypes: '',
database: '',
description: '',
resourceId: -1
resourceId: -1,
fullName: ''
},
saving: false,
rules: {
@ -63,13 +64,22 @@ export const useForm = () => {
}
},
resourceId: {
required: true,
required: false,
trigger: ['input', 'blur'],
validator() {
if (state.functionForm.resourceId === -1) {
return new Error(t('resource.function.enter_name_tips'))
}
}
},
fullName: {
required: true,
trigger: ['input', 'blur'],
validator() {
if (state.functionForm.fullName == "") {
return new Error(t('resource.function.enter_name_tips'))
}
}
}
} as FormRules
})

9
dolphinscheduler-ui/src/views/resource/udf/function/components/use-modal.ts

@ -39,8 +39,7 @@ export function useModal(
await createUdfFunc(
{
...state.functionForm
},
state.functionForm.resourceId
}
)
)
}
@ -52,7 +51,6 @@ export function useModal(
...state.functionForm,
id
},
state.functionForm.resourceId,
id
)
})
@ -84,9 +82,6 @@ export function useModal(
const filterEmptyDirectory = (list: any) => {
for (const item of list) {
if (item.children) {
if (!/\.jar$/.test(item.name)) {
item.disabled = true
}
filterEmptyDirectory(item.children)
}
}
@ -122,7 +117,7 @@ export function useModal(
const getUdfList = () => {
const { state } = useAsyncState(
queryResourceList({ type: 'UDF' }).then((res: any) => {
queryResourceList({ type: 'UDF', fullName: "" }).then((res: any) => {
let item = res
let item1 = _.cloneDeep(res)

1
dolphinscheduler-ui/src/views/resource/udf/function/index.tsx

@ -47,6 +47,7 @@ export default defineComponent({
const requestData = () => {
getTableData({
id: variables.id,
fullName: variables.fullName,
pageSize: variables.pageSize,
pageNo: variables.page,
searchVal: variables.searchVal

1
dolphinscheduler-ui/src/views/resource/udf/function/types.ts

@ -16,6 +16,7 @@
*/
export interface IUdfFunctionParam {
fullName: string
id: number
pageSize: number
pageNo: number

9
dolphinscheduler-ui/src/views/resource/udf/function/use-table.ts

@ -43,7 +43,9 @@ export function useTable() {
tableWidth: DefaultTableWidth,
row: {},
tableData: [],
// here is id not prefix because udf function is still stored in db
id: ref(Number(router.currentRoute.value.params.id) || -1),
fullName: ref(String(router.currentRoute.value.query.prefix || "")),
page: ref(1),
pageSize: ref(10),
searchVal: ref(),
@ -129,7 +131,7 @@ export function useTable() {
NPopconfirm,
{
onPositiveClick: () => {
handleDelete(row.id)
handleDelete(row.id, row.fullName)
}
},
{
@ -188,14 +190,15 @@ export function useTable() {
variables.row = row
}
const handleDelete = (id: number) => {
const handleDelete = (id: number, fullName: string) => {
/* after deleting data from the current page, you need to jump forward when the page is empty. */
if (variables.tableData.length === 1 && variables.page > 1) {
variables.page -= 1
}
deleteUdfFunc(id).then(() =>
deleteUdfFunc(id, {fullName: fullName}).then(() =>
getTableData({
fullName: variables.fullName,
id: variables.id,
pageSize: variables.pageSize,
pageNo: variables.page,

8
dolphinscheduler-ui/src/views/resource/udf/resource/components/folder-modal.tsx

@ -60,7 +60,7 @@ export default defineComponent({
}
const handleRename = () => {
handleRenameResource(props.row.id)
handleRenameResource(props.row.fullName)
}
const trim = getCurrentInstance()?.appContext.config.globalProperties.trim
@ -72,7 +72,7 @@ export default defineComponent({
state.folderForm.description = props.row.description
}
)
const fileEdit = computed(() => props.row.id && !props.row.directory)
const fileEdit = computed(() => props.row.fullName && !props.row.directory)
return {
fileEdit,
@ -90,10 +90,10 @@ export default defineComponent({
<Modal
show={this.$props.show}
title={
this.row.id ? t('resource.udf.edit') : t('resource.udf.create_folder')
this.row.fullName ? t('resource.udf.edit') : t('resource.udf.create_folder')
}
onCancel={this.hideModal}
onConfirm={this.row.id ? this.handleRename : this.handleCreate}
onConfirm={this.row.fullName ? this.handleRename : this.handleCreate}
confirmClassName='btn-submit'
cancelClassName='btn-cancel'
confirmLoading={this.saving}

17
dolphinscheduler-ui/src/views/resource/udf/resource/components/use-modal.ts

@ -36,8 +36,8 @@ export function useModal(
const handleCreateResource = async () => {
const pid = router.currentRoute.value.params.id || -1
const currentDir = pid === -1 ? '/' : fileStore.getCurrentDir || '/'
const currentFullName = String(router.currentRoute.value.query.prefix || "")
const currentDir = currentFullName == "" ? '/' : fileStore.getCurrentDir || '/'
submitRequest(
async () =>
await createDirectory({
@ -47,21 +47,19 @@ export function useModal(
)
}
const handleRenameResource = async (id: number) => {
const handleRenameResource = async (fullName: string) => {
submitRequest(async () => {
await updateResource(
{
fullName: fullName,
...state.folderForm,
...{ id }
},
id
}
)
})
}
const submitRequest = async (serviceHandle: any) => {
await state.folderFormRef.validate()
if (state.saving) return
state.saving = true
@ -89,8 +87,9 @@ export function useModal(
state.saving = true
try {
const pid = router.currentRoute.value.params.id || -1
const currentDir = pid === -1 ? '/' : fileStore.getCurrentDir || '/'
const pid = Number(router.currentRoute.value.params.id) || "-1"
const currentFullName = String(router.currentRoute.value.query.prefix || "")
const currentDir = currentFullName == "" ? '/' : fileStore.getCurrentDir || '/'
const formData = new FormData()
formData.append('file', state.uploadForm.file)

6
dolphinscheduler-ui/src/views/resource/udf/resource/index.tsx

@ -50,7 +50,9 @@ export default defineComponent({
const requestData = () => {
getTableData({
id: variables.id,
id: -1,
fullName: variables.fullName,
tenantCode: variables.tenantCode,
pageSize: variables.pageSize,
pageNo: variables.page,
searchVal: variables.searchVal
@ -88,7 +90,7 @@ export default defineComponent({
let breadName = ''
variables.breadList.forEach((item, i) => {
if (i <= index) {
breadName = breadName + '/' + item
breadName = breadName === "" ? item.toString() : breadName + '/' + item.toString();
}
})
goBread(breadName)

2
dolphinscheduler-ui/src/views/resource/udf/resource/types.ts

@ -16,6 +16,8 @@
*/
export interface IUdfResourceParam {
fullName: string
tenantCode: string
id: number
pageSize: number
pageNo: number

74
dolphinscheduler-ui/src/views/resource/udf/resource/use-table.ts

@ -30,7 +30,8 @@ import {
queryResourceListPaging,
downloadResource,
deleteResource,
queryResourceById
queryCurrentResourceByFileName,
queryCurrentResourceByFullName
} from '@/service/modules/resources'
import ButtonLink from '@/components/button-link'
import {
@ -39,14 +40,15 @@ import {
DefaultTableWidth
} from '@/common/column-width-config'
import type { IUdfResourceParam } from './types'
import { ResourceFile } from '@/service/modules/resources/types'
const goSubFolder = (router: Router, item: any) => {
const fileStore = useFileStore()
fileStore.setFileInfo(`${item.alias}|${item.size}`)
if (item.directory) {
fileStore.setCurrentDir(`${item.fullName}`)
router.push({ name: 'resource-sub-manage', params: { id: item.id } })
router.push({ name: 'resource-sub-manage',
query: {prefix: item.fullName, tenantCode: item.userName} })
}
}
@ -60,8 +62,9 @@ export function useTable() {
tableWidth: DefaultTableWidth,
row: {},
tableData: [],
breadList: [],
id: ref(Number(router.currentRoute.value.params.id) || -1),
breadList: [] as String[],
fullName: ref(String(router.currentRoute.value.query.prefix || "")),
tenantCode: ref(String(router.currentRoute.value.query.tenantCode || "")),
page: ref(1),
pageSize: ref(10),
searchVal: ref(),
@ -103,7 +106,7 @@ export function useTable() {
}
},
{
title: t('resource.udf.user_name'),
title: t('resource.udf.tenant_name'),
...COLUMN_WIDTH_CONFIG['userName'],
key: 'userName'
},
@ -117,7 +120,7 @@ export function useTable() {
{
title: t('resource.udf.file_name'),
...COLUMN_WIDTH_CONFIG['name'],
key: 'fileName'
key: 'fullName'
},
{
title: t('resource.udf.file_size'),
@ -125,11 +128,6 @@ export function useTable() {
...COLUMN_WIDTH_CONFIG['size'],
render: (row) => bytesToSize(row.size)
},
{
title: t('resource.udf.description'),
key: 'description',
...COLUMN_WIDTH_CONFIG['note']
},
{
title: t('resource.udf.create_time'),
key: 'createTime',
@ -185,7 +183,7 @@ export function useTable() {
size: 'tiny',
class: 'btn-download',
disabled: row?.directory ? true : false,
onClick: () => downloadResource(row.id)
onClick: () => downloadResource({fullName: row.fullName})
},
{
icon: () => h(DownloadOutlined)
@ -198,7 +196,7 @@ export function useTable() {
NPopconfirm,
{
onPositiveClick: () => {
handleDelete(row.id)
handleDelete({fullName: row.fullName, tenantCode: row.userName})
}
},
{
@ -242,13 +240,25 @@ export function useTable() {
variables.loadingRef = true
const { state } = useAsyncState(
queryResourceListPaging({ ...params, type: 'UDF' }).then((res: any) => {
const breadList =
variables.id === -1
? []
: (fileStore.getCurrentDir.split('/') as Array<never>)
breadList.shift()
variables.breadList = breadList
// use strict checking here
if (variables.fullName !== ""){
queryCurrentResourceByFullName(
{
type: 'UDF',
fullName: variables.fullName,
tenantCode: variables.tenantCode,
}
).then((res: ResourceFile) => {
if (res.fileName) {
const breadList = res.fileName.split('/')
// pop the alias from the fullname path
breadList.pop()
variables.breadList = breadList
}
})
} else {
variables.breadList = []
}
variables.totalPage = res.totalPage
variables.tableData = res.totalList.map((item: any) => {
return { ...item }
@ -265,15 +275,17 @@ export function useTable() {
variables.row = row
}
const handleDelete = (id: number) => {
const handleDelete = (fullNameObj: {fullName: string, tenantCode: string}) => {
/* after deleting data from the current page, you need to jump forward when the page is empty. */
if (variables.tableData.length === 1 && variables.page > 1) {
variables.page -= 1
}
deleteResource(id).then(() =>
deleteResource(fullNameObj).then(() =>
getTableData({
id: variables.id,
id: -1,
fullName: variables.fullName,
tenantCode: variables.tenantCode,
pageSize: variables.pageSize,
pageNo: variables.page,
searchVal: variables.searchVal
@ -285,18 +297,16 @@ export function useTable() {
router.push({ name: 'resource-manage' })
}
const goBread = (fullName: string) => {
const { id } = variables
queryResourceById(
const goBread = (fileName: string) => {
queryCurrentResourceByFileName(
{
id,
type: 'UDF',
fullName
},
id
fileName: fileName + "/",
tenantCode: variables.tenantCode
}
).then((res: any) => {
fileStore.setCurrentDir(res.fullName)
router.push({ name: 'resource-sub-manage', params: { id: res.id } })
router.push({ name: 'resource-sub-manage', query: {prefix: res.fullName, tenantCode: res.userName} })
})
}

12
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java

@ -118,18 +118,20 @@ public class TaskExecutionCheckerUtils {
if (CollectionUtils.isNotEmpty(downloadFiles)) {
for (Pair<String, String> fileDownload : downloadFiles) {
try {
// query the tenant code of the resource according to the name of the resource
String fullName = fileDownload.getLeft();
// we do not actually get & need tenantCode with this implementation right now.
String tenantCode = fileDownload.getRight();
String resPath = storageOperate.getResourceFileName(tenantCode, fullName);
logger.info("get resource file from path:{}", resPath);
// TODO: Need a better way to get fileName because this implementation is tricky.
String fileName = storageOperate.getResourceFileName(fullName);
logger.info("get resource file from path:{}", fullName);
long resourceDownloadStartTime = System.currentTimeMillis();
storageOperate.download(tenantCode, resPath, execLocalPath + File.separator + fullName, false,
storageOperate.download(tenantCode, fullName, execLocalPath + File.separator + fileName, false,
true);
WorkerServerMetrics
.recordWorkerResourceDownloadTime(System.currentTimeMillis() - resourceDownloadStartTime);
WorkerServerMetrics.recordWorkerResourceDownloadSize(
Files.size(Paths.get(execLocalPath, fullName)));
Files.size(Paths.get(execLocalPath, fileName)));
WorkerServerMetrics.incWorkerResourceDownloadSuccessCount();
} catch (Exception e) {
WorkerServerMetrics.incWorkerResourceDownloadFailureCount();

Loading…
Cancel
Save