From 037692517ac661344dd86d178d6ca5e367cb1fcb Mon Sep 17 00:00:00 2001 From: nobolity Date: Mon, 11 Apr 2022 10:49:46 +0800 Subject: [PATCH] [Fix-9251] [WORKER] reslove the sql task about of add the udf resource failed (#9319) * feat(resource manager): extend s3 to the storage of ds 1.fix some spell question 2.extend the type of storage 3.add the s3utils to manager resource 4.automatic inject the storage in addition to your config * fix(resource manager): update the dependency * fix(resource manager): extend s3 to the storage of ds fix the constant of hadooputils * fix(resource manager): extend s3 to the storage of ds 1.fix some spell question 2.delete the import * * fix(resource manager): merge the unitTest: 1.TenantServiceImpl 2.ResourceServiceImpl 3.UserServiceImpl * fix(resource manager): extend s3 to the storage of ds merge the resourceServiceTest * fix(resource manager): test cancel the test method createTenant verifyTenant * fix(resource manager): merge the code follow the check-result of sonar * fix(resource manager): extend s3 to the storage of ds fit the spell question * fix(resource manager): extend s3 to the storage of ds revert the common.properties * fix(resource manager): extend s3 to the storage of ds update the storageConfig with None * fix(resource manager): extend s3 to the storage of ds fix the judge of resourceType * fix(resource manager): extend s3 to the storage of ds undo the compile-mysql * fix(resource manager): extend s3 to the storage of ds delete hadoop aws * fix(resource manager): extend s3 to the storage of ds update the know-dependencies to delete aws 1.7.4 update the e2e file-manager common.properties * fix(resource manager): extend s3 to the storage of ds update the aws-region * fix(resource manager): extend s3 to the storage of ds fix the storageconfig init * fix(resource manager): update e2e docker-compose update e2e docker-compose * fix(resource manager): extend s3 to the storage of ds revent the e2e common.proprites print the resource type in propertyUtil * fix(resource manager): extend s3 to the storage of ds 1.println the properties * fix(resource manager): println the s3 info * fix(resource manager): extend s3 to the storage of ds delete the info and upgrade the s3 info to e2e * fix(resource manager): extend s3 to the storage of ds add the bucket init * fix(resource manager): extend s3 to the storage of ds 1.fix some spell question 2.delete the import * * fix(resource manager): extend s3 to the storage of ds upgrade the s3 endpoint * fix(resource manager): withPathStyleAccessEnabled(true) * fix(resource manager): extend s3 to the storage of ds 1.fix some spell question 2.delete the import * * fix(resource manager): upgrade the s3client builder * fix(resource manager): correct the s3 point to s3client * fix(resource manager): update the constant BUCKET_NAME * fix(resource manager): e2e s3 endpoint -> s3:9000 * fix(resource manager): extend s3 to the storage of ds 1.fix some spell question 2.delete the import * * style(resource manager): add info to createBucket * style(resource manager): debug the log * ci(resource manager): test test s3 * ci(ci): add INSERT INTO dolphinscheduler.t_ds_tenant (id, tenant_code, description, queue_id, create_time, update_time) VALUES(1, 'root', NULL, 1, NULL, NULL); to h2.sql * fix(resource manager): update the h2 sql * fix(resource manager): solve to delete the tenant * style(resource manager): merge the style end delete the unuse s3 config * fix(resource manager): extend s3 to the storage of ds UPDATE the rename resources when s3 * fix(resource manager): extend s3 to the storage of ds 1.fix the code style of QuartzImpl * fix(resource manager): extend s3 to the storage of ds 1.impoort restore_type to CommonUtils * fix(resource manager): update the work thread * fix(resource manager): update the baseTaskProcessor * fix(resource manager): upgrade dolphinscheduler-standalone-server.xml * fix(resource manager): add user Info to dolphinscheduler_h2.sql * fix(resource manager): merge the resourceType to NONE * style(upgrade the log level to info): * fix(resource manager): sysnc the h2.sql * fix(resource manager): update the merge the user tenant * fix(resource manager): merge the resourcesServiceImpl * fix(resource manager): when the storage is s3 ,that the directory can't be renamed * fix(resource manager): in s3 ,the directory cannot be renamed * fix(resource manager): delete the deleteRenameDirectory in E2E * fix(resource manager): check the style and recoverd the test * fix(resource manager): delete the log.print(LoginUser) * fix(server): fix the udf serialize * fix(master task): update the udfTest to update the json string * fix(test): update the udfFuncTest * fix(common): syn the common.properties * fix(udfTest): upgrade the udfTest * fix(common): revent the common.properties --- .../dolphinscheduler/dao/entity/UdfFunc.java | 27 ++++++++++++------- .../dao/entity/UdfFuncTest.java | 9 ++++--- .../resource/AbstractResourceParameters.java | 3 ++- .../resource/DataSourceParameters.java | 12 +++++++++ .../resource/UdfFuncParameters.java | 13 +++++++++ 5 files changed, 50 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java index 2a389b9bd5..949499d1e8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java @@ -17,19 +17,18 @@ package org.apache.dolphinscheduler.dao.entity; -import org.apache.dolphinscheduler.common.enums.UdfType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; - -import org.apache.commons.lang.StringUtils; - -import java.io.IOException; -import java.util.Date; - import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.KeyDeserializer; +import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.common.enums.UdfType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import java.io.IOException; +import java.util.Date; /** * udf function @@ -39,13 +38,23 @@ public class UdfFunc { /** * id */ - @TableId(value="id", type=IdType.AUTO) + @TableId(value = "id", type = IdType.AUTO) private int id; /** * user id */ private int userId; + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = "UDF"; + } + + @TableField(exist = false) + private String resourceType = "UDF"; /** * udf function name */ diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/UdfFuncTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/UdfFuncTest.java index 5d0fbe86d4..4231ba35f6 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/UdfFuncTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/UdfFuncTest.java @@ -18,10 +18,11 @@ package org.apache.dolphinscheduler.dao.entity; import org.apache.dolphinscheduler.dao.entity.UdfFunc.UdfFuncDeserializer; -import java.io.IOException; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; + public class UdfFuncTest { /** @@ -35,9 +36,9 @@ public class UdfFuncTest { udfFunc.setResourceId(2); udfFunc.setClassName("org.apache.dolphinscheduler.test.mrUpdate"); - Assert.assertEquals("{\"id\":0,\"userId\":0,\"funcName\":null,\"className\":\"org.apache.dolphinscheduler.test.mrUpdate\",\"argTypes\":null,\"database\":null," - + "\"description\":null,\"resourceId\":2,\"resourceName\":\"dolphin_resource_update\",\"type\":null,\"createTime\":null,\"updateTime\":null}" - , udfFunc.toString()); + Assert.assertEquals("{\"id\":0,\"userId\":0,\"resourceType\":\"UDF\",\"funcName\":null,\"className\":\"org.apache.dolphinscheduler.test.mrUpdate\",\"argTypes\":null,\"database\":null," + + "\"description\":null,\"resourceId\":2,\"resourceName\":\"dolphin_resource_update\",\"type\":null,\"createTime\":null,\"updateTime\":null}" + , udfFunc.toString()); } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java index 39193c8f2e..b719f95b4e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java @@ -21,11 +21,12 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, visible = true, property = "resourceType") @JsonSubTypes({ @Type(value = DataSourceParameters.class, name = "DATASOURCE"), @Type(value = UdfFuncParameters.class, name = "UDF") }) public abstract class AbstractResourceParameters { + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java index 71d0dc0dad..ea9cb7ae5a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java @@ -17,12 +17,24 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters.resource; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.dolphinscheduler.spi.enums.DbType; public class DataSourceParameters extends AbstractResourceParameters { private DbType type; + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + @JsonProperty(value = "DATASOURCE") + private String resourceType; + private String connectionParams; public DbType getType() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/UdfFuncParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/UdfFuncParameters.java index d1b1f36e47..f0ce772935 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/UdfFuncParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/UdfFuncParameters.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters.resource; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.dolphinscheduler.plugin.task.api.enums.UdfType; import org.apache.dolphinscheduler.spi.utils.JSONUtils; @@ -30,6 +31,18 @@ public class UdfFuncParameters extends AbstractResourceParameters { * id */ private int id; + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + @JsonProperty(value = "UDF") + private String resourceType; + /** * user id */