diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java index ef0badf7e5..7932be5750 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java @@ -14,14 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.entity; -import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.UdfType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; - +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.KeyDeserializer; +import java.io.IOException; import java.util.Date; /** @@ -215,19 +220,17 @@ public class UdfFunc { @Override public String toString() { - return "UdfFunc{" + - "id=" + id + - ", userId=" + userId + - ", funcName='" + funcName + '\'' + - ", className='" + className + '\'' + - ", argTypes='" + argTypes + '\'' + - ", database='" + database + '\'' + - ", description='" + description + '\'' + - ", resourceId=" + resourceId + - ", resourceName='" + resourceName + '\'' + - ", type=" + type + - ", createTime=" + createTime + - ", updateTime=" + updateTime + - '}'; + return JSONUtils.toJsonString(this); + } + + public static class UdfFuncDeserializer extends KeyDeserializer { + + @Override + public Object deserializeKey(String key, DeserializationContext ctxt) throws IOException { + if (StringUtils.isBlank(key)) { + return null; + } + return JSONUtils.parseObject(key); + } } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/UdfFuncTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/UdfFuncTest.java new file mode 100644 index 0000000000..5d0fbe86d4 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/UdfFuncTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import org.apache.dolphinscheduler.dao.entity.UdfFunc.UdfFuncDeserializer; +import java.io.IOException; +import org.junit.Assert; +import org.junit.Test; + +public class UdfFuncTest { + + /** + * test to String + */ + @Test + public void testUdfFuncToString() { + + UdfFunc udfFunc = new UdfFunc(); + udfFunc.setResourceName("dolphin_resource_update"); + udfFunc.setResourceId(2); + udfFunc.setClassName("org.apache.dolphinscheduler.test.mrUpdate"); + + Assert.assertEquals("{\"id\":0,\"userId\":0,\"funcName\":null,\"className\":\"org.apache.dolphinscheduler.test.mrUpdate\",\"argTypes\":null,\"database\":null," + + "\"description\":null,\"resourceId\":2,\"resourceName\":\"dolphin_resource_update\",\"type\":null,\"createTime\":null,\"updateTime\":null}" + , udfFunc.toString()); + } + + /** + * test UdfFuncDeserializer.deserializeKey + * + * @throws IOException + */ + @Test + public void testUdfFuncDeserializer() throws IOException { + + // UdfFuncDeserializer.deserializeKey key is null + UdfFuncDeserializer udfFuncDeserializer = new UdfFuncDeserializer(); + Assert.assertNull(udfFuncDeserializer.deserializeKey(null, null)); + + // + UdfFunc udfFunc = new UdfFunc(); + udfFunc.setResourceName("dolphin_resource_update"); + udfFunc.setResourceId(2); + udfFunc.setClassName("org.apache.dolphinscheduler.test.mrUpdate"); + + Assert.assertNotNull(udfFuncDeserializer.deserializeKey(udfFunc.toString(), null)); + } + +} \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java index 210db5c4c4..fa670e2853 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java @@ -18,7 +18,8 @@ package org.apache.dolphinscheduler.server.entity; import org.apache.dolphinscheduler.dao.entity.UdfFunc; - +import org.apache.dolphinscheduler.dao.entity.UdfFunc.UdfFuncDeserializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import java.io.Serializable; import java.util.Map; @@ -40,6 +41,7 @@ public class SQLTaskExecutionContext implements Serializable { /** * udf function tenant code map */ + @JsonDeserialize(keyUsing = UdfFuncDeserializer.class) private Map udfFuncTenantCodeMap; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContextTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContextTest.java new file mode 100644 index 0000000000..a2ca6be3cd --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContextTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.entity; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.util.HashMap; +import java.util.Map; +import org.junit.Test; + +public class SQLTaskExecutionContextTest { + + /** + * test parse josn String to TaskExecutionContext + */ + @Test + public void testTaskExecutionContext() { + String contextJson = "{\n" + + " \"taskInstanceId\":32,\n" + + " \"taskName\":\"test-hive-func\",\n" + + " \"startTime\":\"2020-07-19 16:45:46\",\n" + + " \"taskType\":\"SQL\",\n" + + " \"host\":null,\n" + + " \"executePath\":\"/tmp/dolphinscheduler/exec/process/1/5/14/32\",\n" + + " \"logPath\":null,\n" + + " \"taskJson\":\"{\\\"id\\\":\\\"tasks-70999\\\",\\\"name\\\":\\\"test-hive-func\\\"" + + ",\\\"desc\\\":null,\\\"type\\\":\\\"SQL\\\",\\\"runFlag\\\":\\\"NORMAL\\\"," + + "\\\"loc\\\":null,\\\"maxRetryTimes\\\":0,\\\"retryInterval\\\":1," + + "\\\"params\\\":{\\\"type\\\":\\\"HIVE\\\",\\\"datasource\\\":2," + + "\\\"sql\\\":\\\"select mid_id, user_id," + + " version_code, version_name, lang, source, os, area, model, " + + "brand, sdk_version, gmail, height_width, app_time, network," + + " lng, lat, dt,\\\\n Lower(model)\\\\nfrom dws_uv_detail_day limit 5;" + + "\\\",\\\"udfs\\\":\\\"1\\\",\\\"sqlType\\\":\\\"0\\\",\\\"title\\\":\\\"" + + "test-hive-user-func\\\",\\\"receivers\\\":\\\"534634799@qq.com\\\"," + + "\\\"receiversCc\\\":\\\"\\\",\\\"showType\\\":\\\"TABLE\\\",\\\"localParams\\\":[]," + + "\\\"connParams\\\":\\\"\\\",\\\"preStatements\\\":[],\\\"postStatements\\\":[]}," + + "\\\"preTasks\\\":[],\\\"extras\\\":null,\\\"depList\\\":[],\\\"dependence\\\":{}," + + "\\\"conditionResult\\\":{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}," + + "\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"workerGroup\\\":\\\"default\\\"," + + "\\\"workerGroupId\\\":null,\\\"timeout\\\":{\\\"strategy\\\":\\\"\\\",\\\"interval\\\":null," + + "\\\"enable\\\":false},\\\"conditionsTask\\\":false,\\\"forbidden\\\":false," + + "\\\"taskTimeoutParameter\\\":{\\\"enable\\\":false,\\\"strategy\\\":null," + + "\\\"interval\\\":0}}\",\n" + + " \"processId\":0,\n" + + " \"appIds\":null,\n" + + " \"processInstanceId\":14,\n" + + " \"scheduleTime\":null,\n" + + " \"globalParams\":null,\n" + + " \"executorId\":2,\n" + + " \"cmdTypeIfComplement\":2,\n" + + " \"tenantCode\":\"sl\",\n" + + " \"queue\":\"sl\",\n" + + " \"processDefineId\":5,\n" + + " \"projectId\":1,\n" + + " \"taskParams\":null,\n" + + " \"envFile\":null,\n" + + " \"definedParams\":null,\n" + + " \"taskAppId\":null,\n" + + " \"taskTimeoutStrategy\":0,\n" + + " \"taskTimeout\":0,\n" + + " \"workerGroup\":\"default\",\n" + + " \"resources\":{\n" + + " },\n" + + " \"sqlTaskExecutionContext\":{\n" + + " \"warningGroupId\":0,\n" + + " \"connectionParams\":\"{\\\"type\\\":null,\\\"address\\\":" + + "\\\"jdbc:hive2://localhost:10000\\\",\\\"database\\\":\\\"gmall\\\"," + + "\\\"jdbcUrl\\\":\\\"jdbc:hive2://localhost:10000/gmall\\\"," + + "\\\"user\\\":\\\"sl-test\\\",\\\"password\\\":\\\"123456sl\\\"}\",\n" + + " \"udfFuncTenantCodeMap\": null" + + " },\n" + + " \"dataxTaskExecutionContext\":{\n" + + " \"dataSourceId\":0,\n" + + " \"sourcetype\":0,\n" + + " \"sourceConnectionParams\":null,\n" + + " \"dataTargetId\":0,\n" + + " \"targetType\":0,\n" + + " \"targetConnectionParams\":null\n" + + " },\n" + + " \"dependenceTaskExecutionContext\":null,\n" + + " \"sqoopTaskExecutionContext\":{\n" + + " \"dataSourceId\":0,\n" + + " \"sourcetype\":0,\n" + + " \"sourceConnectionParams\":null,\n" + + " \"dataTargetId\":0,\n" + + " \"targetType\":0,\n" + + " \"targetConnectionParams\":null\n" + + " },\n" + + " \"procedureTaskExecutionContext\":{\n" + + " \"connectionParams\":null\n" + + " }\n" + + "}\n"; + + TaskExecutionContext taskExecutionContext = JSONUtils.parseObject(contextJson, TaskExecutionContext.class); + + assertNotNull(taskExecutionContext); + } + + @Test + public void testSqlTaskExecutionContext() { + + SQLTaskExecutionContext sqlTaskExecutionContext = new SQLTaskExecutionContext(); + sqlTaskExecutionContext.setWarningGroupId(0); + + Map udfmap = new HashMap<>(); + + UdfFunc udfFunc = new UdfFunc(); + udfFunc.setArgTypes("1"); + udfFunc.setId(1); + udfFunc.setResourceName("name1"); + udfmap.put(udfFunc, "map1"); + + UdfFunc udfFunc2 = new UdfFunc(); + udfFunc2.setArgTypes("2"); + udfFunc2.setId(2); + udfFunc2.setResourceName("name2"); + udfmap.put(udfFunc2, "map2"); + + sqlTaskExecutionContext.setUdfFuncTenantCodeMap(udfmap); + + String contextJson = JSONUtils.toJsonString(sqlTaskExecutionContext); + SQLTaskExecutionContext parseSqlTask = JSONUtils.parseObject(contextJson, SQLTaskExecutionContext.class); + + assertNotNull(parseSqlTask); + assertEquals(sqlTaskExecutionContext.getWarningGroupId(), parseSqlTask.getWarningGroupId()); + assertEquals(sqlTaskExecutionContext.getUdfFuncTenantCodeMap().size(), parseSqlTask.getUdfFuncTenantCodeMap().size()); + } + + /** + * test the SQLTaskExecutionContext + */ + @Test + public void testSqlTaskExecutionContextParse() { + + // SQLTaskExecutionContext.udfFuncTenantCodeMap is null + String contextJson = "{\n" + + " \"warningGroupId\":0,\n" + + " \"connectionParams\":null,\n" + + " \"udfFuncTenantCodeMap\":null" + + "}\n}"; + SQLTaskExecutionContext parseSqlTask = JSONUtils.parseObject(contextJson, SQLTaskExecutionContext.class); + + assertNotNull(parseSqlTask); + assertEquals(0,parseSqlTask.getWarningGroupId()); + assertNull(parseSqlTask.getUdfFuncTenantCodeMap()); + + // SQLTaskExecutionContext.udfFuncTenantCodeMap is not null + contextJson = "{\"warningGroupId\":0," + + "\"connectionParams\":null," + + "\"udfFuncTenantCodeMap\":{\"" + + "{\\\"id\\\":2,\\\"userId\\\":0," + + "\\\"funcName\\\":null,\\\"className\\\":null,\\\"argTypes\\\":\\\"2\\\",\\\"database\\\":null," + + "\\\"description\\\":null,\\\"resourceId\\\":0,\\\"resourceName\\\":\\\"name2\\\",\\\"type\\\":null," + + "\\\"createTime\\\":null,\\\"updateTime\\\":null}\":\"map2\"," + + "\"{\\\"id\\\":1,\\\"userId\\\":0,\\\"funcName\\\":null," + + "\\\"className\\\":null,\\\"argTypes\\\":\\\"1\\\"," + + "\\\"database\\\":null,\\\"description\\\":null," + + "\\\"resourceId\\\":0,\\\"resourceName\\\":\\\"name1\\\"," + + "\\\"type\\\":null,\\\"createTime\\\":null,\\\"updateTime\\\":null}\":\"map1\"}}\n"; + + SQLTaskExecutionContext parseSqlTask2 = JSONUtils.parseObject(contextJson, SQLTaskExecutionContext.class); + + assertNotNull(parseSqlTask2); + assertEquals(0,parseSqlTask2.getWarningGroupId()); + assertEquals(2,parseSqlTask2.getUdfFuncTenantCodeMap().size()); + } + +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 0591525288..4103c75ddb 100644 --- a/pom.xml +++ b/pom.xml @@ -785,12 +785,14 @@ **/dao/mapper/CommandMapperTest.java **/dao/mapper/ConnectionFactoryTest.java **/dao/mapper/DataSourceMapperTest.java + **/dao/entity/UdfFuncTest.java **/remote/JsonSerializerTest.java **/remote/RemoveTaskLogResponseCommandTest.java **/remote/RemoveTaskLogRequestCommandTest.java **/remote/ResponseFutureTest.java + **/server/entity/SQLTaskExecutionContextTest.java **/server/log/MasterLogFilterTest.java **/server/log/SensitiveDataConverterTest.java