From bc8e06bb08be1eb7361e88f6c4f2c799aa170677 Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Tue, 31 Dec 2019 22:51:56 +0800 Subject: [PATCH 1/6] SchemaUtils.java TaskParametersUtils.java unit test (#1660) * schemautils test and TaskParametersUtils test --- .../common/utils/SchemaUtils.java | 24 ++-- .../common/utils/SchemaUtilsTest.java | 119 ++++++++++++++++++ .../common/utils/TaskParametersUtilsTest.java | 48 +++++++ 3 files changed, 179 insertions(+), 12 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java index a4e16ba8b6..7d341f3b20 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java @@ -35,7 +35,7 @@ import java.util.regex.Pattern; * */ public class SchemaUtils { - + private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); private static Pattern p = Pattern.compile("\\s*|\t|\r|\n"); @@ -50,11 +50,11 @@ public class SchemaUtils { if(schemaDirArr == null || schemaDirArr.length == 0) { return null; } - + for(File file : schemaDirArr) { schemaDirList.add(file.getName()); } - + Collections.sort(schemaDirList , new Comparator() { @Override public int compare(Object o1 , Object o2){ @@ -66,23 +66,23 @@ public class SchemaUtils { if(version1.equals(version2)) { return 0; } - + if(SchemaUtils.isAGreatVersion(version1, version2)) { return 1; } - + return -1; - + } catch (Exception e) { logger.error(e.getMessage(),e); throw new RuntimeException(e); } } }); - + return schemaDirList; } - + /** * Determine whether schemaVersion is higher than version * @param schemaVersion schema version @@ -93,7 +93,7 @@ public class SchemaUtils { if(StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) { throw new RuntimeException("schemaVersion or version is empty"); } - + String[] schemaVersionArr = schemaVersion.split("\\."); String[] versionArr = version.split("\\."); int arrLength = schemaVersionArr.length < versionArr.length ? schemaVersionArr.length : versionArr.length; @@ -104,11 +104,11 @@ public class SchemaUtils { return false; } } - + // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number return schemaVersionArr.length > versionArr.length; } - + /** * Gets the current software version number of the system * @return current software version @@ -127,7 +127,7 @@ public class SchemaUtils { } return soft_version; } - + /** * Strips the string of space carriage returns and tabs * @param str string diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java new file mode 100644 index 0000000000..907a09e458 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ LoggerFactory.class, FileUtils.class }) +public class SchemaUtilsTest { + + @Test + public void testReplaceBlank() { + Assert.assertEquals("abc", SchemaUtils.replaceBlank(" abc")); + Assert.assertEquals("abc", SchemaUtils.replaceBlank("abc ")); + Assert.assertEquals("abc", SchemaUtils.replaceBlank("a b c")); + Assert.assertEquals("abc", SchemaUtils.replaceBlank("a b c")); + Assert.assertEquals("", SchemaUtils.replaceBlank(" ")); + Assert.assertEquals("", SchemaUtils.replaceBlank(null)); + Assert.assertEquals("我怕的你", SchemaUtils.replaceBlank("我怕的 你")); + } + + @Test + public void testGetSoftVersion() { + // file not found + try { + SchemaUtils.getSoftVersion(); + } catch (RuntimeException e) { + Assert.assertEquals("Failed to get the product version description file. The file could not be found", + e.getMessage()); + } + + // file exists, fmt is invalid + FileUtils.writeContent2File("32432423", "sql/soft_version"); + Assert.assertEquals("32432423", SchemaUtils.getSoftVersion()); + } + + @Test + public void testIsAGreatVersion() { + // param is null + try { + SchemaUtils.isAGreatVersion(null, null); + } catch (RuntimeException e) { + Assert.assertEquals("schemaVersion or version is empty", e.getMessage()); + } + + // param is "" + try { + SchemaUtils.isAGreatVersion("", ""); + } catch (RuntimeException e) { + Assert.assertEquals("schemaVersion or version is empty", e.getMessage()); + } + Assert.assertFalse(SchemaUtils.isAGreatVersion("1", "1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("2", "1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1.0.1")); + Assert.assertFalse(SchemaUtils.isAGreatVersion("1.1", "1.2")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1.1", "1.1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("10.1.1", "1.01.100")); + try { + SchemaUtils.isAGreatVersion("10.1.1", ".1"); + } catch (Exception e) { + Assert.assertNotNull(e); + } + try { + SchemaUtils.isAGreatVersion("a.1.1", "b.1"); + } catch (Exception e) { + Assert.assertNotNull(e); + } + } + + @Test + public void testGetAllSchemaList() { + //normal + PowerMockito.mockStatic(FileUtils.class); + File[] files = new File[4]; + files[0] = new File("sql/upgrade/1.2.0_schema"); + files[1] = new File("sql/upgrade/1.0.1_schema"); + files[2] = new File("sql/upgrade/1.0.2_schema"); + files[3] = new File("sql/upgrade/1.1.0_schema"); + PowerMockito.when(FileUtils.getAllDir("sql/upgrade")).thenReturn(files); + List real = SchemaUtils.getAllSchemaList(); + List expect = Arrays.asList("1.0.1_schema", "1.0.2_schema", + "1.1.0_schema", "1.2.0_schema"); + Assert.assertTrue(CollectionUtils.isEqualCollection(real, expect)); + + //normal + files = new File[0]; + PowerMockito.when(FileUtils.getAllDir("sql/upgrade")).thenReturn(files); + real = SchemaUtils.getAllSchemaList(); + Assert.assertNull(real); + } +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java new file mode 100644 index 0000000000..db4a86bc26 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.LoggerFactory; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(LoggerFactory.class) +public class TaskParametersUtilsTest { + + @Test + public void testGetParameters() { + Assert.assertNull(TaskParametersUtils.getParameters("xx", "ttt")); + Assert.assertNull(TaskParametersUtils.getParameters("SHELL", "ttt")); + Assert.assertNotNull(TaskParametersUtils.getParameters("SHELL", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("SQL", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("SUB_PROCESS", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("PROCEDURE", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("MR", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("SPARK", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("PYTHON", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("DEPENDENT", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("FLINK", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("HTTP", "{}")); + } +} From dc6c18902a864cb5a00f57d6fc9a9d91fbd3bbe8 Mon Sep 17 00:00:00 2001 From: samz406 Date: Tue, 31 Dec 2019 23:21:09 +0800 Subject: [PATCH 2/6] modify UdfFuncServiceTest UT (#1644) --- dolphinscheduler-api/pom.xml | 18 ++ .../api/service/UdfFuncServiceTest.java | 196 ++++++++++++++++-- pom.xml | 1 + 3 files changed, 201 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 06a444aa6d..014799b0b6 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -250,5 +250,23 @@ ${servlet-api.version} + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index 814a9ee7cd..9ec24bbb50 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -16,43 +16,211 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; +import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest(PropertyUtils.class) public class UdfFuncServiceTest { private static final Logger logger = LoggerFactory.getLogger(UdfFuncServiceTest.class); - @Autowired + @InjectMocks private UdfFuncService udfFuncService; + @Mock + private ResourceMapper resourceMapper; + @Mock + private UdfFuncMapper udfFuncMapper; + @Mock + private UDFUserMapper udfUserMapper; + + + @Before + public void setUp() { + PowerMockito.mockStatic(PropertyUtils.class); + } + + @Test + public void testCreateUdfFunction(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + //hdfs not start + Result result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + //resource not exist + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + // success + PowerMockito.when(resourceMapper.selectById(1)).thenReturn(getResource()); + result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + } @Test - public void queryUdfFuncListPaging(){ + public void testQueryUdfFuncDetail(){ + + PowerMockito.when(udfFuncMapper.selectById(1)).thenReturn(getUdfFunc()); + //resource not exist + Map result = udfFuncService.queryUdfFuncDetail(2); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST,result.get(Constants.STATUS)); + // success + result = udfFuncService.queryUdfFuncDetail(1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testUpdateUdfFunc(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + PowerMockito.when(udfFuncMapper.selectUdfById(1)).thenReturn(getUdfFunc()); + PowerMockito.when(resourceMapper.selectById(1)).thenReturn(getResource()); + + //UDF_FUNCTION_NOT_EXIST + Map result = udfFuncService.updateUdfFunc(12, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.UDF_FUNCTION_NOT_EXIST,result.get(Constants.STATUS)); + + //HDFS_NOT_STARTUP + result = udfFuncService.updateUdfFunc(1, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP,result.get(Constants.STATUS)); + + //RESOURCE_NOT_EXIST + PowerMockito.when(udfFuncMapper.selectUdfById(11)).thenReturn(getUdfFunc()); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = udfFuncService.updateUdfFunc(11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 12); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST,result.get(Constants.STATUS)); + + //success + result = udfFuncService.updateUdfFunc(11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + + @Test + public void testQueryUdfFuncListPaging(){ + + IPage page = new Page<>(1,10); + page.setTotal(1L); + page.setRecords(getList()); + Mockito.when(udfFuncMapper.queryUdfFuncPaging(Mockito.any(Page.class), Mockito.eq(0),Mockito.eq("test"))).thenReturn(page); + Map result = udfFuncService.queryUdfFuncListPaging(getLoginUser(),"test",1,10); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + } + + @Test + public void testQueryResourceList(){ + Mockito.when(udfFuncMapper.getUdfFuncByType(1, 1)).thenReturn(getList()); + Map result = udfFuncService.queryResourceList(getLoginUser(),1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List udfFuncList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncList)); + } + + @Test + public void testDelete(){ + Result result= udfFuncService.delete(122); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + } + + @Test + public void testVerifyUdfFuncByName(){ + + //success + Mockito.when(udfFuncMapper.queryUdfByIdStr(null, "UdfFuncServiceTest")).thenReturn(getList()); + Result result = udfFuncService.verifyUdfFuncByName("test"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + //exist + result = udfFuncService.verifyUdfFuncByName("UdfFuncServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.UDF_FUNCTION_EXISTS.getMsg(),result.getMsg()); + } + + /** + * create admin user + * @return + */ + private User getLoginUser(){ User loginUser = new User(); - loginUser.setId(-1); - loginUser.setUserType(UserType.GENERAL_USER); + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setId(1); + return loginUser; + } + + /** + * get resourceId + */ + private Resource getResource(){ - Map map = udfFuncService.queryUdfFuncListPaging(loginUser, "", 1, 10); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); + Resource resource = new Resource(); + resource.setId(1); + resource.setAlias("test"); + return resource; + } - PageInfo pageInfo = (PageInfo) map.get("data"); - logger.info(pageInfo.getLists().toString()); + private List getList(){ + List udfFuncList = new ArrayList<>(); + udfFuncList.add(getUdfFunc()); + return udfFuncList; + } + /** + * get UdfFunc id + */ + private UdfFunc getUdfFunc(){ + UdfFunc udfFunc = new UdfFunc(); + udfFunc.setFuncName("UdfFuncServiceTest"); + udfFunc.setClassName("org.apache.dolphinscheduler.api.service.UdfFuncServiceTest"); + udfFunc.setResourceId(0); + udfFunc.setResourceName("UdfFuncServiceTest"); + udfFunc.setCreateTime(new Date()); + udfFunc.setDatabase("database"); + udfFunc.setUpdateTime(new Date()); + udfFunc.setType(UdfType.HIVE); + return udfFunc; } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index fafa7be6ef..5e94b16768 100644 --- a/pom.xml +++ b/pom.xml @@ -675,6 +675,7 @@ **/api/service/WorkerGroupServiceTest.java **/api/service/AlertGroupServiceTest.java **/api/service/ProjectServiceTest.java + **/api/service/UdfFuncServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From b4711194438ff34b3aa4d6e3430e0ec920dace06 Mon Sep 17 00:00:00 2001 From: zhukai Date: Tue, 31 Dec 2019 23:33:27 +0800 Subject: [PATCH 3/6] Add ParamUtilsTest which is the UT of ParamUtils (#1634) 1. Add ParamUtilsTest. 2. Add a null check in the method convert. 3. Add the UT path to the root pom. --- .../server/utils/ParamUtils.java | 4 + .../server/utils/ParamUtilsTest.java | 129 ++++++++++++++++++ pom.xml | 1 + 3 files changed, 134 insertions(+) create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java index 3965b0e8f4..1d7a80daf0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java @@ -93,6 +93,10 @@ public class ParamUtils { * @return Map of converted */ public static Map convert(Map paramsMap){ + if(paramsMap == null){ + return null; + } + Map map = new HashMap<>(); Iterator> iter = paramsMap.entrySet().iterator(); while (iter.hasNext()){ diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java new file mode 100644 index 0000000000..b91bff7659 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.utils; + +import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DataType; +import org.apache.dolphinscheduler.common.enums.Direct; +import org.apache.dolphinscheduler.common.process.Property; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Test ParamUtils + */ +public class ParamUtilsTest { + + private static final Logger logger = LoggerFactory.getLogger(ParamUtilsTest.class); + + //Define global variables + public Map globalParams = new HashMap<>(); + + public Map globalParamsMap = new HashMap<>(); + + public Map localParams = new HashMap<>(); + + /** + * Init params + * @throws Exception + */ + @Before + public void setUp() throws Exception { + + Property property = new Property(); + property.setProp("global_param"); + property.setDirect(Direct.IN); + property.setType(DataType.VARCHAR); + property.setValue("${system.biz.date}"); + globalParams.put("global_param", property); + + globalParamsMap.put("global_param", "${system.biz.date}"); + + Property localProperty = new Property(); + localProperty.setProp("local_param"); + localProperty.setDirect(Direct.IN); + localProperty.setType(DataType.VARCHAR); + localProperty.setValue("${global_param}"); + localParams.put("local_param", localProperty); + } + + /** + * Test convert + */ + @Test + public void testConvert() { + + //The expected value + String expected = "{\"global_param\":{\"direct\":\"IN\",\"prop\":\"global_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"},\"local_param\":{\"direct\":\"IN\",\"prop\":\"local_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; + + //The expected value when globalParams is null but localParams is not null + String expected1 = "{\"local_param\":{\"direct\":\"IN\",\"prop\":\"local_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; + + //Invoke convert + Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, CommandType.START_PROCESS, new Date()); + String result = JSON.toJSONString(paramsMap); + assertEquals(expected, result); + + for (Map.Entry entry : paramsMap.entrySet()) { + + String key = entry.getKey(); + Property prop = entry.getValue(); + logger.info(key + " : " + prop.getValue()); + } + + //Invoke convert with null globalParams + Map paramsMap1 = ParamUtils.convert(null, globalParamsMap, localParams, CommandType.START_PROCESS, new Date()); + String result1 = JSON.toJSONString(paramsMap1); + assertEquals(expected1, result1); + + //Null check, invoke convert with null globalParams and null localParams + Map paramsMap2 = ParamUtils.convert(null, globalParamsMap, null, CommandType.START_PROCESS, new Date()); + assertNull(paramsMap2); + } + + /** + * Test the overload method of convert + */ + @Test + public void testConvert1() { + + //The expected value + String expected = "{\"global_param\":\"${system.biz.date}\"}"; + + //Invoke convert + Map paramsMap = ParamUtils.convert(globalParams); + String result = JSON.toJSONString(paramsMap); + assertEquals(expected, result); + + logger.info(result); + + //Null check + Map paramsMap1 = ParamUtils.convert(null); + assertNull(paramsMap1); + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5e94b16768..6892b387f1 100644 --- a/pom.xml +++ b/pom.xml @@ -682,6 +682,7 @@ **/alert/utils/PropertyUtilsTest.java **/server/utils/SparkArgsUtilsTest.java **/server/utils/FlinkArgsUtilsTest.java + **/server/utils/ParamUtilsTest.java **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/AlertMapperTest.java From f311c966ab2f7cfcbea8cb8983689ddbdacab7b7 Mon Sep 17 00:00:00 2001 From: zhukai Date: Wed, 1 Jan 2020 21:09:19 +0800 Subject: [PATCH 4/6] Fix bug in #1634 ,replace the current datetime with a fixed time (#1665) --- .../server/utils/ParamUtilsTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java index b91bff7659..60a5259f44 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -84,8 +85,13 @@ public class ParamUtilsTest { //The expected value when globalParams is null but localParams is not null String expected1 = "{\"local_param\":{\"direct\":\"IN\",\"prop\":\"local_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; + //Define expected date , the month is 0-base + Calendar calendar = Calendar.getInstance(); + calendar.set(2019,11,30); + Date date = calendar.getTime(); + //Invoke convert - Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, CommandType.START_PROCESS, new Date()); + Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, CommandType.START_PROCESS, date); String result = JSON.toJSONString(paramsMap); assertEquals(expected, result); @@ -97,12 +103,12 @@ public class ParamUtilsTest { } //Invoke convert with null globalParams - Map paramsMap1 = ParamUtils.convert(null, globalParamsMap, localParams, CommandType.START_PROCESS, new Date()); + Map paramsMap1 = ParamUtils.convert(null, globalParamsMap, localParams, CommandType.START_PROCESS, date); String result1 = JSON.toJSONString(paramsMap1); assertEquals(expected1, result1); //Null check, invoke convert with null globalParams and null localParams - Map paramsMap2 = ParamUtils.convert(null, globalParamsMap, null, CommandType.START_PROCESS, new Date()); + Map paramsMap2 = ParamUtils.convert(null, globalParamsMap, null, CommandType.START_PROCESS, date); assertNull(paramsMap2); } From 7e53ceb3c423a2a49343c2e9e96f36211f82a9c4 Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Wed, 1 Jan 2020 21:28:39 +0800 Subject: [PATCH 5/6] fix 0% UT coverage on new code . issue #1611 (#1662) --- .github/workflows/ci_ut.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index f76c403a8c..152d7ee8c9 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -51,17 +51,16 @@ jobs: CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) - name: Run SonarCloud analysis run: > - mvn clean --batch-mode - org.jacoco:jacoco-maven-plugin:prepare-agent - verify - org.sonarsource.scanner.maven:sonar-maven-plugin:sonar - -Dsonar.host.url=https://sonarcloud.io - -Dsonar.organization=apache - -Dsonar.projectKey=apache-dolphinscheduler - -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 + mvn clean --batch-mode + verify + org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.1.1688:sonar + -Dsonar.host.url=https://sonarcloud.io + -Dsonar.organization=apache + -Dsonar.projectKey=apache-dolphinscheduler + -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - name: Collect logs run: | mkdir -p ${LOG_DIR} From bb414b079451005ab1ab200830bed896eb482d4c Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Thu, 2 Jan 2020 10:15:37 +0800 Subject: [PATCH 6/6] delete bytes file (#1669) --- .../dolphinscheduler/common/utils/Bytes.java | 699 ------------------ .../server/worker/task/http/HttpTask.java | 6 +- 2 files changed, 3 insertions(+), 702 deletions(-) delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Bytes.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Bytes.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Bytes.java deleted file mode 100644 index 39050d9952..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Bytes.java +++ /dev/null @@ -1,699 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.utils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; - -/** - * Utility class that handles Bytes - */ -public class Bytes { - - private static final Logger logger = LoggerFactory.getLogger(Bytes.class); - public static final String UTF8_ENCODING = "UTF-8"; - //An empty instance. - public static final byte [] EMPTY_BYTE_ARRAY = new byte [0]; - - /** - * Size of int in bytes - */ - public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE; - - /** - * Size of long in bytes - */ - public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE; - - /** - * Size of short in bytes - */ - public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE; - - - - /** - * Put bytes at the specified byte array position. - * @param tgtBytes the byte array - * @param tgtOffset position in the array - * @param srcBytes array to write out - * @param srcOffset source offset - * @param srcLength source length - * @return incremented offset - */ - public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, - int srcOffset, int srcLength) { - System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength); - return tgtOffset + srcLength; - } - - /** - * Write a single byte out to the specified byte array position. - * @param bytes the byte array - * @param offset position in the array - * @param b byte to write out - * @return incremented offset - */ - public static int putByte(byte[] bytes, int offset, byte b) { - bytes[offset] = b; - return offset + 1; - } - - /** - * Returns a new byte array, copied from the passed ByteBuffer. - * @param bb A ByteBuffer - * @return the byte array - */ - public static byte[] toBytes(ByteBuffer bb) { - int length = bb.limit(); - byte [] result = new byte[length]; - System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length); - return result; - } - - /** - * @param b Presumed UTF-8 encoded byte array. - * @return String made from b - */ - public static String toString(final byte [] b) { - if (b == null) { - return null; - } - return toString(b, 0, b.length); - } - - /** - * Joins two byte arrays together using a separator. - * @param b1 The first byte array. - * @param sep The separator to use. - * @param b2 The second byte array. - * @return two byte arrays together using a separator. - */ - public static String toString(final byte [] b1, - String sep, - final byte [] b2) { - return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length); - } - - /** - * This method will convert utf8 encoded bytes into a string. If - * an UnsupportedEncodingException occurs, this method will eat it - * and return null instead. - * - * @param b Presumed UTF-8 encoded byte array. - * @param off offset into array - * @param len length of utf-8 sequence - * @return String made from b or null - */ - public static String toString(final byte [] b, int off, int len) { - if (b == null) { - return null; - } - if (len == 0) { - return ""; - } - return new String(b, off, len, StandardCharsets.UTF_8); - } - - - /** - * Converts a string to a UTF-8 byte array. - * @param s string - * @return the byte array - */ - public static byte[] toBytes(String s) { - return s.getBytes(StandardCharsets.UTF_8); - } - - /** - * Convert a boolean to a byte array. True becomes -1 - * and false becomes 0. - * - * @param b value - * @return b encoded in a byte array. - */ - public static byte [] toBytes(final boolean b) { - return new byte[] { b ? (byte) -1 : (byte) 0 }; - } - - /** - * Reverses {@link #toBytes(boolean)} - * @param b array - * @return True or false. - */ - public static boolean toBoolean(final byte [] b) { - if (b.length != 1) { - throw new IllegalArgumentException("Array has wrong size: " + b.length); - } - return b[0] != (byte) 0; - } - - /** - * Convert a long value to a byte array using big-endian. - * - * @param val value to convert - * @return the byte array - */ - public static byte[] toBytes(long val) { - byte [] b = new byte[8]; - for (int i = 7; i > 0; i--) { - b[i] = (byte) val; - val >>>= 8; - } - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to a long value. Reverses - * {@link #toBytes(long)} - * @param bytes array - * @return the long value - */ - public static long toLong(byte[] bytes) { - return toLong(bytes, 0, SIZEOF_LONG); - } - - /** - * Converts a byte array to a long value. Assumes there will be - * {@link #SIZEOF_LONG} bytes available. - * - * @param bytes bytes - * @param offset offset - * @return the long value - */ - public static long toLong(byte[] bytes, int offset) { - return toLong(bytes, offset, SIZEOF_LONG); - } - - /** - * Converts a byte array to a long value. - * - * @param bytes array of bytes - * @param offset offset into array - * @param length length of data (must be {@link #SIZEOF_LONG}) - * @return the long value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or - * if there's not enough room in the array at the offset indicated. - */ - public static long toLong(byte[] bytes, int offset, final int length) { - if (length != SIZEOF_LONG || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG); - } - long l = 0; - for(int i = offset; i < offset + length; i++) { - l <<= 8; - l ^= bytes[i] & 0xFF; - } - return l; - } - - private static IllegalArgumentException - explainWrongLengthOrOffset(final byte[] bytes, - final int offset, - final int length, - final int expectedLength) { - String reason; - if (length != expectedLength) { - reason = "Wrong length: " + length + ", expected " + expectedLength; - } else { - reason = "offset (" + offset + ") + length (" + length + ") exceed the" - + " capacity of the array: " + bytes.length; - } - return new IllegalArgumentException(reason); - } - - /** - * Put a long value out to the specified byte array position. - * @param bytes the byte array - * @param offset position in the array - * @param val long to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have - * enough room at the offset specified. - */ - public static int putLong(byte[] bytes, int offset, long val) { - if (bytes.length - offset < SIZEOF_LONG) { - throw new IllegalArgumentException("Not enough room to put a long at" - + " offset " + offset + " in a " + bytes.length + " byte array"); - } - for(int i = offset + 7; i > offset; i--) { - bytes[i] = (byte) val; - val >>>= 8; - } - bytes[offset] = (byte) val; - return offset + SIZEOF_LONG; - } - - /** - * Presumes float encoded as IEEE 754 floating-point "single format" - * @param bytes byte array - * @return Float made from passed byte array. - */ - public static float toFloat(byte [] bytes) { - return toFloat(bytes, 0); - } - - /** - * Presumes float encoded as IEEE 754 floating-point "single format" - * @param bytes array to convert - * @param offset offset into array - * @return Float made from passed byte array. - */ - public static float toFloat(byte [] bytes, int offset) { - return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT)); - } - - /** - * @param bytes byte array - * @param offset offset to write to - * @param f float value - * @return New offset in bytes - */ - public static int putFloat(byte [] bytes, int offset, float f) { - return putInt(bytes, offset, Float.floatToRawIntBits(f)); - } - - /** - * @param f float value - * @return the float represented as byte [] - */ - public static byte [] toBytes(final float f) { - // Encode it as int - return Bytes.toBytes(Float.floatToRawIntBits(f)); - } - - /** - * @param bytes byte array - * @return Return double made from passed bytes. - */ - public static double toDouble(final byte [] bytes) { - return toDouble(bytes, 0); - } - - /** - * @param bytes byte array - * @param offset offset where double is - * @return Return double made from passed bytes. - */ - public static double toDouble(final byte [] bytes, final int offset) { - return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG)); - } - - /** - * @param bytes byte array - * @param offset offset to write to - * @param d value - * @return New offset into array bytes - */ - public static int putDouble(byte [] bytes, int offset, double d) { - return putLong(bytes, offset, Double.doubleToLongBits(d)); - } - - /** - * Serialize a double as the IEEE 754 double format output. The resultant - * array will be 8 bytes long. - * - * @param d value - * @return the double represented as byte [] - */ - public static byte [] toBytes(final double d) { - // Encode it as a long - return Bytes.toBytes(Double.doubleToRawLongBits(d)); - } - - /** - * Convert an int value to a byte array - * @param val value - * @return the byte array - */ - public static byte[] toBytes(int val) { - byte [] b = new byte[4]; - for(int i = 3; i > 0; i--) { - b[i] = (byte) val; - val >>>= 8; - } - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to an int value - * @param bytes byte array - * @return the int value - */ - public static int toInt(byte[] bytes) { - return toInt(bytes, 0, SIZEOF_INT); - } - - /** - * Converts a byte array to an int value - * @param bytes byte array - * @param offset offset into array - * @return the int value - */ - public static int toInt(byte[] bytes, int offset) { - return toInt(bytes, offset, SIZEOF_INT); - } - - /** - * Converts a byte array to an int value - * @param bytes byte array - * @param offset offset into array - * @param length length of int (has to be {@link #SIZEOF_INT}) - * @return the int value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or - * if there's not enough room in the array at the offset indicated. - */ - public static int toInt(byte[] bytes, int offset, final int length) { - if (length != SIZEOF_INT || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT); - } - int n = 0; - for(int i = offset; i < (offset + length); i++) { - n <<= 8; - n ^= bytes[i] & 0xFF; - } - return n; - } - - /** - * Put an int value out to the specified byte array position. - * @param bytes the byte array - * @param offset position in the array - * @param val int to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have - * enough room at the offset specified. - */ - public static int putInt(byte[] bytes, int offset, int val) { - if (bytes.length - offset < SIZEOF_INT) { - throw new IllegalArgumentException("Not enough room to put an int at" - + " offset " + offset + " in a " + bytes.length + " byte array"); - } - for(int i= offset + 3; i > offset; i--) { - bytes[i] = (byte) val; - val >>>= 8; - } - bytes[offset] = (byte) val; - return offset + SIZEOF_INT; - } - - /** - * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long. - * @param val value - * @return the byte array - */ - public static byte[] toBytes(short val) { - byte[] b = new byte[SIZEOF_SHORT]; - b[1] = (byte) val; - val >>= 8; - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to a short value - * @param bytes byte array - * @return the short value - */ - public static short toShort(byte[] bytes) { - return toShort(bytes, 0, SIZEOF_SHORT); - } - - /** - * Converts a byte array to a short value - * @param bytes byte array - * @param offset offset into array - * @return the short value - */ - public static short toShort(byte[] bytes, int offset) { - return toShort(bytes, offset, SIZEOF_SHORT); - } - - /** - * Converts a byte array to a short value - * @param bytes byte array - * @param offset offset into array - * @param length length, has to be {@link #SIZEOF_SHORT} - * @return the short value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT} - * or if there's not enough room in the array at the offset indicated. - */ - public static short toShort(byte[] bytes, int offset, final int length) { - if (length != SIZEOF_SHORT || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); - } - short n = 0; - n ^= bytes[offset] & 0xFF; - n <<= 8; - n ^= bytes[offset+1] & 0xFF; - return n; - } - - /** - * This method will get a sequence of bytes from pos to limit, - * but will restore pos after. - * @param buf buffer - * @return byte array - */ - public static byte[] getBytes(ByteBuffer buf) { - int savedPos = buf.position(); - byte [] newBytes = new byte[buf.remaining()]; - buf.get(newBytes); - buf.position(savedPos); - return newBytes; - } - - /** - * Put a short value out to the specified byte array position. - * @param bytes the byte array - * @param offset position in the array - * @param val short to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have - * enough room at the offset specified. - */ - public static int putShort(byte[] bytes, int offset, short val) { - if (bytes.length - offset < SIZEOF_SHORT) { - throw new IllegalArgumentException("Not enough room to put a short at" - + " offset " + offset + " in a " + bytes.length + " byte array"); - } - bytes[offset+1] = (byte) val; - val >>= 8; - bytes[offset] = (byte) val; - return offset + SIZEOF_SHORT; - } - - /** - * Convert a BigDecimal value to a byte array - * - * @param val value - * @return the byte array - */ - public static byte[] toBytes(BigDecimal val) { - byte[] valueBytes = val.unscaledValue().toByteArray(); - byte[] result = new byte[valueBytes.length + SIZEOF_INT]; - int offset = putInt(result, 0, val.scale()); - putBytes(result, offset, valueBytes, 0, valueBytes.length); - return result; - } - - - /** - * Converts a byte array to a BigDecimal - * - * @param bytes bytes array - * @return the char value - */ - public static BigDecimal toBigDecimal(byte[] bytes) { - return toBigDecimal(bytes, 0, bytes.length); - } - - /** - * Converts a byte array to a BigDecimal value - * - * @param bytes bytes array - * @param offset offset - * @param length length - * @return the char value - */ - public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) { - if (bytes == null || length < SIZEOF_INT + 1 || - (offset + length > bytes.length)) { - return null; - } - - int scale = toInt(bytes, offset); - byte[] tcBytes = new byte[length - SIZEOF_INT]; - System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT); - return new BigDecimal(new BigInteger(tcBytes), scale); - } - - /** - * Put a BigDecimal value out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param val BigDecimal to write out - * @return incremented offset - */ - public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) { - if (bytes == null) { - return offset; - } - - byte[] valueBytes = val.unscaledValue().toByteArray(); - byte[] result = new byte[valueBytes.length + SIZEOF_INT]; - offset = putInt(result, offset, val.scale()); - return putBytes(result, offset, valueBytes, 0, valueBytes.length); - } - - /** - * @param a lower half - * @param b upper half - * @return New array that has a in lower half and b in upper half. - */ - public static byte [] add(final byte [] a, final byte [] b) { - return add(a, b, EMPTY_BYTE_ARRAY); - } - - /** - * @param a first third - * @param b second third - * @param c third third - * @return New array made from a, b and c - */ - public static byte [] add(final byte [] a, final byte [] b, final byte [] c) { - byte [] result = new byte[a.length + b.length + c.length]; - System.arraycopy(a, 0, result, 0, a.length); - System.arraycopy(b, 0, result, a.length, b.length); - System.arraycopy(c, 0, result, a.length + b.length, c.length); - return result; - } - - /** - * @param a array - * @param length amount of bytes to grab - * @return First length bytes from a - */ - public static byte [] head(final byte [] a, final int length) { - if (a.length < length) { - return null; - } - byte [] result = new byte[length]; - System.arraycopy(a, 0, result, 0, length); - return result; - } - - /** - * @param a array - * @param length amount of bytes to snarf - * @return Last length bytes from a - */ - public static byte [] tail(final byte [] a, final int length) { - if (a.length < length) { - return null; - } - byte [] result = new byte[length]; - System.arraycopy(a, a.length - length, result, 0, length); - return result; - } - - /** - * @param a array - * @param length new array size - * @return Value in a plus length prepended 0 bytes - */ - public static byte [] padHead(final byte [] a, final int length) { - byte[] padding = getPadding(length); - return add(padding,a); - } - - private static byte[] getPadding(int length) { - byte[] padding = new byte[length]; - for (int i = 0; i < length; i++) { - padding[i] = 0; - } - return padding; - } - - /** - * @param a array - * @param length new array size - * @return Value in a plus length appended 0 bytes - */ - public static byte [] padTail(final byte [] a, final int length) { - byte[] padding = getPadding(length); - return add(a,padding); - } - - - - /** - * @param bytes array to hash - * @param offset offset to start from - * @param length length to hash - * @return hash code - * */ - public static int hashCode(byte[] bytes, int offset, int length) { - int hash = 1; - for (int i = offset; i < offset + length; i++) { - hash = (31 * hash) + (int) bytes[i]; - } - return hash; - } - - /** - * @param t operands - * @return Array of byte arrays made from passed array of Text - */ - public static byte [][] toByteArrays(final String [] t) { - byte [][] result = new byte[t.length][]; - for (int i = 0; i < t.length; i++) { - result[i] = Bytes.toBytes(t[i]); - } - return result; - } - - /** - * @param column operand - * @return A byte array of a byte array where first and only entry is - * column - */ - public static byte [][] toByteArrays(final String column) { - return toByteArrays(toBytes(column)); - } - - /** - * @param column operand - * @return A byte array of a byte array where first and only entry is - * column - */ - public static byte [][] toByteArrays(final byte [] column) { - byte [][] result = new byte[1][]; - result[0] = column; - return result; - } - - -} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index 993310f6ec..39bcd922c8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.common.process.HttpProperty; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters; -import org.apache.dolphinscheduler.common.utils.Bytes; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; @@ -50,6 +49,7 @@ import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -176,7 +176,7 @@ public class HttpTask extends AbstractTask { if (entity == null) { return null; } - String webPage = EntityUtils.toString(entity, Bytes.UTF8_ENCODING); + String webPage = EntityUtils.toString(entity, StandardCharsets.UTF_8.name()); return webPage; } @@ -264,7 +264,7 @@ public class HttpTask extends AbstractTask { } } StringEntity postingString = new StringEntity(jsonParam.toString(), Charsets.UTF_8); - postingString.setContentEncoding(Bytes.UTF8_ENCODING); + postingString.setContentEncoding(StandardCharsets.UTF_8.name()); postingString.setContentType(APPLICATION_JSON); builder.setEntity(postingString); }