From ebf7a980d2922a5659d8f0da28f519556aa3a2f0 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Tue, 10 Mar 2020 18:59:45 +0800 Subject: [PATCH] Adapting partial code(file name start with F) to the sonar cloud rule (#2045) * Adapting partial code(file name start with F) to the sonar cloud rule * add more unit test * add License * add includes configuration to maven-surefire-plugin * fix getResourceFilesList incorrect logic --- .../alert/utils/FuncUtils.java | 2 +- .../alert/utils/FuncUtilsTest.java | 2 +- .../common/task/flink/FlinkParameters.java | 14 +++-- .../common/utils/FileUtils.java | 6 +- .../common/task/FlinkParametersTest.java | 55 +++++++++++++++++++ .../common/utils/FileUtilsTest.java | 17 +++--- .../server/utils/FlinkArgsUtils.java | 16 ++---- .../worker/runner/TaskScheduleThread.java | 4 +- .../server/utils/FlinkArgsUtilsTest.java | 24 ++++---- pom.xml | 1 + 10 files changed, 97 insertions(+), 44 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java index dd6ca4b8a6..d68532a82b 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java @@ -20,7 +20,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils; public class FuncUtils { - static public String mkString(Iterable list, String split) { + public static String mkString(Iterable list, String split) { if (null == list || StringUtils.isEmpty(split)){ return null; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java index e2b25d961d..a4aeea9c0c 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java @@ -46,7 +46,7 @@ public class FuncUtilsTest { logger.info(result); //Expected result string - assertEquals(result, "user1|user2|user3"); + assertEquals("user1|user2|user3", result); //Null list expected return null result = FuncUtils.mkString(null, split); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java index 0638b3858e..1fbd9ab354 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -207,12 +208,15 @@ public class FlinkParameters extends AbstractParameters { @Override public List getResourceFilesList() { - if(resourceList !=null ) { - this.resourceList.add(mainJar); - return resourceList.stream() - .map(p -> p.getRes()).collect(Collectors.toList()); + if(resourceList != null ) { + List resourceFiles = resourceList.stream() + .map(ResourceInfo::getRes).collect(Collectors.toList()); + if(mainJar != null) { + resourceFiles.add(mainJar.getRes()); + } + return resourceFiles; } - return null; + return Collections.emptyList(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index c84848fbae..dc60b04c59 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -44,7 +44,7 @@ public class FileUtils { String fileSuffix = ""; if (StringUtils.isNotEmpty(filename)) { - int lastIndex = filename.lastIndexOf("."); + int lastIndex = filename.lastIndexOf('.'); if (lastIndex > 0) { fileSuffix = filename.substring(lastIndex + 1); } @@ -325,10 +325,8 @@ public class FileUtils { } } else { File parent = file.getParentFile(); - if (parent != null) { - if (!parent.mkdirs() && !parent.isDirectory()) { + if (parent != null && !parent.mkdirs() && !parent.isDirectory()) { throw new IOException("Directory '" + parent + "' could not be created"); - } } } return new FileOutputStream(file, append); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java new file mode 100644 index 0000000000..7ce00e875a --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.task; + +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; +import org.junit.Assert; +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; + +public class FlinkParametersTest { + @Test + public void getResourceFilesList() { + FlinkParameters flinkParameters = new FlinkParameters(); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertTrue(flinkParameters.getResourceFilesList().isEmpty()); + + ResourceInfo mainResource = new ResourceInfo(); + mainResource.setRes("testFlinkMain-1.0.0-SNAPSHOT.jar"); + flinkParameters.setMainJar(mainResource); + + List resourceInfos = new LinkedList<>(); + ResourceInfo resourceInfo1 = new ResourceInfo(); + resourceInfo1.setRes("testFlinkParameters1.jar"); + resourceInfos.add(resourceInfo1); + + flinkParameters.setResourceList(resourceInfos); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertEquals(2, flinkParameters.getResourceFilesList().size()); + + ResourceInfo resourceInfo2 = new ResourceInfo(); + resourceInfo2.setRes("testFlinkParameters2.jar"); + resourceInfos.add(resourceInfo2); + + flinkParameters.setResourceList(resourceInfos); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertEquals(3, flinkParameters.getResourceFilesList().size()); + } +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index 89458f6f1c..96217842bf 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -30,29 +30,32 @@ public class FileUtilsTest { @Test public void suffix() { - Assert.assertEquals(FileUtils.suffix("ninfor.java"),"java"); + Assert.assertEquals("java", FileUtils.suffix("ninfor.java")); + Assert.assertEquals("", FileUtils.suffix(null)); + Assert.assertEquals("", FileUtils.suffix("")); + Assert.assertEquals("", FileUtils.suffix("ninfor-java")); } @Test public void testGetDownloadFilename() { PowerMockito.mockStatic(DateUtils.class); PowerMockito.when(DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059"); - Assert.assertEquals(FileUtils.getDownloadFilename("test"), - "/tmp/dolphinscheduler/download/20190101101059/test"); + Assert.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test", + FileUtils.getDownloadFilename("test")); } @Test public void testGetUploadFilename() { - Assert.assertEquals(FileUtils.getUploadFilename("aaa","bbb"), - "/tmp/dolphinscheduler/aaa/resources/bbb"); + Assert.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb", + FileUtils.getUploadFilename("aaa","bbb")); } @Test public void testGetProcessExecDir() { String dir = FileUtils.getProcessExecDir(1,2,3, 4); - Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3/4"); + Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3/4", dir); dir = FileUtils.getProcessExecDir(1,2,3); - Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3"); + Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3", dir); } @Test diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java index 4c33ef8db2..12c7eb2d56 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.server.utils; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; -import org.apache.commons.lang.StringUtils; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -32,12 +31,7 @@ import java.util.List; * spark args utils */ public class FlinkArgsUtils { - - /** - * logger of FlinkArgsUtils - */ - private static final org.slf4j.Logger logger = LoggerFactory.getLogger(FlinkArgsUtils.class); - + private static final String LOCAL_DEPLOY_MODE = "local"; /** * build args * @param param flink parameters @@ -52,7 +46,7 @@ public class FlinkArgsUtils { deployMode = tmpDeployMode; } - if (!"local".equals(deployMode)) { + if (!LOCAL_DEPLOY_MODE.equals(deployMode)) { args.add(Constants.FLINK_RUN_MODE); //-m args.add(Constants.FLINK_YARN_CLUSTER); //yarn-cluster @@ -113,12 +107,12 @@ public class FlinkArgsUtils { String queue = param.getQueue(); if (StringUtils.isNotEmpty(others)) { - if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) { + if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) { args.add(Constants.FLINK_QUEUE); args.add(param.getQueue()); } args.add(others); - } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) { + } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) { args.add(Constants.FLINK_QUEUE); args.add(param.getQueue()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java index 5e68acf94e..21ee1dfa40 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java @@ -296,9 +296,7 @@ public class TaskScheduleThread implements Runnable { if (baseParam != null) { List projectResourceFiles = baseParam.getResourceFilesList(); - if (projectResourceFiles != null) { - projectFiles.addAll(projectResourceFiles); - } + projectFiles.addAll(projectResourceFiles); } return new ArrayList<>(projectFiles); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java index 710d2c2505..2e4861e2a2 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java @@ -87,35 +87,35 @@ public class FlinkArgsUtilsTest { } //Expected values and order - assertEquals(result.size(),20); + assertEquals(20, result.size()); - assertEquals(result.get(0),"-m"); - assertEquals(result.get(1),"yarn-cluster"); + assertEquals("-m", result.get(0)); + assertEquals("yarn-cluster", result.get(1)); - assertEquals(result.get(2),"-ys"); + assertEquals("-ys", result.get(2)); assertSame(Integer.valueOf(result.get(3)),slot); - assertEquals(result.get(4),"-ynm"); + assertEquals("-ynm",result.get(4)); assertEquals(result.get(5),appName); - assertEquals(result.get(6),"-yn"); + assertEquals("-yn", result.get(6)); assertSame(Integer.valueOf(result.get(7)),taskManager); - assertEquals(result.get(8),"-yjm"); + assertEquals("-yjm", result.get(8)); assertEquals(result.get(9),jobManagerMemory); - assertEquals(result.get(10),"-ytm"); + assertEquals("-ytm", result.get(10)); assertEquals(result.get(11),taskManagerMemory); - assertEquals(result.get(12),"-d"); + assertEquals("-d", result.get(12)); - assertEquals(result.get(13),"-c"); + assertEquals("-c", result.get(13)); assertEquals(result.get(14),mainClass); assertEquals(result.get(15),mainJar.getRes()); assertEquals(result.get(16),mainArgs); - assertEquals(result.get(17),"--qu"); + assertEquals("--qu", result.get(17)); assertEquals(result.get(18),queue); assertEquals(result.get(19),others); @@ -125,7 +125,7 @@ public class FlinkArgsUtilsTest { param1.setQueue(queue); param1.setDeployMode(mode); result = FlinkArgsUtils.buildArgs(param1); - assertEquals(result.size(),5); + assertEquals(5, result.size()); } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index a7feec0e14..8b91add6d7 100644 --- a/pom.xml +++ b/pom.xml @@ -691,6 +691,7 @@ **/common/threadutils/*.java **/common/graph/*.java **/common/queue/*.java + **/common/task/FlinkParametersTest.java **/common/task/SqoopParameterEntityTest.java **/api/utils/CheckUtilsTest.java **/api/utils/FileUtilsTest.java