From d74c80e2a1f1de96f0bb31385005d50e8516bd5f Mon Sep 17 00:00:00 2001 From: gaojun2048 Date: Fri, 18 Mar 2022 08:16:01 +0800 Subject: [PATCH] [Fix-Test][LoggerServer] fix LoggerServer UT (#8973) * [Fix-Test][LoggerServer] fix LoggerServer UT * fix checkstyle * fix code style * fix code style * fix code style * fix code style --- .../server/log/LoggerRequestProcessor.java | 36 ++++++++-------- .../server/log/MasterLogFilter.java | 8 ++-- .../log/LoggerRequestProcessorTest.java | 21 +++++++--- .../server/log/LoggerServerTest.java | 41 ++++++++++++------- 4 files changed, 67 insertions(+), 39 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java index fb92849159..78a4d32c29 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.log; +import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.Command; @@ -31,6 +32,9 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; import java.io.File; @@ -46,16 +50,11 @@ import java.util.concurrent.Executors; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.Channel; - /** * logger request process logic */ -public class LoggerRequestProcessor implements NettyRequestProcessor { +public class LoggerRequestProcessor + implements NettyRequestProcessor { private final Logger logger = LoggerFactory.getLogger(LoggerRequestProcessor.class); @@ -128,7 +127,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { if (taskLogFile.exists()) { status = taskLogFile.delete(); } - } catch (Exception e) { + } + catch (Exception e) { status = false; } @@ -142,6 +142,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { private boolean checkPathSecurity(String path) { String dsHome = System.getProperty("DOLPHINSCHEDULER_HOME"); + // if we run server in IDE, user.dir is the DS Home. if (StringUtils.isBlank(dsHome)) { dsHome = System.getProperty("user.dir"); } @@ -149,7 +150,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { if (path.startsWith(dsHome) && !path.contains("../") && path.endsWith(".log")) { return true; } - } else { + } + else { logger.warn("path is null"); } return false; @@ -168,14 +170,15 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { */ private byte[] getFileContentBytes(String filePath) { try (InputStream in = new FileInputStream(filePath); - ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + ByteArrayOutputStream bos = new ByteArrayOutputStream()) { byte[] buf = new byte[1024]; int len; while ((len = in.read(buf)) != -1) { bos.write(buf, 0, len); } return bos.toByteArray(); - } catch (IOException e) { + } + catch (IOException e) { logger.error("get file bytes error", e); } return new byte[0]; @@ -190,19 +193,20 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { * @return part file content */ private List readPartFileContent(String filePath, - int skipLine, - int limit) { + int skipLine, + int limit) { File file = new File(filePath); if (file.exists() && file.isFile()) { try (Stream stream = Files.lines(Paths.get(filePath))) { return stream.skip(skipLine).limit(limit).collect(Collectors.toList()); - } catch (IOException e) { + } + catch (IOException e) { logger.error("read file error", e); } - } else { + } + else { logger.info("file path: {} not exists", filePath); } return Collections.emptyList(); } - } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java index 575571d9ac..255b0c5d76 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java @@ -24,7 +24,8 @@ import ch.qos.logback.core.spi.FilterReply; /** * master log filter */ -public class MasterLogFilter extends Filter { +public class MasterLogFilter + extends Filter { /** * log level */ @@ -32,12 +33,13 @@ public class MasterLogFilter extends Filter { /** * Accept or reject based on thread name + * * @param event event * @return FilterReply */ @Override public FilterReply decide(ILoggingEvent event) { - if (event.getThreadName().startsWith("Master-") ){ + if (event.getThreadName().startsWith("Master-")) { return FilterReply.ACCEPT; } return FilterReply.DENY; @@ -46,4 +48,4 @@ public class MasterLogFilter extends Filter { public void setLevel(String level) { this.level = Level.toLevel(level); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java index b10c60a6cd..13eb520db9 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java @@ -17,26 +17,37 @@ package org.apache.dolphinscheduler.server.log; +import io.netty.channel.Channel; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; - +import org.junit.Before; import org.junit.Test; -import org.junit.Test.None; import org.junit.runner.RunWith; import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import io.netty.channel.Channel; - @RunWith(PowerMockRunner.class) @PrepareForTest({LoggerUtils.class}) public class LoggerRequestProcessorTest { + private String dsHome; + + @Before + public void initDsHome() { + // DOLPHINSCHEDULER_HOME is be set in start.sh. if we run test in IDE user.dir is DS Home. + dsHome = System.getProperty("DOLPHINSCHEDULER_HOME"); + if (StringUtils.isBlank(dsHome)) { + dsHome = System.getProperty("user.dir"); + System.setProperty("DOLPHINSCHEDULER_HOME", dsHome); + } + } + @Test public void testProcessViewWholeLogRequest() { System.setProperty("DOLPHINSCHEDULER_HOME", System.getProperty("user.dir")); @@ -107,4 +118,4 @@ public class LoggerRequestProcessorTest { LoggerRequestProcessor loggerRequestProcessor = new LoggerRequestProcessor(); loggerRequestProcessor.process(channel, command); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java index 74bdceae8b..0dc5fb8f2e 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java @@ -17,57 +17,68 @@ package org.apache.dolphinscheduler.server.log; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.service.log.LogClientService; - -import org.apache.commons.lang.StringUtils; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; - import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; + public class LoggerServerTest { private LoggerServer loggerServer; private LogClientService logClientService; + private String dsHome; + @Before public void startServerAndClient() { this.loggerServer = new LoggerServer(); this.loggerServer.start(); this.logClientService = new LogClientService(); + + // DOLPHINSCHEDULER_HOME is be set in start.sh. if we run test in IDE user.dir is DS Home. + dsHome = System.getProperty("DOLPHINSCHEDULER_HOME"); + if (StringUtils.isBlank(dsHome)) { + dsHome = System.getProperty("user.dir"); + System.setProperty("DOLPHINSCHEDULER_HOME", dsHome); + } } @Test - public void testRollViewLog() throws IOException { + public void testRollViewLog() + throws IOException { String expectedTmpDemoString = "testRolloViewLog"; - org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/demo.txt"), expectedTmpDemoString, Charset.defaultCharset()); + String testFile = dsHome + "/tmp/demo.log"; + org.apache.commons.io.FileUtils.writeStringToFile(new File(testFile), expectedTmpDemoString, Charset.defaultCharset()); String resultTmpDemoString = this.logClientService.rollViewLog( - "localhost", Constants.RPC_PORT,"/tmp/demo.txt", 0, 1000); + "localhost", Constants.RPC_PORT, testFile, 0, 1000); Assert.assertEquals(expectedTmpDemoString, resultTmpDemoString.replaceAll("[\r|\n|\t]", StringUtils.EMPTY)); - FileUtils.deleteFile("/tmp/demo.txt"); + FileUtils.deleteFile(testFile); } @Test - public void testRemoveTaskLog() throws IOException { + public void testRemoveTaskLog() + throws IOException { String expectedTmpRemoveString = "testRemoveTaskLog"; - org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/remove.txt"), expectedTmpRemoveString, Charset.defaultCharset()); + String testFile = dsHome + "/tmp/remove.log"; + org.apache.commons.io.FileUtils.writeStringToFile(new File(testFile), expectedTmpRemoveString, Charset.defaultCharset()); - Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/tmp/remove.txt"); + Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT, testFile); Assert.assertTrue(b); - String result = this.logClientService.viewLog("localhost", Constants.RPC_PORT,"/tmp/demo.txt"); + String result = this.logClientService.viewLog("localhost", Constants.RPC_PORT, testFile); Assert.assertEquals(StringUtils.EMPTY, result); }