Browse Source

[Fix-Test][LoggerServer] fix LoggerServer UT (#8973)

* [Fix-Test][LoggerServer] fix LoggerServer UT

* fix checkstyle

* fix code style

* fix code style

* fix code style

* fix code style
2.0.7-release
gaojun2048 3 years ago committed by GitHub
parent
commit
d74c80e2a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 30
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
  2. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java
  3. 19
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
  4. 41
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java

30
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.server.log; package org.apache.dolphinscheduler.server.log;
import io.netty.channel.Channel;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
@ -31,6 +32,9 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
@ -46,16 +50,11 @@ import java.util.concurrent.Executors;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.netty.channel.Channel;
/** /**
* logger request process logic * logger request process logic
*/ */
public class LoggerRequestProcessor implements NettyRequestProcessor { public class LoggerRequestProcessor
implements NettyRequestProcessor {
private final Logger logger = LoggerFactory.getLogger(LoggerRequestProcessor.class); private final Logger logger = LoggerFactory.getLogger(LoggerRequestProcessor.class);
@ -128,7 +127,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
if (taskLogFile.exists()) { if (taskLogFile.exists()) {
status = taskLogFile.delete(); status = taskLogFile.delete();
} }
} catch (Exception e) { }
catch (Exception e) {
status = false; status = false;
} }
@ -142,6 +142,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
private boolean checkPathSecurity(String path) { private boolean checkPathSecurity(String path) {
String dsHome = System.getProperty("DOLPHINSCHEDULER_HOME"); String dsHome = System.getProperty("DOLPHINSCHEDULER_HOME");
// if we run server in IDE, user.dir is the DS Home.
if (StringUtils.isBlank(dsHome)) { if (StringUtils.isBlank(dsHome)) {
dsHome = System.getProperty("user.dir"); dsHome = System.getProperty("user.dir");
} }
@ -149,7 +150,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
if (path.startsWith(dsHome) && !path.contains("../") && path.endsWith(".log")) { if (path.startsWith(dsHome) && !path.contains("../") && path.endsWith(".log")) {
return true; return true;
} }
} else { }
else {
logger.warn("path is null"); logger.warn("path is null");
} }
return false; return false;
@ -175,7 +177,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
bos.write(buf, 0, len); bos.write(buf, 0, len);
} }
return bos.toByteArray(); return bos.toByteArray();
} catch (IOException e) { }
catch (IOException e) {
logger.error("get file bytes error", e); logger.error("get file bytes error", e);
} }
return new byte[0]; return new byte[0];
@ -196,13 +199,14 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
if (file.exists() && file.isFile()) { if (file.exists() && file.isFile()) {
try (Stream<String> stream = Files.lines(Paths.get(filePath))) { try (Stream<String> stream = Files.lines(Paths.get(filePath))) {
return stream.skip(skipLine).limit(limit).collect(Collectors.toList()); return stream.skip(skipLine).limit(limit).collect(Collectors.toList());
} catch (IOException e) { }
catch (IOException e) {
logger.error("read file error", e); logger.error("read file error", e);
} }
} else { }
else {
logger.info("file path: {} not exists", filePath); logger.info("file path: {} not exists", filePath);
} }
return Collections.emptyList(); return Collections.emptyList();
} }
} }

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java

@ -24,7 +24,8 @@ import ch.qos.logback.core.spi.FilterReply;
/** /**
* master log filter * master log filter
*/ */
public class MasterLogFilter extends Filter<ILoggingEvent> { public class MasterLogFilter
extends Filter<ILoggingEvent> {
/** /**
* log level * log level
*/ */
@ -32,6 +33,7 @@ public class MasterLogFilter extends Filter<ILoggingEvent> {
/** /**
* Accept or reject based on thread name * Accept or reject based on thread name
*
* @param event event * @param event event
* @return FilterReply * @return FilterReply
*/ */

19
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java

@ -17,26 +17,37 @@
package org.apache.dolphinscheduler.server.log; package org.apache.dolphinscheduler.server.log;
import io.netty.channel.Channel;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.Test.None;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito; import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunner;
import io.netty.channel.Channel;
@RunWith(PowerMockRunner.class) @RunWith(PowerMockRunner.class)
@PrepareForTest({LoggerUtils.class}) @PrepareForTest({LoggerUtils.class})
public class LoggerRequestProcessorTest { public class LoggerRequestProcessorTest {
private String dsHome;
@Before
public void initDsHome() {
// DOLPHINSCHEDULER_HOME is be set in start.sh. if we run test in IDE user.dir is DS Home.
dsHome = System.getProperty("DOLPHINSCHEDULER_HOME");
if (StringUtils.isBlank(dsHome)) {
dsHome = System.getProperty("user.dir");
System.setProperty("DOLPHINSCHEDULER_HOME", dsHome);
}
}
@Test @Test
public void testProcessViewWholeLogRequest() { public void testProcessViewWholeLogRequest() {
System.setProperty("DOLPHINSCHEDULER_HOME", System.getProperty("user.dir")); System.setProperty("DOLPHINSCHEDULER_HOME", System.getProperty("user.dir"));

41
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java

@ -17,57 +17,68 @@
package org.apache.dolphinscheduler.server.log; package org.apache.dolphinscheduler.server.log;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.service.log.LogClientService; import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.commons.lang.StringUtils;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
public class LoggerServerTest { public class LoggerServerTest {
private LoggerServer loggerServer; private LoggerServer loggerServer;
private LogClientService logClientService; private LogClientService logClientService;
private String dsHome;
@Before @Before
public void startServerAndClient() { public void startServerAndClient() {
this.loggerServer = new LoggerServer(); this.loggerServer = new LoggerServer();
this.loggerServer.start(); this.loggerServer.start();
this.logClientService = new LogClientService(); this.logClientService = new LogClientService();
// DOLPHINSCHEDULER_HOME is be set in start.sh. if we run test in IDE user.dir is DS Home.
dsHome = System.getProperty("DOLPHINSCHEDULER_HOME");
if (StringUtils.isBlank(dsHome)) {
dsHome = System.getProperty("user.dir");
System.setProperty("DOLPHINSCHEDULER_HOME", dsHome);
}
} }
@Test @Test
public void testRollViewLog() throws IOException { public void testRollViewLog()
throws IOException {
String expectedTmpDemoString = "testRolloViewLog"; String expectedTmpDemoString = "testRolloViewLog";
org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/demo.txt"), expectedTmpDemoString, Charset.defaultCharset()); String testFile = dsHome + "/tmp/demo.log";
org.apache.commons.io.FileUtils.writeStringToFile(new File(testFile), expectedTmpDemoString, Charset.defaultCharset());
String resultTmpDemoString = this.logClientService.rollViewLog( String resultTmpDemoString = this.logClientService.rollViewLog(
"localhost", Constants.RPC_PORT,"/tmp/demo.txt", 0, 1000); "localhost", Constants.RPC_PORT, testFile, 0, 1000);
Assert.assertEquals(expectedTmpDemoString, resultTmpDemoString.replaceAll("[\r|\n|\t]", StringUtils.EMPTY)); Assert.assertEquals(expectedTmpDemoString, resultTmpDemoString.replaceAll("[\r|\n|\t]", StringUtils.EMPTY));
FileUtils.deleteFile("/tmp/demo.txt"); FileUtils.deleteFile(testFile);
} }
@Test @Test
public void testRemoveTaskLog() throws IOException { public void testRemoveTaskLog()
throws IOException {
String expectedTmpRemoveString = "testRemoveTaskLog"; String expectedTmpRemoveString = "testRemoveTaskLog";
org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/remove.txt"), expectedTmpRemoveString, Charset.defaultCharset()); String testFile = dsHome + "/tmp/remove.log";
org.apache.commons.io.FileUtils.writeStringToFile(new File(testFile), expectedTmpRemoveString, Charset.defaultCharset());
Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/tmp/remove.txt"); Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT, testFile);
Assert.assertTrue(b); Assert.assertTrue(b);
String result = this.logClientService.viewLog("localhost", Constants.RPC_PORT,"/tmp/demo.txt"); String result = this.logClientService.viewLog("localhost", Constants.RPC_PORT, testFile);
Assert.assertEquals(StringUtils.EMPTY, result); Assert.assertEquals(StringUtils.EMPTY, result);
} }

Loading…
Cancel
Save