From 4d9db3451ed7d038f5cf7815e61b669cdcf8164a Mon Sep 17 00:00:00 2001 From: zhuxt2015 <594754793@qq.com> Date: Tue, 26 Jul 2022 10:14:45 +0800 Subject: [PATCH] [Bug] [API] If the single log length is long, it will causes the View Log page to hang (#10612) --- .../api/controller/LoggerController.java | 9 +++--- .../api/service/LoggerService.java | 3 +- .../api/service/impl/LoggerServiceImpl.java | 10 +++--- .../impl/ProcessInstanceServiceImpl.java | 5 +-- .../dao/entity/ResponseTaskLog.java | 31 +++++++++++++++++++ .../server/log/LoggerRequestProcessor.java | 19 +++++++++++- .../views/projects/task/instance/index.tsx | 8 ++--- 7 files changed, 69 insertions(+), 16 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java index 88c715fab6..f24ee0273b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.LoggerService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog; import org.apache.dolphinscheduler.dao.entity.User; import org.springframework.beans.factory.annotation.Autowired; @@ -77,10 +78,10 @@ public class LoggerController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_TASK_INSTANCE_LOG_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") - public Result queryLog(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "taskInstanceId") int taskInstanceId, - @RequestParam(value = "skipLineNum") int skipNum, - @RequestParam(value = "limit") int limit) { + public Result queryLog(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "taskInstanceId") int taskInstanceId, + @RequestParam(value = "skipLineNum") int skipNum, + @RequestParam(value = "limit") int limit) { return loggerService.queryLog(taskInstanceId, skipNum, limit); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java index b252522c7c..f7ba946e4c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog; import org.apache.dolphinscheduler.dao.entity.User; import java.util.Map; @@ -35,7 +36,7 @@ public interface LoggerService { * @param limit limit * @return log string data */ - Result queryLog(int taskInstId, int skipLineNum, int limit); + Result queryLog(int taskInstId, int skipLineNum, int limit); /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index 65aa7b3aee..39f9208b5f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.User; @@ -100,7 +101,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService */ @Override @SuppressWarnings("unchecked") - public Result queryLog(int taskInstId, int skipLineNum, int limit) { + public Result queryLog(int taskInstId, int skipLineNum, int limit) { TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); @@ -110,9 +111,10 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService if (StringUtils.isBlank(taskInstance.getHost())) { return Result.error(Status.TASK_INSTANCE_HOST_IS_NULL); } - Result result = new Result<>(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); - String log = queryLog(taskInstance,skipLineNum,limit); - result.setData(log); + Result result = new Result<>(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); + String log = queryLog(taskInstance, skipLineNum, limit); + int lineNum = log.split("\\r\\n").length; + result.setData(new ResponseTaskLog(lineNum, log)); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java index 2335ff4c88..99427b954a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java @@ -53,6 +53,7 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -352,10 +353,10 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce private void addDependResultForTaskList(List taskInstanceList) throws IOException { for (TaskInstance taskInstance : taskInstanceList) { if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) { - Result logResult = loggerService.queryLog( + Result logResult = loggerService.queryLog( taskInstance.getId(), Constants.LOG_QUERY_SKIP_LINE_NUMBER, Constants.LOG_QUERY_LIMIT); if (logResult.getCode() == Status.SUCCESS.ordinal()) { - String log = logResult.getData(); + String log = logResult.getData().getMessage(); Map resultMap = parseLogForDependentResult(log); taskInstance.setDependentResult(JSONUtils.toJsonString(resultMap)); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java new file mode 100644 index 0000000000..6876c060f8 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * log of the logger service response + */ +@Data +@AllArgsConstructor +public class ResponseTaskLog { + private int lineNum; + private String message; +} diff --git a/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java index 1ab4aa13fe..b5cae3562d 100644 --- a/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -40,6 +40,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Collections; @@ -113,8 +114,24 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { List lines = readPartFileContent(rollViewLogPath, rollViewLogRequest.getSkipLineNum(), rollViewLogRequest.getLimit()); StringBuilder builder = new StringBuilder(); + final int MaxResponseLogSize = 65535; + int totalLogByteSize = 0; for (String line : lines) { - builder.append(line).append("\r\n"); + //If a single line of log is exceed max response size, cut off the line + final int lineByteSize = line.getBytes(StandardCharsets.UTF_8).length; + if (lineByteSize >= MaxResponseLogSize) { + builder.append(line, 0, MaxResponseLogSize) + .append(" [this line's size ").append(lineByteSize).append(" bytes is exceed ") + .append(MaxResponseLogSize).append(" bytes, so only ") + .append(MaxResponseLogSize).append(" characters are reserved for performance reasons.]") + .append("\r\n"); + } else { + builder.append(line).append("\r\n"); + } + totalLogByteSize += lineByteSize; + if (totalLogByteSize >= MaxResponseLogSize) { + break; + } } RollViewLogResponseCommand rollViewLogRequestResponse = new RollViewLogResponseCommand(builder.toString()); channel.writeAndFlush(rollViewLogRequestResponse.convert2Command(command.getOpaque())); diff --git a/dolphinscheduler-ui/src/views/projects/task/instance/index.tsx b/dolphinscheduler-ui/src/views/projects/task/instance/index.tsx index 9986f8844a..c742a20d20 100644 --- a/dolphinscheduler-ui/src/views/projects/task/instance/index.tsx +++ b/dolphinscheduler-ui/src/views/projects/task/instance/index.tsx @@ -76,11 +76,11 @@ const TaskInstance = defineComponent({ taskInstanceId: Number(row.id), limit: variables.limit, skipLineNum: variables.skipLineNum - }).then((res: string) => { - variables.logRef += res - if (res) { + }).then((res: any) => { + if (res?.message) { + variables.logRef += res.message variables.limit += 1000 - variables.skipLineNum += 1000 + variables.skipLineNum += res.lineNum getLogs(row) } else { variables.logLoadingRef = false