Browse Source

[Bug] [API] If the single log length is long, it will causes the View Log page to hang (#10612)

3.1.0-release
zhuxt2015 2 years ago committed by GitHub
parent
commit
4d9db3451e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java
  2. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java
  3. 10
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
  4. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
  5. 31
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java
  6. 17
      dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
  7. 8
      dolphinscheduler-ui/src/views/projects/task/instance/index.tsx

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java

@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.LoggerService; import org.apache.dolphinscheduler.api.service.LoggerService;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -77,7 +78,7 @@ public class LoggerController extends BaseController {
@ResponseStatus(HttpStatus.OK) @ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_TASK_INSTANCE_LOG_ERROR) @ApiException(QUERY_TASK_INSTANCE_LOG_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser") @AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result<String> queryLog(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, public Result<ResponseTaskLog> queryLog(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "taskInstanceId") int taskInstanceId, @RequestParam(value = "taskInstanceId") int taskInstanceId,
@RequestParam(value = "skipLineNum") int skipNum, @RequestParam(value = "skipLineNum") int skipNum,
@RequestParam(value = "limit") int limit) { @RequestParam(value = "limit") int limit) {

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.api.service; package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import java.util.Map; import java.util.Map;
@ -35,7 +36,7 @@ public interface LoggerService {
* @param limit limit * @param limit limit
* @return log string data * @return log string data
*/ */
Result<String> queryLog(int taskInstId, int skipLineNum, int limit); Result<ResponseTaskLog> queryLog(int taskInstId, int skipLineNum, int limit);
/** /**

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java

@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.api.service.ProjectService;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
@ -100,7 +101,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService
*/ */
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Result<String> queryLog(int taskInstId, int skipLineNum, int limit) { public Result<ResponseTaskLog> queryLog(int taskInstId, int skipLineNum, int limit) {
TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId);
@ -110,9 +111,10 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService
if (StringUtils.isBlank(taskInstance.getHost())) { if (StringUtils.isBlank(taskInstance.getHost())) {
return Result.error(Status.TASK_INSTANCE_HOST_IS_NULL); return Result.error(Status.TASK_INSTANCE_HOST_IS_NULL);
} }
Result<String> result = new Result<>(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); Result<ResponseTaskLog> result = new Result<>(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg());
String log = queryLog(taskInstance,skipLineNum,limit); String log = queryLog(taskInstance, skipLineNum, limit);
result.setData(log); int lineNum = log.split("\\r\\n").length;
result.setData(new ResponseTaskLog(lineNum, log));
return result; return result;
} }

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java

@ -53,6 +53,7 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@ -352,10 +353,10 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
private void addDependResultForTaskList(List<TaskInstance> taskInstanceList) throws IOException { private void addDependResultForTaskList(List<TaskInstance> taskInstanceList) throws IOException {
for (TaskInstance taskInstance : taskInstanceList) { for (TaskInstance taskInstance : taskInstanceList) {
if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) { if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) {
Result<String> logResult = loggerService.queryLog( Result<ResponseTaskLog> logResult = loggerService.queryLog(
taskInstance.getId(), Constants.LOG_QUERY_SKIP_LINE_NUMBER, Constants.LOG_QUERY_LIMIT); taskInstance.getId(), Constants.LOG_QUERY_SKIP_LINE_NUMBER, Constants.LOG_QUERY_LIMIT);
if (logResult.getCode() == Status.SUCCESS.ordinal()) { if (logResult.getCode() == Status.SUCCESS.ordinal()) {
String log = logResult.getData(); String log = logResult.getData().getMessage();
Map<String, DependResult> resultMap = parseLogForDependentResult(log); Map<String, DependResult> resultMap = parseLogForDependentResult(log);
taskInstance.setDependentResult(JSONUtils.toJsonString(resultMap)); taskInstance.setDependentResult(JSONUtils.toJsonString(resultMap));
} }

31
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
/**
* log of the logger service response
*/
@Data
@AllArgsConstructor
public class ResponseTaskLog {
private int lineNum;
private String message;
}

17
dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java

@ -40,6 +40,7 @@ import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.Collections; import java.util.Collections;
@ -113,9 +114,25 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
List<String> lines = readPartFileContent(rollViewLogPath, List<String> lines = readPartFileContent(rollViewLogPath,
rollViewLogRequest.getSkipLineNum(), rollViewLogRequest.getLimit()); rollViewLogRequest.getSkipLineNum(), rollViewLogRequest.getLimit());
StringBuilder builder = new StringBuilder(); StringBuilder builder = new StringBuilder();
final int MaxResponseLogSize = 65535;
int totalLogByteSize = 0;
for (String line : lines) { for (String line : lines) {
//If a single line of log is exceed max response size, cut off the line
final int lineByteSize = line.getBytes(StandardCharsets.UTF_8).length;
if (lineByteSize >= MaxResponseLogSize) {
builder.append(line, 0, MaxResponseLogSize)
.append(" [this line's size ").append(lineByteSize).append(" bytes is exceed ")
.append(MaxResponseLogSize).append(" bytes, so only ")
.append(MaxResponseLogSize).append(" characters are reserved for performance reasons.]")
.append("\r\n");
} else {
builder.append(line).append("\r\n"); builder.append(line).append("\r\n");
} }
totalLogByteSize += lineByteSize;
if (totalLogByteSize >= MaxResponseLogSize) {
break;
}
}
RollViewLogResponseCommand rollViewLogRequestResponse = new RollViewLogResponseCommand(builder.toString()); RollViewLogResponseCommand rollViewLogRequestResponse = new RollViewLogResponseCommand(builder.toString());
channel.writeAndFlush(rollViewLogRequestResponse.convert2Command(command.getOpaque())); channel.writeAndFlush(rollViewLogRequestResponse.convert2Command(command.getOpaque()));
break; break;

8
dolphinscheduler-ui/src/views/projects/task/instance/index.tsx

@ -76,11 +76,11 @@ const TaskInstance = defineComponent({
taskInstanceId: Number(row.id), taskInstanceId: Number(row.id),
limit: variables.limit, limit: variables.limit,
skipLineNum: variables.skipLineNum skipLineNum: variables.skipLineNum
}).then((res: string) => { }).then((res: any) => {
variables.logRef += res if (res?.message) {
if (res) { variables.logRef += res.message
variables.limit += 1000 variables.limit += 1000
variables.skipLineNum += 1000 variables.skipLineNum += res.lineNum
getLogs(row) getLogs(row)
} else { } else {
variables.logLoadingRef = false variables.logLoadingRef = false

Loading…
Cancel
Save