From 0b873e68705acab7cbd23b7c28c2844a2070c1a8 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 15 Jan 2024 11:53:55 +0800 Subject: [PATCH] [HotFix] Fix createTaskInstanceWorkingDirectory failed if the old path exist (#15377) --- .../common/utils/FileUtils.java | 12 ++-- .../common/utils/FileUtilsTest.java | 2 +- .../service/utils/ProcessUtils.java | 2 +- .../utils/TaskExecutionContextUtils.java | 10 ++- .../utils/TaskExecutionContextUtilsTest.java | 64 +++++++++++++++++++ 5 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index a995b29dd6..c71c5f2e47 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -117,12 +117,12 @@ public class FileUtils { * @param taskInstanceId task instance id * @return directory of process execution */ - public static String getProcessExecDir(String tenant, - long projectCode, - long processDefineCode, - int processDefineVersion, - int processInstanceId, - int taskInstanceId) { + public static String getTaskInstanceWorkingDirectory(String tenant, + long projectCode, + long processDefineCode, + int processDefineVersion, + int processInstanceId, + int taskInstanceId) { return String.format( "%s/exec/process/%s/%d/%d_%d/%d/%d", DATA_BASEDIR, diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index faa99e2a32..0ce33bef52 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -50,7 +50,7 @@ public class FileUtilsTest { @Test public void testGetProcessExecDir() { - String dir = FileUtils.getProcessExecDir("test", 1L, 2L, 1, 3, 4); + String dir = FileUtils.getTaskInstanceWorkingDirectory("test", 1L, 2L, 1, 3, 4); Assertions.assertEquals("/tmp/dolphinscheduler/exec/process/test/1/2_1/3/4", dir); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java index 71f224334b..6b97590c89 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java @@ -168,7 +168,7 @@ public class ProcessUtils { taskExecutionContext.setAppIds(String.join(TaskConstants.COMMA, appIds)); if (StringUtils.isEmpty(taskExecutionContext.getExecutePath())) { taskExecutionContext - .setExecutePath(FileUtils.getProcessExecDir( + .setExecutePath(FileUtils.getTaskInstanceWorkingDirectory( taskExecutionContext.getTenantCode(), taskExecutionContext.getProjectCode(), taskExecutionContext.getProcessDefineCode(), diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java index 42d2894f8a..ddd489adeb 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java @@ -37,7 +37,6 @@ import org.apache.commons.lang3.SystemUtils; import java.io.File; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; @@ -85,7 +84,7 @@ public class TaskExecutionContextUtils { public static void createTaskInstanceWorkingDirectory(TaskExecutionContext taskExecutionContext) throws TaskException { // local execute path - String taskInstanceWorkingDirectory = FileUtils.getProcessExecDir( + String taskInstanceWorkingDirectory = FileUtils.getTaskInstanceWorkingDirectory( taskExecutionContext.getTenantCode(), taskExecutionContext.getProjectCode(), taskExecutionContext.getProcessDefineCode(), @@ -93,13 +92,12 @@ public class TaskExecutionContextUtils { taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId()); try { - Path path = Paths.get(taskInstanceWorkingDirectory); - if (Files.deleteIfExists(path)) { + if (new File(taskInstanceWorkingDirectory).exists()) { + FileUtils.deleteFile(taskInstanceWorkingDirectory); log.warn("The TaskInstance WorkingDirectory: {} is exist, will recreate again", taskInstanceWorkingDirectory); } - Files.createDirectories(path); - taskExecutionContext.setExecutePath(taskInstanceWorkingDirectory); + Files.createDirectories(Paths.get(taskInstanceWorkingDirectory)); taskExecutionContext.setExecutePath(taskInstanceWorkingDirectory); taskExecutionContext.setAppInfoPath(FileUtils.getAppInfoPath(taskInstanceWorkingDirectory)); diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java new file mode 100644 index 0000000000..b73d073ef6 --- /dev/null +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.utils; + +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class TaskExecutionContextUtilsTest { + + @Test + void createTaskInstanceWorkingDirectory() throws IOException { + TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setTenantCode("tenantCode"); + taskExecutionContext.setProjectCode(1); + taskExecutionContext.setProcessDefineCode(1L); + taskExecutionContext.setProcessDefineVersion(1); + taskExecutionContext.setProcessInstanceId(1); + taskExecutionContext.setTaskInstanceId(1); + + String taskWorkingDirectory = FileUtils.getTaskInstanceWorkingDirectory( + taskExecutionContext.getTenantCode(), + taskExecutionContext.getProjectCode(), + taskExecutionContext.getProcessDefineCode(), + taskExecutionContext.getProcessDefineVersion(), + taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId()); + try { + // Test if the working directory is exist + // will delete it and recreate + Files.createDirectories(Paths.get(taskWorkingDirectory)); + Files.createFile(Paths.get(taskWorkingDirectory, "text.txt")); + Assertions.assertTrue(Files.exists(Paths.get(taskWorkingDirectory, "text.txt"))); + + TaskExecutionContextUtils.createTaskInstanceWorkingDirectory(taskExecutionContext); + + Assertions.assertEquals(taskWorkingDirectory, taskExecutionContext.getExecutePath()); + Assertions.assertFalse(Files.exists(Paths.get(taskWorkingDirectory, "text.txt"))); + } finally { + FileUtils.deleteFile(taskWorkingDirectory); + } + } +}