Browse Source

[Feature-16229] Add python using file e2e test case (#16240)

* feature 16229

* resolve conflicts
dev
xiangzihao 4 months ago committed by GitHub
parent
commit
1fe8a6e868
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 1
      .github/actions/workflow-telemetry-action
  2. 13
      .github/workflows/e2e.yml
  3. 3
      .gitmodules
  4. 2
      dolphinscheduler-dist/release-docs/LICENSE
  5. 338
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/tasks/PythonTaskE2ETest.java
  6. 28
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/tasks/ShellTaskE2ETest.java
  7. 27
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/workflow/BaseWorkflowE2ETest.java
  8. 30
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/models/environment/IEnvironment.java
  9. 52
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/models/environment/PythonEnvironment.java
  10. 80
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/common/CodeEditor.java
  11. 6
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowForm.java
  12. 7
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowRunDialog.java
  13. 49
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/PythonTaskForm.java
  14. 3
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/TaskNodeForm.java
  15. 11
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/security/EnvironmentPage.java
  16. 25
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/python-task/Dockerfile
  17. 40
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/python-task/docker-compose.yaml
  18. 4
      dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/Constants.java
  19. 29
      dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java
  20. 2
      dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/WebDriverWaitFactory.java
  21. 2
      dolphinscheduler-e2e/lombok.config
  22. 8
      dolphinscheduler-e2e/pom.xml
  23. 2
      dolphinscheduler-standalone-server/src/main/resources/application.yaml
  24. 2
      dolphinscheduler-ui/package.json
  25. 2
      dolphinscheduler-ui/pnpm-lock.yaml

1
.github/actions/workflow-telemetry-action

@ -0,0 +1 @@
Subproject commit f974e0c5942f8f37973c4cc395704165fbe629ba

13
.github/workflows/e2e.yml

@ -29,6 +29,8 @@ concurrency:
group: e2e-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
pull-requests: write
jobs:
paths-filter:
@ -124,12 +126,23 @@ jobs:
class: org.apache.dolphinscheduler.e2e.cases.PostgresDataSourceE2ETest
- name: ShellTaskE2ETest
class: org.apache.dolphinscheduler.e2e.cases.tasks.ShellTaskE2ETest
- name: PythonTaskE2ETest
class: org.apache.dolphinscheduler.e2e.cases.tasks.PythonTaskE2ETest
env:
RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Set up JDK 11
uses: actions/setup-java@v4
with:
java-version: 11
distribution: 'adopt'
- name: Collect Workflow Telemetry
uses: ./.github/actions/workflow-telemetry-action
with:
comment_on_pr: false
- name: Cache local Maven repository
uses: actions/cache@v3
with:

3
.gitmodules vendored

@ -25,3 +25,6 @@
path = .github/actions/auto-assign-action
url = https://github.com/kentaro-m/auto-assign-action.git
branch = 288f36f
[submodule ".github/actions/workflow-telemetry-action"]
path = .github/actions/workflow-telemetry-action
url = https://github.com/catchpoint/workflow-telemetry-action

2
dolphinscheduler-dist/release-docs/LICENSE vendored

@ -746,7 +746,7 @@ MIT licenses
axios 0.27.2: https://github.com/axios/axios MIT
date-fns 2.29.3: https://github.com/date-fns/date-fns MIT
lodash 4.17.21: https://github.com/lodash/lodash MIT
monaco-editor 0.34.0: https://github.com/microsoft/monaco-editor MIT
monaco-editor 0.50.0: https://github.com/microsoft/monaco-editor MIT
naive-ui 2.30.7: https://github.com/TuSimple/naive-ui MIT
nprogress 0.2.0: https://github.com/rstacruz/nprogress MIT
pinia 2.0.22: https://github.com/vuejs/pinia MIT

338
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/tasks/PythonTaskE2ETest.java

@ -0,0 +1,338 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.e2e.cases.tasks;
import static org.assertj.core.api.Assertions.assertThat;
import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest;
import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
import org.apache.dolphinscheduler.e2e.core.WebDriverHolder;
import org.apache.dolphinscheduler.e2e.models.environment.PythonEnvironment;
import org.apache.dolphinscheduler.e2e.pages.LoginPage;
import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.PythonTaskForm;
import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage;
import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage;
import org.apache.dolphinscheduler.e2e.pages.security.EnvironmentPage;
import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage;
import org.apache.dolphinscheduler.e2e.pages.security.TenantPage;
import org.apache.dolphinscheduler.e2e.pages.security.UserPage;
import java.util.Date;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
@DolphinScheduler(composeFiles = "docker/python-task/docker-compose.yaml")
public class PythonTaskE2ETest extends BaseWorkflowE2ETest {
private static final PythonEnvironment pythonEnvironment = new PythonEnvironment();
@BeforeAll
public static void setup() {
browser = WebDriverHolder.getWebDriver();
TenantPage tenantPage = new LoginPage(browser)
.login(adminUser)
.goToNav(SecurityPage.class)
.goToTab(TenantPage.class);
if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) {
tenantPage
.create(adminUser.getTenant())
.goToNav(SecurityPage.class)
.goToTab(UserPage.class)
.update(adminUser);
}
tenantPage
.goToNav(SecurityPage.class)
.goToTab(EnvironmentPage.class)
.createEnvironmentUntilSuccess(pythonEnvironment.getEnvironmentName(),
pythonEnvironment.getEnvironmentConfig(),
pythonEnvironment.getEnvironmentDesc(),
pythonEnvironment.getEnvironmentWorkerGroup());
tenantPage
.goToNav(ProjectPage.class)
.createProjectUntilSuccess(projectName);
}
@Test
@Order(10)
void testRunPythonTasks_SuccessCase() {
WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
.goTo(projectName)
.goToTab(WorkflowDefinitionTab.class);
// todo: use yaml to define the workflow
String workflowName = "PythonSuccessCase";
String taskName = "PythonSuccessTask";
String pythonScripts = "print(\"success\")";
workflowDefinitionPage
.createWorkflow()
.<PythonTaskForm>addTask(WorkflowForm.TaskType.PYTHON)
.script(pythonScripts)
.name(taskName)
.submit()
.submit()
.name(workflowName)
.submit();
untilWorkflowDefinitionExist(workflowName);
workflowDefinitionPage.publish(workflowName);
runWorkflow(workflowName);
untilWorkflowInstanceExist(workflowName);
WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName);
assertThat(workflowInstance.executionTime()).isEqualTo(1);
TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName);
assertThat(taskInstance.retryTimes()).isEqualTo(0);
}
@Test
@Order(20)
void testRunPythonTasks_WorkflowParamsCase() {
WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
.goTo(projectName)
.goToTab(WorkflowDefinitionTab.class);
// todo: use yaml to define the workflow
String workflowName = "PythonWorkflowParamsCase";
String taskName = "PythonWorkflowParamsTask";
String pythonScripts = "import sys\n"
+ "\n"
+ "if '${name}' == 'tom':\n"
+ " print('success')\n"
+ "else:\n"
+ " sys.exit(2)";
workflowDefinitionPage
.createWorkflow()
.<PythonTaskForm>addTask(WorkflowForm.TaskType.PYTHON)
.script(pythonScripts)
.name(taskName)
.submit()
.submit()
.name(workflowName)
.addGlobalParam("name", "tom")
.submit();
untilWorkflowDefinitionExist(workflowName);
workflowDefinitionPage.publish(workflowName);
runWorkflow(workflowName);
untilWorkflowInstanceExist(workflowName);
WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName);
assertThat(workflowInstance.executionTime()).isEqualTo(1);
TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName);
assertThat(taskInstance.retryTimes()).isEqualTo(0);
}
@Test
@Order(30)
void testRunPythonTasks_LocalParamsCase() {
WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
.goTo(projectName)
.goToTab(WorkflowDefinitionTab.class);
String workflowName = "PythonLocalParamsCase";
String taskName = "PythonLocalParamsSuccess";
String pythonScripts = "import sys\n"
+ "\n"
+ "if '${name}' == 'tom':\n"
+ " print('success')\n"
+ "else:\n"
+ " sys.exit(2)";
workflowDefinitionPage
.createWorkflow()
.<PythonTaskForm>addTask(WorkflowForm.TaskType.PYTHON)
.script(pythonScripts)
.name(taskName)
.addParam("name", "tom")
.submit()
.submit()
.name(workflowName)
.submit();
untilWorkflowDefinitionExist(workflowName);
workflowDefinitionPage.publish(workflowName);
runWorkflow(workflowName);
untilWorkflowInstanceExist(workflowName);
WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName);
assertThat(workflowInstance.executionTime()).isEqualTo(1);
TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName);
assertThat(taskInstance.retryTimes()).isEqualTo(0);
}
@Test
@Order(40)
void testRunPythonTasks_GlobalParamsOverrideLocalParamsCase() {
WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
.goTo(projectName)
.goToTab(WorkflowDefinitionTab.class);
String workflowName = "PythonLocalParamsOverrideWorkflowParamsCase";
String taskName = "PythonLocalParamsOverrideWorkflowParamsSuccess";
String pythonScripts = "import sys\n"
+ "\n"
+ "if '${name}' == 'jerry':\n"
+ " print('success')\n"
+ "else:\n"
+ " sys.exit(2)";
workflowDefinitionPage
.createWorkflow()
.<PythonTaskForm>addTask(WorkflowForm.TaskType.PYTHON)
.script(pythonScripts)
.name(taskName)
.addParam("name", "tom")
.submit()
.submit()
.name(workflowName)
.addGlobalParam("name", "jerry")
.submit();
untilWorkflowDefinitionExist(workflowName);
workflowDefinitionPage.publish(workflowName);
runWorkflow(workflowName);
untilWorkflowInstanceExist(workflowName);
WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName);
assertThat(workflowInstance.executionTime()).isEqualTo(1);
TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName);
assertThat(taskInstance.retryTimes()).isEqualTo(0);
}
@Test
@Order(50)
void testRunPythonTasks_UsingResourceFile() {
long current_timestamp = new Date().getTime();
String testFileName = String.format("echo_%s", current_timestamp);
new ResourcePage(browser)
.goToNav(ResourcePage.class)
.goToTab(FileManagePage.class)
.createFileUntilSuccess(testFileName, "echo 123");
final WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
.goTo(projectName)
.goToTab(WorkflowDefinitionTab.class);
String workflowName = "PythonUsingResourceFileWorkflowCase";
String taskName = "PythonUsingResourceFileSuccessTask";
String pythonScripts = "import sys\n"
+ "\n"
+ "file_content = \"\"\n"
+ "\n"
+ "with open('${file_name}', 'r', encoding='UTF8') as f:\n"
+ " file_content = f.read()\n"
+ "\n"
+ "if len(file_content) != 0:\n"
+ " print(f'file_content: {file_content}')\n"
+ "else:\n"
+ " sys.exit(2)\n"
+ " ";
workflowDefinitionPage
.createWorkflow()
.<PythonTaskForm>addTask(WorkflowForm.TaskType.PYTHON)
.script(pythonScripts)
.name(taskName)
.selectResource(testFileName)
.addParam("file_name", String.format("%s.sh", testFileName))
.submit()
.submit()
.name(workflowName)
.submit();
untilWorkflowDefinitionExist(workflowName);
workflowDefinitionPage.publish(workflowName);
runWorkflow(workflowName);
untilWorkflowInstanceExist(workflowName);
WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName);
assertThat(workflowInstance.executionTime()).isEqualTo(1);
TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName);
assertThat(taskInstance.retryTimes()).isEqualTo(0);
}
@Test
@Order(60)
void testRunPythonTasks_FailedCase() {
WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
.goTo(projectName)
.goToTab(WorkflowDefinitionTab.class);
String workflowName = "PythonFailedWorkflowCase";
String taskName = "PythonFailedTask";
String pythonScripts = "import sys\n"
+ "sys.exit(1)";
workflowDefinitionPage
.createWorkflow()
.<PythonTaskForm>addTask(WorkflowForm.TaskType.PYTHON)
.script(pythonScripts)
.name(taskName)
.submit()
.submit()
.name(workflowName)
.submit();
untilWorkflowDefinitionExist(workflowName);
workflowDefinitionPage.publish(workflowName);
runWorkflow(workflowName);
untilWorkflowInstanceExist(workflowName);
WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceFailed(workflowName);
assertThat(workflowInstance.executionTime()).isEqualTo(1);
TaskInstanceTab.Row taskInstance = untilTaskInstanceFailed(workflowName, taskName);
assertThat(taskInstance.retryTimes()).isEqualTo(0);
}
}

28
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/tasks/ShellTaskE2ETest.java

@ -21,6 +21,8 @@ import static org.assertj.core.api.Assertions.assertThat;
import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest;
import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
import org.apache.dolphinscheduler.e2e.core.WebDriverHolder;
import org.apache.dolphinscheduler.e2e.pages.LoginPage;
import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab;
@ -29,7 +31,11 @@ import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTa
import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm;
import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage;
import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage;
import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage;
import org.apache.dolphinscheduler.e2e.pages.security.TenantPage;
import org.apache.dolphinscheduler.e2e.pages.security.UserPage;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
@ -38,6 +44,28 @@ import org.junit.jupiter.api.TestMethodOrder;
@DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml")
public class ShellTaskE2ETest extends BaseWorkflowE2ETest {
@BeforeAll
public static void setup() {
browser = WebDriverHolder.getWebDriver();
TenantPage tenantPage = new LoginPage(browser)
.login(adminUser)
.goToNav(SecurityPage.class)
.goToTab(TenantPage.class);
if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) {
tenantPage
.create(adminUser.getTenant())
.goToNav(SecurityPage.class)
.goToTab(UserPage.class)
.update(adminUser);
}
tenantPage
.goToNav(ProjectPage.class)
.createProjectUntilSuccess(projectName);
}
@Test
void testRunShellTasks_SuccessCase() {
WorkflowDefinitionTab workflowDefinitionPage =

27
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/workflow/BaseWorkflowE2ETest.java

@ -20,17 +20,12 @@ package org.apache.dolphinscheduler.e2e.cases.workflow;
import static org.assertj.core.api.Assertions.assertThat;
import static org.testcontainers.shaded.org.awaitility.Awaitility.await;
import org.apache.dolphinscheduler.e2e.core.WebDriverHolder;
import org.apache.dolphinscheduler.e2e.models.users.AdminUser;
import org.apache.dolphinscheduler.e2e.pages.LoginPage;
import org.apache.dolphinscheduler.e2e.pages.project.ProjectDetailPage;
import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab;
import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage;
import org.apache.dolphinscheduler.e2e.pages.security.TenantPage;
import org.apache.dolphinscheduler.e2e.pages.security.UserPage;
import java.util.List;
import java.util.Objects;
@ -39,7 +34,6 @@ import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeAll;
import org.openqa.selenium.remote.RemoteWebDriver;
@Slf4j
@ -51,27 +45,6 @@ public abstract class BaseWorkflowE2ETest {
protected static RemoteWebDriver browser;
@BeforeAll
public static void setup() {
browser = WebDriverHolder.getWebDriver();
TenantPage tenantPage = new LoginPage(browser)
.login(adminUser)
.goToNav(SecurityPage.class)
.goToTab(TenantPage.class);
if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) {
tenantPage
.create(adminUser.getTenant())
.goToNav(SecurityPage.class)
.goToTab(UserPage.class)
.update(adminUser);
}
tenantPage
.goToNav(ProjectPage.class)
.createProjectUntilSuccess(projectName);
}
protected void untilWorkflowDefinitionExist(String workflowName) {
WorkflowDefinitionTab workflowDefinitionPage = new ProjectPage(browser)
.goToNav(ProjectPage.class)

30
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/models/environment/IEnvironment.java

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.e2e.models.environment;
public interface IEnvironment {
String getEnvironmentName();
String getEnvironmentConfig();
String getEnvironmentDesc();
String getEnvironmentWorkerGroup();
}

52
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/models/environment/PythonEnvironment.java

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.e2e.models.environment;
import lombok.Data;
@Data
public class PythonEnvironment implements IEnvironment {
private String environmentName;
private String environmentConfig;
private String environmentDesc;
private String environmentWorkerGroup;
@Override
public String getEnvironmentName() {
return "python-e2e";
}
@Override
public String getEnvironmentConfig() {
return "export PYTHON_LAUNCHER=/usr/bin/python3";
}
@Override
public String getEnvironmentDesc() {
return "pythonEnvDesc";
}
@Override
public String getEnvironmentWorkerGroup() {
return "default";
}
}

80
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/common/CodeEditor.java

@ -19,10 +19,18 @@
*/
package org.apache.dolphinscheduler.e2e.pages.common;
import org.apache.dolphinscheduler.e2e.core.Constants;
import org.apache.dolphinscheduler.e2e.core.WebDriverWaitFactory;
import java.util.List;
import lombok.Getter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.junit.platform.commons.util.StringUtils;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.Keys;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.interactions.Actions;
@ -32,13 +40,17 @@ import org.openqa.selenium.support.PageFactory;
import org.openqa.selenium.support.ui.ExpectedConditions;
@Getter
@Slf4j
public final class CodeEditor {
@FindBys({
@FindBy(className = "monaco-editor"),
@FindBy(className = "view-line"),
})
private WebElement editor;
private List<WebElement> editor;
@FindBy(className = "pre-tasks-model")
private WebElement scrollBar;
private WebDriver driver;
@ -47,14 +59,72 @@ public final class CodeEditor {
this.driver = driver;
}
@SneakyThrows
public CodeEditor content(String content) {
WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.elementToBeClickable(editor));
editor.click();
WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.elementToBeClickable(editor.get(0)));
Actions actions = new Actions(this.driver);
actions.moveToElement(editor).sendKeys(content).perform();
List<String> contentList = List.of(content.split(Constants.LINE_SEPARATOR));
try {
((JavascriptExecutor) driver).executeScript("arguments[0].scrollIntoView();", scrollBar);
} catch (org.openqa.selenium.NoSuchElementException ignored) {
log.warn("scroll bar not found, skipping...");
}
for (int i = 0; i < contentList.size(); i++) {
String editorLineText;
String inputContent = contentList.get(i);
if (i == 0) {
actions.moveToElement(editor.get(i))
.click()
.sendKeys(inputContent)
.sendKeys(Constants.LINE_SEPARATOR)
.perform();
continue;
} else {
editorLineText = editor.get(i).getText();
}
if (StringUtils.isNotBlank(inputContent)) {
if (editorLineText.isEmpty()) {
actions.moveToElement(editor.get(i))
.click()
.sendKeys(inputContent)
.sendKeys(Constants.LINE_SEPARATOR)
.perform();
Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS);
} else {
for (int p = 0; p < editorLineText.strip().length(); p++) {
clearLine(actions, editor.get(i));
}
if (!editorLineText.isEmpty()) {
clearLine(actions, editor.get(i));
}
actions.moveToElement(editor.get(i))
.click()
.sendKeys(inputContent)
.sendKeys(Constants.LINE_SEPARATOR)
.perform();
Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS);
}
} else {
actions.moveToElement(editor.get(i))
.click()
.sendKeys(Constants.LINE_SEPARATOR)
.perform();
Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS);
}
}
return this;
}
private void clearLine(Actions actions, WebElement element) {
actions.moveToElement(element)
.click()
.sendKeys(Keys.BACK_SPACE)
.perform();
}
}

6
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowForm.java

@ -22,6 +22,7 @@ package org.apache.dolphinscheduler.e2e.pages.project.workflow;
import org.apache.dolphinscheduler.e2e.core.WebDriverWaitFactory;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.HttpTaskForm;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.JavaTaskForm;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.PythonTaskForm;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.SubWorkflowTaskForm;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.SwitchTaskForm;
@ -75,6 +76,8 @@ public final class WorkflowForm {
final String dragAndDrop = String.join("\n",
Resources.readLines(Resources.getResource("dragAndDrop.js"), StandardCharsets.UTF_8));
js.executeScript(dragAndDrop, task, canvas);
WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions
.visibilityOfElementLocated(By.xpath("//*[contains(text(), 'Current node settings')]")));
switch (type) {
case SHELL:
@ -87,6 +90,8 @@ public final class WorkflowForm {
return (T) new HttpTaskForm(this);
case JAVA:
return (T) new JavaTaskForm(this);
case PYTHON:
return (T) new PythonTaskForm(this);
}
throw new UnsupportedOperationException("Unknown task type");
}
@ -126,5 +131,6 @@ public final class WorkflowForm {
SWITCH,
HTTP,
JAVA,
PYTHON
}
}

7
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowRunDialog.java

@ -26,6 +26,7 @@ import lombok.Getter;
import org.openqa.selenium.By;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.FindBy;
import org.openqa.selenium.support.FindBys;
import org.openqa.selenium.support.PageFactory;
import org.openqa.selenium.support.ui.ExpectedConditions;
@ -34,7 +35,10 @@ public final class WorkflowRunDialog {
private final WorkflowDefinitionTab parent;
@FindBys({
@FindBy(xpath = "//div[contains(text(), 'Please set the parameters before starting')]/../.."),
@FindBy(className = "btn-submit")
})
private WebElement buttonSubmit;
public WorkflowRunDialog(WorkflowDefinitionTab parent) {
@ -52,7 +56,8 @@ public final class WorkflowRunDialog {
.until(ExpectedConditions.elementToBeClickable(buttonSubmit()));
buttonSubmit().click();
WebDriverWaitFactory.createWebDriverWait(parent.driver())
.until(ExpectedConditions.invisibilityOfElementLocated(runDialogTitleXpath));
return parent();
}
}

49
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/PythonTaskForm.java

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package org.apache.dolphinscheduler.e2e.pages.project.workflow.task;
import org.apache.dolphinscheduler.e2e.pages.common.CodeEditor;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm;
import lombok.Getter;
import org.openqa.selenium.WebDriver;
@Getter
public final class PythonTaskForm extends TaskNodeForm {
private CodeEditor codeEditor;
private WebDriver driver;
public PythonTaskForm(WorkflowForm parent) {
super(parent);
this.codeEditor = new CodeEditor(parent.driver());
this.driver = parent.driver();
}
public PythonTaskForm script(String script) {
codeEditor.content(script);
return this;
}
}

3
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/TaskNodeForm.java

@ -28,6 +28,7 @@ import lombok.Getter;
import org.openqa.selenium.By;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.Keys;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.FindBy;
@ -178,6 +179,8 @@ public abstract class TaskNodeForm {
.findFirst()
.orElseThrow(() -> new RuntimeException("No such resource: " + resourceName))
.click();
parent.driver().switchTo().activeElement().sendKeys(Keys.ESCAPE);
return this;
}

11
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/security/EnvironmentPage.java

@ -19,6 +19,9 @@
package org.apache.dolphinscheduler.e2e.pages.security;
import static org.assertj.core.api.Assertions.assertThat;
import static org.testcontainers.shaded.org.awaitility.Awaitility.await;
import org.apache.dolphinscheduler.e2e.core.WebDriverWaitFactory;
import org.apache.dolphinscheduler.e2e.pages.common.NavBarPage;
@ -82,6 +85,14 @@ public final class EnvironmentPage extends NavBarPage implements SecurityPage.Ta
return this;
}
public EnvironmentPage createEnvironmentUntilSuccess(String name, String config, String desc, String workerGroup) {
create(name, config, desc, workerGroup);
await().untilAsserted(() -> assertThat(environmentList())
.as("environment list should contain newly-created environment")
.anyMatch(it -> it.getText().contains(name)));
return this;
}
public EnvironmentPage update(String oldName, String name, String config, String desc, String workerGroup) {
environmentList()
.stream()

25
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/python-task/Dockerfile

@ -0,0 +1,25 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM apache/dolphinscheduler-standalone-server:ci
RUN apt update \
&& apt install -y software-properties-common \
&& add-apt-repository ppa:deadsnakes/ppa \
&& apt update \
&& apt-get install -y python3.8 libpython3.8-dev python3.8-dev python3.8-distutils \
&& rm -rf /var/lib/apt/lists/*

40
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/python-task/docker-compose.yaml

@ -0,0 +1,40 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: "3.8"
services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci-python
build:
context: .
dockerfile: ./Dockerfile
environment:
MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true'
ports:
- "12345:12345"
networks:
- e2e
healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ]
interval: 5s
timeout: 5s
retries: 120
networks:
e2e:

4
dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/Constants.java

@ -39,4 +39,8 @@ public final class Constants {
* chrome download path in selenium/standalone-chrome-debug container
*/
public static final String SELENIUM_CONTAINER_CHROME_DOWNLOAD_PATH = "/home/seluser/Downloads";
public static final String LINE_SEPARATOR = "\n";
public static final long DEFAULT_SLEEP_MILLISECONDS = 500;
}

29
dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java

@ -47,6 +47,7 @@ import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.testcontainers.Testcontainers;
import org.testcontainers.containers.BindMode;
import org.testcontainers.containers.BrowserWebDriverContainer;
import org.testcontainers.containers.ComposeContainer;
import org.testcontainers.containers.wait.strategy.Wait;
@ -98,12 +99,15 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
browser.withAccessToHost(true);
}
browser.start();
driver = new RemoteWebDriver(browser.getSeleniumAddress(), new ChromeOptions());
ChromeOptions chromeOptions = new ChromeOptions();
chromeOptions.addArguments("--allow-running-insecure-content");
chromeOptions.addArguments(String.format("--unsafely-treat-insecure-origin-as-secure=http://%s:%s",
address.getHost(), address.getPort()));
driver = new RemoteWebDriver(browser.getSeleniumAddress(), chromeOptions);
driver.manage().timeouts()
.implicitlyWait(Duration.ofSeconds(10))
.pageLoadTimeout(Duration.ofSeconds(10));
.implicitlyWait(Duration.ofSeconds(1))
.pageLoadTimeout(Duration.ofSeconds(5));
driver.manage().window()
.maximize();
@ -141,11 +145,20 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
imageName = DockerImageName.parse("seleniarm/standalone-chromium:124.0-chromedriver-124.0")
.asCompatibleSubstituteFor("selenium/standalone-chrome");
if (!Files.exists(Constants.HOST_CHROME_DOWNLOAD_PATH)) {
try {
Files.createDirectories(Constants.HOST_CHROME_DOWNLOAD_PATH);
} catch (IOException e) {
log.error("Failed to create chrome download directory: {}", Constants.HOST_CHROME_DOWNLOAD_PATH);
throw new RuntimeException(e);
}
}
browser = new BrowserWebDriverContainer<>(imageName)
.withCapabilities(new ChromeOptions())
.withCreateContainerCmdModifier(cmd -> cmd.withUser("root"))
.withFileSystemBind(Constants.HOST_CHROME_DOWNLOAD_PATH.toFile().getAbsolutePath(),
Constants.SELENIUM_CONTAINER_CHROME_DOWNLOAD_PATH)
Constants.SELENIUM_CONTAINER_CHROME_DOWNLOAD_PATH, BindMode.READ_WRITE)
.withRecordingMode(RECORD_ALL, record.toFile(), MP4)
.withStartupTimeout(Duration.ofSeconds(300));
} else {
@ -153,7 +166,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
.withCapabilities(new ChromeOptions())
.withCreateContainerCmdModifier(cmd -> cmd.withUser("root"))
.withFileSystemBind(Constants.HOST_CHROME_DOWNLOAD_PATH.toFile().getAbsolutePath(),
Constants.SELENIUM_CONTAINER_CHROME_DOWNLOAD_PATH)
Constants.SELENIUM_CONTAINER_CHROME_DOWNLOAD_PATH, BindMode.READ_WRITE)
.withRecordingMode(RECORD_ALL, record.toFile(), MP4)
.withStartupTimeout(Duration.ofSeconds(300));
}
@ -194,7 +207,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
field.setAccessible(true);
field.set(object, driver);
} catch (IllegalAccessException e) {
LOGGER.error("Failed to inject web driver to field: {}", field.getName(), e);
log.error("Failed to inject web driver to field: {}", field.getName(), e);
}
}
@ -215,7 +228,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
.withExposedService(
serviceName,
DOCKER_PORT, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(300)))
.withLogConsumer(serviceName, outputFrame -> LOGGER.info(outputFrame.getUtf8String()))
.withLogConsumer(serviceName, outputFrame -> log.info(outputFrame.getUtf8String()))
.waitingFor(serviceName, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
return compose;

2
dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/WebDriverWaitFactory.java

@ -26,7 +26,7 @@ public class WebDriverWaitFactory {
private static final Duration DEFAULT_INTERVAL = Duration.ofMillis(500);
private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(60);
private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(10);
/**
* Create a WebDriverWait instance with default timeout 60s and interval 100ms.

2
dolphinscheduler-e2e/lombok.config

@ -16,5 +16,5 @@
#
lombok.accessors.fluent=true
lombok.log.fieldname=LOGGER
lombok.log.fieldname=log
lombok.accessors.fluent=true

8
dolphinscheduler-e2e/pom.xml

@ -36,13 +36,15 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<junit.version>5.8.1</junit.version>
<selenium.version>4.13.0</selenium.version>
<selenium.version>4.21.0</selenium.version>
<lombok.version>1.18.20</lombok.version>
<assertj-core.version>3.20.2</assertj-core.version>
<kotlin.version>1.5.30</kotlin.version>
<slf4j-api.version>1.7.36</slf4j-api.version>
<log4j-slf4j-impl.version>2.17.2</log4j-slf4j-impl.version>
<guava.version>31.0.1-jre</guava.version>
<maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version>
<testcontainers.version>1.19.8</testcontainers.version>
</properties>
<dependencies>
@ -119,7 +121,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-bom</artifactId>
<version>1.19.8</version>
<version>${testcontainers.version}</version>
<scope>import</scope>
<type>pom</type>
</dependency>
@ -131,7 +133,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
<version>${maven-surefire-plugin.version}</version>
</plugin>
</plugins>
</build>

2
dolphinscheduler-standalone-server/src/main/resources/application.yaml

@ -179,7 +179,7 @@ master:
server-load-protection:
enabled: true
# Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow.
max-system-cpu-usage-percentage-thresholds: 0.9
max-system-cpu-usage-percentage-thresholds: 1
# Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow.
max-jvm-cpu-usage-percentage-thresholds: 0.9
# Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.

2
dolphinscheduler-ui/package.json

@ -19,7 +19,7 @@
"echarts": "^5.3.3",
"js-cookie": "^3.0.1",
"lodash": "^4.17.21",
"monaco-editor": "^0.34.0",
"monaco-editor": "^0.50.0",
"naive-ui": "2.33.5",
"nprogress": "^0.2.0",
"pinia": "^2.0.22",

2
dolphinscheduler-ui/pnpm-lock.yaml

@ -42,7 +42,7 @@ specifiers:
eslint-plugin-vue: ^9.5.1
js-cookie: ^3.0.1
lodash: ^4.17.21
monaco-editor: ^0.34.0
monaco-editor: ^0.50.0
naive-ui: 2.33.5
nprogress: ^0.2.0
pinia: ^2.0.22

Loading…
Cancel
Save