[E2E][CI] Add DataSource E2E test (#8085)

.github/workflows/e2e.yml

@@ -85,6 +85,14 @@ jobs:
             class: org.apache.dolphinscheduler.e2e.cases.WorkflowE2ETest
           - name: FileManage
             class: org.apache.dolphinscheduler.e2e.cases.FileManageE2ETest
+          - name: MysqlDataSource
+            class: org.apache.dolphinscheduler.e2e.cases.MysqlDataSourceE2ETest
+          - name: ClickhouseDataSource
+            class: org.apache.dolphinscheduler.e2e.cases.ClickhouseDataSourceE2ETest
+          - name: PostgresDataSource
+            class: org.apache.dolphinscheduler.e2e.cases.PostgresDataSourceE2ETest
+          - name: SqlServerDataSource
+            class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest
     env:
       RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
     steps:

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ClickhouseDataSourceE2ETest.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.dolphinscheduler.e2e.cases;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
+import org.apache.dolphinscheduler.e2e.pages.LoginPage;
+import org.apache.dolphinscheduler.e2e.pages.datasource.DataSourcePage;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.remote.RemoteWebDriver;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.openqa.selenium.support.ui.WebDriverWait;
+
+
+@DolphinScheduler(composeFiles = "docker/datasource-clickhouse/docker-compose.yaml")
+public class ClickhouseDataSourceE2ETest {
+    private static RemoteWebDriver browser;
+
+    private static final String tenant = System.getProperty("user.name");
+
+    private static final String user = "admin";
+
+    private static final String password = "dolphinscheduler123";
+
+    private static final String dataSourceType = "CLICKHOUSE";
+
+    private static final String dataSourceName = "clickhouse_test";
+
+    private static final String dataSourceDescription = "clickhouse_test";
+
+    private static final String ip = "clickhouse";
+
+    private static final String port = "8123";
+
+    private static final String userName = "ch_test";
+
+    private static final String pgPassword = "ch_test";
+
+    private static final String database = "ch_test";
+
+    private static final String jdbcParams = "";
+
+
+    @BeforeAll
+    public static void setup() {
+        new LoginPage(browser)
+            .login(user, password)
+            .goToNav(DataSourcePage.class);
+    }
+
+    @Test
+    @Order(10)
+    void testCreateClickhouseDataSource() {
+        final DataSourcePage page = new DataSourcePage(browser);
+
+        page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, pgPassword, database, jdbcParams);
+
+        new WebDriverWait(page.driver(), 10).until(ExpectedConditions.invisibilityOfElementLocated(new By.ById("dialogCreateDataSource")));
+
+        await().untilAsserted(() -> assertThat(page.dataSourceItemsList())
+            .as("DataSource list should contain newly-created database")
+            .extracting(WebElement::getText)
+            .anyMatch(it -> it.contains(dataSourceName)));
+    }
+
+    @Test
+    @Order(20)
+    void testDeleteClickhouseDataSource() {
+        final DataSourcePage page = new DataSourcePage(browser);
+
+        page.delete(dataSourceName);
+
+        await().untilAsserted(() -> {
+            browser.navigate().refresh();
+
+            assertThat(
+                page.dataSourceItemsList()
+            ).noneMatch(
+                it -> it.getText().contains(dataSourceName)
+            );
+        });
+    }
+}

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/MysqlDataSourceE2ETest.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.dolphinscheduler.e2e.cases;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
+import org.apache.dolphinscheduler.e2e.pages.LoginPage;
+import org.apache.dolphinscheduler.e2e.pages.datasource.DataSourcePage;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.remote.RemoteWebDriver;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.openqa.selenium.support.ui.WebDriverWait;
+
+
+@DolphinScheduler(composeFiles = "docker/datasource-mysql/docker-compose.yaml")
+public class MysqlDataSourceE2ETest {
+    private static RemoteWebDriver browser;
+
+    private static final String tenant = System.getProperty("user.name");
+
+    private static final String user = "admin";
+
+    private static final String password = "dolphinscheduler123";
+
+    private static final String dataSourceType = "MYSQL";
+
+    private static final String dataSourceName = "mysql_test";
+
+    private static final String dataSourceDescription = "mysql_test";
+
+    private static final String ip = "mysql";
+
+    private static final String port = "3306";
+
+    private static final String userName = "root";
+
+    private static final String mysqlPassword = "123456";
+
+    private static final String database = "mysql";
+
+    private static final String jdbcParams = "{\"useSSL\": false}";
+
+
+    @BeforeAll
+    public static void setup() {
+        new LoginPage(browser)
+            .login(user, password)
+            .goToNav(DataSourcePage.class);
+    }
+
+    @Test
+    @Order(10)
+    void testCreateMysqlDataSource() {
+        final DataSourcePage page = new DataSourcePage(browser);
+
+        page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, mysqlPassword, database, jdbcParams);
+
+        new WebDriverWait(page.driver(), 10).until(ExpectedConditions.invisibilityOfElementLocated(new By.ById("dialogCreateDataSource")));
+
+        await().untilAsserted(() -> assertThat(page.dataSourceItemsList())
+            .as("DataSource list should contain newly-created database")
+            .extracting(WebElement::getText)
+            .anyMatch(it -> it.contains(dataSourceName)));
+    }
+
+    @Test
+    @Order(20)
+    void testDeleteMysqlDataSource() {
+        final DataSourcePage page = new DataSourcePage(browser);
+
+        page.delete(dataSourceName);
+
+        await().untilAsserted(() -> {
+            browser.navigate().refresh();
+
+            assertThat(
+                    page.dataSourceItemsList()
+            ).noneMatch(
+                    it -> it.getText().contains(dataSourceName)
+            );
+        });
+    }
+}

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/PostgresDataSourceE2ETest.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.dolphinscheduler.e2e.cases;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
+import org.apache.dolphinscheduler.e2e.pages.LoginPage;
+import org.apache.dolphinscheduler.e2e.pages.datasource.DataSourcePage;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.remote.RemoteWebDriver;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.openqa.selenium.support.ui.WebDriverWait;
+
+
+@DolphinScheduler(composeFiles = "docker/datasource-postgresql/docker-compose.yaml")
+public class PostgresDataSourceE2ETest {
+    private static RemoteWebDriver browser;
+
+    private static final String tenant = System.getProperty("user.name");
+
+    private static final String user = "admin";
+
+    private static final String password = "dolphinscheduler123";
+
+    private static final String dataSourceType = "POSTGRESQL";
+
+    private static final String dataSourceName = "postgres_test";
+
+    private static final String dataSourceDescription = "postgres_test";
+
+    private static final String ip = "postgres";
+
+    private static final String port = "5432";
+
+    private static final String userName = "postgres";
+
+    private static final String pgPassword = "postgres";
+
+    private static final String database = "postgres";
+
+    private static final String jdbcParams = "";
+
+
+    @BeforeAll
+    public static void setup() {
+        new LoginPage(browser)
+            .login(user, password)
+            .goToNav(DataSourcePage.class);
+    }
+
+    @Test
+    @Order(10)
+    void testCreatePostgresDataSource() {
+        final DataSourcePage page = new DataSourcePage(browser);
+
+        page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, pgPassword, database, jdbcParams);
+
+        new WebDriverWait(page.driver(), 10).until(ExpectedConditions.invisibilityOfElementLocated(new By.ById("dialogCreateDataSource")));
+
+        await().untilAsserted(() -> assertThat(page.dataSourceItemsList())
+            .as("DataSource list should contain newly-created database")
+            .extracting(WebElement::getText)
+            .anyMatch(it -> it.contains(dataSourceName)));
+    }
+
+    @Test
+    @Order(20)
+    void testDeletePostgresDataSource() {
+        final DataSourcePage page = new DataSourcePage(browser);
+
+        page.delete(dataSourceName);
+
+        await().untilAsserted(() -> {
+            browser.navigate().refresh();
+
+            assertThat(
+                    page.dataSourceItemsList()
+            ).noneMatch(
+                    it -> it.getText().contains(dataSourceName)
+            );
+        });
+    }
+}

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/SqlServerDataSourceE2ETest.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.dolphinscheduler.e2e.cases;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
+import org.apache.dolphinscheduler.e2e.pages.LoginPage;
+import org.apache.dolphinscheduler.e2e.pages.datasource.DataSourcePage;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.remote.RemoteWebDriver;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.openqa.selenium.support.ui.WebDriverWait;
+
+
+@DolphinScheduler(composeFiles = "docker/datasource-sqlserver/docker-compose.yaml")
+public class SqlServerDataSourceE2ETest {
+    private static RemoteWebDriver browser;
+
+    private static final String tenant = System.getProperty("user.name");
+
+    private static final String user = "admin";
+
+    private static final String password = "dolphinscheduler123";
+
+    private static final String dataSourceType = "SQLSERVER";
+
+    private static final String dataSourceName = "sqlserver_test";
+
+    private static final String dataSourceDescription = "sqlserver_test";
+
+    private static final String ip = "sqlserver";
+
+    private static final String port = "1433";
+
+    private static final String userName = "sa";
+
+    private static final String pgPassword = "OcP2020123";
+
+    private static final String database = "master";
+
+    private static final String jdbcParams = "";
+
+
+    @BeforeAll
+    public static void setup() {
+        new LoginPage(browser)
+            .login(user, password)
+            .goToNav(DataSourcePage.class);
+    }
+
+    @Test
+    @Order(10)
+    void testCreateSqlServerDataSource() {
+        final DataSourcePage page = new DataSourcePage(browser);
+
+        page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, pgPassword, database, jdbcParams);
+
+        new WebDriverWait(page.driver(), 10).until(ExpectedConditions.invisibilityOfElementLocated(new By.ById("dialogCreateDataSource")));
+
+        await().untilAsserted(() -> assertThat(page.dataSourceItemsList())
+            .as("DataSource list should contain newly-created database")
+            .extracting(WebElement::getText)
+            .anyMatch(it -> it.contains(dataSourceName)));
+    }
+
+    @Test
+    @Order(20)
+    void testDeleteSqlServerDataSource() {
+        final DataSourcePage page = new DataSourcePage(browser);
+
+        page.delete(dataSourceName);
+
+        await().untilAsserted(() -> {
+            browser.navigate().refresh();
+
+            assertThat(
+                page.dataSourceItemsList()
+            ).noneMatch(
+                it -> it.getText().contains(dataSourceName)
+            );
+        });
+    }
+}

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/common/NavBarPage.java

@@ -19,6 +19,7 @@
  */
 package org.apache.dolphinscheduler.e2e.pages.common;
 
+import org.apache.dolphinscheduler.e2e.pages.datasource.DataSourcePage;
 import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage;
 import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage;
 import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage;
@@ -46,6 +47,9 @@ public class NavBarPage {
     @FindBy(id = "tabResource")
     private WebElement resourceTab;
 
+    @FindBy(id = "tabDataSource")
+    private WebElement dataSourceTab;
+
     public NavBarPage(RemoteWebDriver driver) {
         this.driver = driver;
 
@@ -74,6 +78,13 @@ public class NavBarPage {
             return nav.cast(new ResourcePage(driver));
         }
 
+        if (nav == DataSourcePage.class) {
+            WebElement dataSourceTabElement = new WebDriverWait(driver, 60)
+                .until(ExpectedConditions.elementToBeClickable(dataSourceTab));
+            ((JavascriptExecutor)driver).executeScript("arguments[0].click();", dataSourceTabElement);
+            return nav.cast(new DataSourcePage(driver));
+        }
+
         throw new UnsupportedOperationException("Unknown nav bar");
     }
 

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java

@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.dolphinscheduler.e2e.pages.datasource;
+
+import lombok.Getter;
+
+import org.apache.dolphinscheduler.e2e.pages.common.NavBarPage;
+
+import java.util.List;
+
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.remote.RemoteWebDriver;
+import org.openqa.selenium.support.FindBy;
+import org.openqa.selenium.support.FindBys;
+import org.openqa.selenium.support.PageFactory;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.openqa.selenium.support.ui.Select;
+import org.openqa.selenium.support.ui.WebDriverWait;
+
+
+@Getter
+public class DataSourcePage extends NavBarPage implements NavBarPage.NavBarItem {
+
+    @FindBy(id = "btnCreateDataSource")
+    private WebElement buttonCreateDataSource;
+
+    @FindBy(className = "data-source-items")
+    private List<WebElement> dataSourceItemsList;
+
+    @FindBys({
+        @FindBy(className = "el-popconfirm"),
+        @FindBy(className = "el-button--primary"),
+    })
+    private List<WebElement> buttonConfirm;
+
+    private final CreateDataSourceForm createDataSourceForm;
+
+    public DataSourcePage(RemoteWebDriver driver) {
+        super(driver);
+
+        createDataSourceForm = new CreateDataSourceForm();
+    }
+
+    public DataSourcePage createDataSource(String dataSourceType, String dataSourceName, String dataSourceDescription, String ip, String port, String userName, String password, String database,
+                                           String jdbcParams) {
+        buttonCreateDataSource().click();
+
+        createDataSourceForm().btnDataSourceTypeDropdown().click();
+
+        new WebDriverWait(driver, 10).until(ExpectedConditions.visibilityOfElementLocated(new By.ById("dialogCreateDataSource")));
+
+        createDataSourceForm().selectDataSourceType()
+            .stream()
+            .filter(it -> it.getText().contains(dataSourceType.toUpperCase()))
+            .findFirst()
+            .orElseThrow(() -> new RuntimeException(String.format("No %s in data source type list", dataSourceType.toUpperCase())))
+            .click();
+
+        createDataSourceForm().inputDataSourceName().sendKeys(dataSourceName);
+        createDataSourceForm().inputDataSourceDescription().sendKeys(dataSourceDescription);
+        createDataSourceForm().inputIP().sendKeys(ip);
+        createDataSourceForm().inputPort().clear();
+        createDataSourceForm().inputPort().sendKeys(port);
+        createDataSourceForm().inputUserName().sendKeys(userName);
+        createDataSourceForm().inputPassword().sendKeys(password);
+        createDataSourceForm().inputDataBase().sendKeys(database);
+
+        if (!"".equals(jdbcParams)) {
+            createDataSourceForm().inputJdbcParams().sendKeys(jdbcParams);
+        }
+
+        createDataSourceForm().buttonSubmit().click();
+
+        return this;
+    }
+
+    public DataSourcePage delete(String name) {
+        dataSourceItemsList()
+            .stream()
+            .filter(it -> it.getText().contains(name))
+            .flatMap(it -> it.findElements(By.id("btnDelete")).stream())
+            .filter(WebElement::isDisplayed)
+            .findFirst()
+            .orElseThrow(() -> new RuntimeException("No delete button in data source list"))
+            .click();
+
+        buttonConfirm()
+            .stream()
+            .filter(WebElement::isDisplayed)
+            .findFirst()
+            .orElseThrow(() -> new RuntimeException("No confirm button when deleting"))
+            .click();
+
+        return this;
+    }
+
+    @Getter
+    public class CreateDataSourceForm {
+        CreateDataSourceForm() {
+            PageFactory.initElements(driver, this);
+        }
+
+        @FindBy(className = "options-datasource-type")
+        private List<WebElement> selectDataSourceType;
+
+        @FindBy(id = "btnDataSourceTypeDropDown")
+        private WebElement btnDataSourceTypeDropdown;
+
+        @FindBy(id = "inputDataSourceName")
+        private WebElement inputDataSourceName;
+
+        @FindBy(id = "inputDataSourceDescription")
+        private WebElement inputDataSourceDescription;
+
+        @FindBy(id = "inputIP")
+        private WebElement inputIP;
+
+        @FindBy(id = "inputPort")
+        private WebElement inputPort;
+
+        @FindBy(id = "inputUserName")
+        private WebElement inputUserName;
+
+        @FindBy(id = "inputPassword")
+        private WebElement inputPassword;
+
+        @FindBy(id = "inputDataBase")
+        private WebElement inputDataBase;
+
+        @FindBy(id = "inputJdbcParams")
+        private WebElement inputJdbcParams;
+
+        @FindBy(id = "btnSubmit")
+        private WebElement buttonSubmit;
+
+        @FindBy(id = "btnCancel")
+        private WebElement buttonCancel;
+
+        @FindBy(id = "btnTestConnection")
+        private WebElement btnTestConnection;
+    }
+}

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/security/SecurityPage.java

@@ -63,6 +63,7 @@ public class SecurityPage extends NavBarPage implements NavBarItem {
             WebElement menUserManageElement = new WebDriverWait(driver, 60)
                     .until(ExpectedConditions.elementToBeClickable(menUserManage));
             ((JavascriptExecutor)driver).executeScript("arguments[0].click();", menUserManageElement);
+            new WebDriverWait(driver, 25).until(ExpectedConditions.urlContains("/#/security/users"));
             return tab.cast(new UserPage(driver));
         }
         if (tab == WorkerGroupPage.class) {

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml

@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "2.1"
+
+services:
+  dolphinscheduler:
+    image: apache/dolphinscheduler-standalone-server:ci
+    environment:
+      MASTER_MAX_CPU_LOAD_AVG: 100
+      WORKER_TENANT_AUTO_CREATE: 'true'
+    expose:
+      - 12345
+    networks:
+      - e2e
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+  clickhouse:
+    image: yandex/clickhouse-server:21.12.3.32
+    environment:
+      - CLICKHOUSE_DB=ch_test
+      - CLICKHOUSE_USER=ch_test
+      - CLICKHOUSE_PASSWORD=ch_test
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    expose:
+      - 8123
+    mem_limit: 3G
+    mem_reservation: 1G
+    networks:
+      - e2e
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
+networks:
+  e2e:

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml

@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "2.1"
+
+services:
+  dolphinscheduler:
+    image: apache/dolphinscheduler-standalone-server:ci
+    environment:
+      MASTER_MAX_CPU_LOAD_AVG: 100
+      WORKER_TENANT_AUTO_CREATE: 'true'
+    expose:
+      - 12345
+    networks:
+      - e2e
+    volumes:
+      - ./download-mysql.sh:/tmp/download-mysql.sh
+    entrypoint: ['bash', '-c', '/bin/bash /tmp/download-mysql.sh && /opt/dolphinscheduler/bin/start.sh && tail -f /dev/null']
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    depends_on:
+      mysql:
+        condition: service_healthy
+  mysql:
+    image: mysql:5.7.36
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: 123456
+    networks:
+      - e2e
+    expose:
+      - 3306
+    healthcheck:
+      test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
+networks:
+  e2e:

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/download-mysql.sh

@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+DS_HOME=/opt/dolphinscheduler/libs
+MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
+MYSQL_DRIVER="mysql-connector-java-8.0.16.jar"
+
+if ! curl -Lo "${DS_HOME}/${MYSQL_DRIVER}" ${MYSQL_URL}; then
+    echo "Fail to download ${MYSQL_DRIVER}."
+    exit 1
+fi

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml

@@ -0,0 +1,54 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "2.1"
+
+services:
+  dolphinscheduler:
+    image: apache/dolphinscheduler-standalone-server:ci
+    environment:
+      MASTER_MAX_CPU_LOAD_AVG: 100
+      WORKER_TENANT_AUTO_CREATE: 'true'
+    expose:
+      - 12345
+    networks:
+      - e2e
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    depends_on:
+      postgres:
+        condition: service_healthy
+  postgres:
+    image: postgres:14.1
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: postgres
+    networks:
+      - e2e
+    expose:
+      - 5432
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
+networks:
+  e2e:

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml

@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "2.1"
+
+services:
+  dolphinscheduler:
+    image: apache/dolphinscheduler-standalone-server:ci
+    environment:
+      MASTER_MAX_CPU_LOAD_AVG: 100
+      WORKER_TENANT_AUTO_CREATE: 'true'
+    expose:
+      - 12345
+    networks:
+      - e2e
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    depends_on:
+      sqlserver:
+        condition: service_healthy
+  sqlserver:
+    image: alexk002/sqlserver2019_demo:1
+    expose:
+      - 1433
+    networks:
+      - e2e
+    healthcheck:
+      test: /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P OcP2020123 -Q "SELECT 1"
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
+networks:
+  e2e:

dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue

@@ -21,8 +21,13 @@
         <m-list-box-f>
           <template slot="name"><strong>*</strong>{{$t('Datasource')}}</template>
           <template slot="content" size="small">
-              <el-select style="width: 100%;" v-model="type" :disabled="this.item.id">
-                <el-option v-for="item in datasourceTypeList" :key="item.value" :value="item.value" :label="item.label">
+              <el-select id="btnDataSourceTypeDropDown" style="width: 100%;" v-model="type" :disabled="this.item.id">
+                <el-option
+                      class="options-datasource-type"
+                      v-for="item in datasourceTypeList"
+                      :key="item.value"
+                      :value="item.value"
+                      :label="item.label">
                 </el-option>
               </el-select>
           </template>
@@ -31,6 +36,7 @@
           <template slot="name"><strong>*</strong>{{$t('Datasource Name')}}</template>
           <template slot="content">
             <el-input
+                    id="inputDataSourceName"
                     type="input"
                     v-model="name"
                     maxlength="60"
@@ -43,6 +49,7 @@
           <template slot="name">{{$t('Description')}}</template>
           <template slot="content">
             <el-input
+                    id="inputDataSourceDescription"
                     type="textarea"
                     v-model="note"
                     size="small"
@@ -54,6 +61,7 @@
           <template slot="name"><strong>*</strong>{{$t('IP')}}</template>
           <template slot="content">
             <el-input
+                    id="inputIP"
                     type="input"
                     v-model="host"
                     maxlength="255"
@@ -66,6 +74,7 @@
           <template slot="name"><strong>*</strong>{{$t('Port')}}</template>
           <template slot="content">
             <el-input
+                    id="inputPort"
                     type="input"
                     v-model="port"
                     size="small"
@@ -121,6 +130,7 @@
           <template slot="name"><strong>*</strong>{{$t('User Name')}}</template>
           <template slot="content">
             <el-input
+                    id="inputUserName"
                     type="input"
                     v-model="userName"
                     maxlength="60"
@@ -133,6 +143,7 @@
           <template slot="name">{{$t('Password')}}</template>
           <template slot="content">
             <el-input
+                    id="inputPassword"
                     type="password"
                     v-model="password"
                     size="small"
@@ -144,6 +155,7 @@
           <template slot="name"><strong :class="{hidden:showDatabase}">*</strong>{{$t('Database Name')}}</template>
           <template slot="content">
             <el-input
+                    id="inputDataBase"
                     type="input"
                     v-model="database"
                     maxlength="60"
@@ -165,6 +177,7 @@
           <template slot="name">{{$t('jdbc connect parameters')}}</template>
           <template slot="content">
             <el-input
+                    id="inputJdbcParams"
                     type="textarea"
                     v-model="other"
                     :autosize="{minRows:2}"
@@ -176,9 +189,9 @@
       </div>
     </div>
     <div class="bottom-p">
-      <el-button type="text" ize="mini" @click="_close()"> {{$t('Cancel')}} </el-button>
-      <el-button type="success" size="mini" round @click="_testConnect()" :loading="testLoading">{{testLoading ? $t('Loading...') : $t('Test Connect')}}</el-button>
-      <el-button type="primary" size="mini" round :loading="spinnerLoading" @click="_ok()">{{spinnerLoading ? $t('Loading...') :item ? `${$t('Edit')}` : `${$t('Submit')}`}} </el-button>
+      <el-button id="btnCancel" type="text" ize="mini" @click="_close()"> {{$t('Cancel')}} </el-button>
+      <el-button id="btnTestConnection" type="success" size="mini" round @click="_testConnect()" :loading="testLoading">{{testLoading ? $t('Loading...') : $t('Test Connect')}}</el-button>
+      <el-button id="btnSubmit" type="primary" size="mini" round :loading="spinnerLoading" @click="_ok()">{{spinnerLoading ? $t('Loading...') :item ? `${$t('Edit')}` : `${$t('Submit')}`}} </el-button>
     </div>
   </div>
 </template>

dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue

@@ -17,7 +17,7 @@
 <template>
   <div class="list-model">
     <div class="table-box">
-      <el-table :data="list" size="mini" style="width: 100%">
+      <el-table :data="list" size="mini" style="width: 100%" row-class-name="data-source-items">
         <el-table-column type="index" :label="$t('#')" width="50"></el-table-column>
         <el-table-column prop="name" :label="$t('Datasource Name')"></el-table-column>
         <el-table-column prop="userName" :label="$t('Datasource userName')"></el-table-column>
@@ -62,7 +62,7 @@
                 :title="$t('Delete?')"
                 @onConfirm="_delete(scope.row,scope.row.id)"
               >
-                <el-button type="danger" size="mini" icon="el-icon-delete" circle slot="reference"></el-button>
+                <el-button id="btnDelete" type="danger" size="mini" icon="el-icon-delete" circle slot="reference"></el-button>
               </el-popconfirm>
             </el-tooltip>
           </template>

dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue

@@ -19,8 +19,9 @@
     <template slot="conditions">
       <m-conditions @on-conditions="_onConditions">
         <template slot="button-group">
-          <el-button size="mini" @click="_create('')">{{$t('Create Datasource')}}</el-button>
+          <el-button id="btnCreateDataSource" size="mini" @click="_create('')">{{$t('Create Datasource')}}</el-button>
           <el-dialog
+            id="dialogCreateDataSource"
             :title="item ?($t('Edit')+$t('Datasource')) : ($t('Create')+$t('Datasource'))"
             v-if="dialogVisible"
             :visible.sync="dialogVisible"

dolphinscheduler-ui/src/js/module/components/nav/nav.vue

@@ -43,7 +43,7 @@
       </div>
       <div class="clearfix list">
         <div class="nav-links">
-          <router-link :to="{ path: '/datasource'}" tag="a" active-class="active">
+          <router-link :to="{ path: '/datasource'}" tag="a" active-class="active" id="tabDataSource">
             <span><em class="ansfont ri-database-2-line"></em>{{$t('Datasource manage')}}</span><strong></strong>
           </router-link>
         </div>