diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js index 85e681b17f..d94250e9af 100644 --- a/docs/configs/docsdev.js +++ b/docs/configs/docsdev.js @@ -358,6 +358,10 @@ export default { { title: 'HANA', link: '/en-us/docs/dev/user_doc/guide/datasource/hana.html', + }, + { + title: 'doris', + link: '/en-us/docs/dev/user_doc/guide/datasource/doris.html', } ], }, @@ -1067,6 +1071,10 @@ export default { { title: 'HANA', link: '/zh-cn/docs/dev/user_doc/guide/datasource/hana.html', + }, + { + title: 'Doris', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/doris.html', } ], }, diff --git a/docs/docs/en/guide/datasource/doris.md b/docs/docs/en/guide/datasource/doris.md new file mode 100644 index 0000000000..b2d376a7d6 --- /dev/null +++ b/docs/docs/en/guide/datasource/doris.md @@ -0,0 +1,23 @@ +# DORIS + +![doris](../../../../img/new_ui/dev/datasource/doris.png) + +## Datasource Parameters + +| **Datasource** | **Description** | +|----------------------------|---------------------------------------------------------------------------------------| +| Datasource | Select DORIS. | +| Datasource name | Enter the name of the DataSource. | +| Description | Enter a description of the DataSource. | +| IP/Host Name | Enter the DORIS service IP.(If there are multiple IPs, please separate them with `,`) | +| Port | Enter the DORIS service port. | +| Username | Set the username for DORIS connection. | +| Password | Set the password for DORIS connection. | +| Database name | Enter the database name of the DORIS connection. | +| Jdbc connection parameters | Parameter settings for DORIS connection, in JSON format. | + +## Native Supported + +No, read section example in [datasource-setting](../howto/datasource-setting.md) `DataSource Center` section to activate +this datasource. + diff --git a/docs/docs/zh/guide/datasource/doris.md b/docs/docs/zh/guide/datasource/doris.md new file mode 100644 index 0000000000..9b63503530 --- /dev/null +++ b/docs/docs/zh/guide/datasource/doris.md @@ -0,0 +1,17 @@ +# DORIS 数据源 + +![DORIS](../../../../img/new_ui/dev/datasource/doris.png) + +- 数据源：选择 DORIS +- 数据源名称：输入数据源的名称 +- 描述：输入数据源的描述 +- IP 主机名：输入连接 DORIS 的 IP(如有多个IP，请用`,`分隔) +- 端口：输入连接 DORIS 的端口 +- 用户名：设置连接 DORIS 的用户名 +- 密码：设置连接 DORIS 的密码 +- 数据库名：输入连接 DORIS 的数据库名称 +- Jdbc 连接参数：用于 DORIS 连接的参数设置，以 JSON 形式填写 + +## 是否原生支持 + +否，使用前需请参考 [数据源配置](../howto/datasource-setting.md) 中的 "数据源中心" 章节激活数据源。 diff --git a/docs/img/new_ui/dev/datasource/doris.png b/docs/img/new_ui/dev/datasource/doris.png new file mode 100644 index 0000000000..29d1043fa5 Binary files /dev/null and b/docs/img/new_ui/dev/datasource/doris.png differ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java index 4c1c3e30ac..b525d23ba7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java @@ -71,6 +71,7 @@ public class DataSourceConstants { * jdbc url */ public static final String JDBC_MYSQL = "jdbc:mysql://"; + public static final String JDBC_MYSQL_LOADBALANCE = "jdbc:mysql:loadbalance://"; public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; public static final String JDBC_HIVE_2 = "jdbc:hive2://"; public static final String JDBC_KYUUBI = "jdbc:kyuubi://"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml index d8ce6ec46f..586948cab6 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml @@ -132,5 +132,11 @@ dolphinscheduler-datasource-vertica ${project.version} + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-doris + ${project.version} + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml new file mode 100644 index 0000000000..a0d31b23e8 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml @@ -0,0 +1,46 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin + dev-SNAPSHOT + + + dolphinscheduler-datasource-doris + jar + ${project.artifactId} + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + + + + mysql + mysql-connector-java + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannel.java new file mode 100644 index 0000000000..33e0a9c501 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.doris; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class DorisDataSourceChannel implements DataSourceChannel { + + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + return null; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java new file mode 100644 index 0000000000..5fe7bda4f0 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.doris; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +public class DorisDataSourceChannelFactory implements DataSourceChannelFactory { + + @Override + public DataSourceChannel create() { + return new DorisDataSourceChannel(); + } + + @Override + public String getName() { + return "doris"; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceClient.java new file mode 100644 index 0000000000..ae84cbbb8a --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceClient.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.doris; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class DorisDataSourceClient extends CommonDataSourceClient { + + public DorisDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + super(baseConnectionParam, dbType); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisConnectionParam.java new file mode 100644 index 0000000000..c6257038e7 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisConnectionParam.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.doris.param; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +public class DorisConnectionParam extends BaseConnectionParam { + + @Override + public String toString() { + return "DorisConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceParamDTO.java new file mode 100644 index 0000000000..44b4b1e3c4 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceParamDTO.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.doris.param; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class DorisDataSourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "DorisDataSourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + @Override + public DbType getType() { + return DbType.DORIS; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java new file mode 100644 index 0000000000..99f115c2c8 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.doris.param; + +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.constants.DataSourceConstants; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.google.auto.service.AutoService; + +@AutoService(DataSourceProcessor.class) +public class DorisDataSourceProcessor extends AbstractDataSourceProcessor { + + @Override + public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { + return JSONUtils.parseObject(paramJson, DorisDataSourceParamDTO.class); + } + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) throws NumberFormatException { + DorisConnectionParam connectionParams = (DorisConnectionParam) createConnectionParams(connectionJson); + DorisDataSourceParamDTO dorisDataSourceParamDTO = new DorisDataSourceParamDTO(); + + dorisDataSourceParamDTO.setUserName(connectionParams.getUser()); + dorisDataSourceParamDTO.setDatabase(connectionParams.getDatabase()); + dorisDataSourceParamDTO.setOther(connectionParams.getOther()); + + String address = connectionParams.getAddress(); + String[] hostSeperator = address.split(Constants.DOUBLE_SLASH); + String[] hostPortArrays = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); + + dorisDataSourceParamDTO.setPort(Integer.parseInt(hostPortArrays[0].split(Constants.COLON)[1])); + + for (int i = 0; i < hostPortArrays.length; i++) { + hostPortArrays[i] = hostPortArrays[i].split(Constants.COLON)[0]; + } + dorisDataSourceParamDTO.setHost(String.join(",", hostPortArrays)); + + return dorisDataSourceParamDTO; + } + + @Override + public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + DorisDataSourceParamDTO mysqlDatasourceParam = (DorisDataSourceParamDTO) datasourceParam; + String[] hosts = datasourceParam.getHost().split(Constants.COMMA); + + for (int i = 0; i < hosts.length; i++) { + hosts[i] = String.format(Constants.FORMAT_S_S_COLON, hosts[i], mysqlDatasourceParam.getPort()); + } + + String address = String.format("%s%s", DataSourceConstants.JDBC_MYSQL_LOADBALANCE, String.join(",", hosts)); + String jdbcUrl = String.format(Constants.FORMAT_S_S, address, mysqlDatasourceParam.getDatabase()); + + DorisConnectionParam mysqlConnectionParam = new DorisConnectionParam(); + mysqlConnectionParam.setJdbcUrl(jdbcUrl); + mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase()); + mysqlConnectionParam.setAddress(address); + mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName()); + mysqlConnectionParam.setPassword(PasswordUtils.encodePassword(mysqlDatasourceParam.getPassword())); + mysqlConnectionParam.setDriverClassName(getDatasourceDriver()); + mysqlConnectionParam.setValidationQuery(getValidationQuery()); + mysqlConnectionParam.setOther(mysqlDatasourceParam.getOther()); + + return mysqlConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, DorisConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return DataSourceConstants.COM_MYSQL_CJ_JDBC_DRIVER; + } + + @Override + public String getValidationQuery() { + return DataSourceConstants.MYSQL_VALIDATION_QUERY; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + DorisConnectionParam mysqlConnectionParam = (DorisConnectionParam) connectionParam; + String jdbcUrl = mysqlConnectionParam.getJdbcUrl(); + if (MapUtils.isNotEmpty(mysqlConnectionParam.getOther())) { + return String.format("%s?%s", jdbcUrl, transformOther(mysqlConnectionParam.getOther())); + } + return String.format("%s", jdbcUrl); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + DorisConnectionParam dorisConnectionParam = (DorisConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + + return DriverManager.getConnection(getJdbcUrl(connectionParam), dorisConnectionParam.getUser(), + PasswordUtils.decodePassword(dorisConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.DORIS; + } + + @Override + public DataSourceProcessor create() { + return new DorisDataSourceProcessor(); + } + + private String transformOther(Map paramMap) { + if (MapUtils.isEmpty(paramMap)) { + return null; + } + Map otherMap = new HashMap<>(); + paramMap.forEach((k, v) -> otherMap.put(k, v)); + if (MapUtils.isEmpty(otherMap)) { + return null; + } + List otherList = new ArrayList<>(); + otherMap.forEach((key, value) -> otherList.add(String.format("%s=%s", key, value))); + return String.join("&", otherList); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactoryTest.java new file mode 100644 index 0000000000..66f3753f2e --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactoryTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.doris; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class DorisDataSourceChannelFactoryTest { + + @Test + public void testCreate() { + DorisDataSourceChannelFactory sourceChannelFactory = new DorisDataSourceChannelFactory(); + DataSourceChannel dataSourceChannel = sourceChannelFactory.create(); + Assertions.assertNotNull(dataSourceChannel); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelTest.java new file mode 100644 index 0000000000..114dd7aa9d --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.doris; + +import org.apache.dolphinscheduler.plugin.doris.param.DorisConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class DorisDataSourceChannelTest { + + @Test + public void testCreateDataSourceClient() { + DorisDataSourceChannel sourceChannel = Mockito.mock(DorisDataSourceChannel.class); + DorisDataSourceClient dataSourceClient = Mockito.mock(DorisDataSourceClient.class); + Mockito.when(sourceChannel.createDataSourceClient(Mockito.any(), Mockito.any())).thenReturn(dataSourceClient); + Assertions.assertNotNull(sourceChannel.createDataSourceClient(new DorisConnectionParam(), DbType.DORIS)); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessorTest.java new file mode 100644 index 0000000000..8939070c5a --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessorTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.doris.param; + +import org.apache.dolphinscheduler.common.constants.DataSourceConstants; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class DorisDataSourceProcessorTest { + + private DorisDataSourceProcessor dorisDatasourceProcessor = new DorisDataSourceProcessor(); + + @Test + public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); + DorisDataSourceParamDTO dorisDatasourceParamDTO = new DorisDataSourceParamDTO(); + + dorisDatasourceParamDTO.setUserName("root"); + dorisDatasourceParamDTO.setPassword("123456"); + dorisDatasourceParamDTO.setHost("localhost"); + dorisDatasourceParamDTO.setPort(3306); + dorisDatasourceParamDTO.setDatabase("default"); + dorisDatasourceParamDTO.setOther(props); + try (MockedStatic mockedPasswordUtils = Mockito.mockStatic(PasswordUtils.class)) { + Mockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); + DorisConnectionParam connectionParams = (DorisConnectionParam) dorisDatasourceProcessor + .createConnectionParams(dorisDatasourceParamDTO); + Assertions.assertEquals("jdbc:mysql:loadbalance://localhost:3306", connectionParams.getAddress()); + Assertions.assertEquals("jdbc:mysql:loadbalance://localhost:3306/default", connectionParams.getJdbcUrl()); + } + + dorisDatasourceParamDTO.setUserName("root"); + dorisDatasourceParamDTO.setPassword("123456"); + dorisDatasourceParamDTO.setHost("localhost,localhost1"); + dorisDatasourceParamDTO.setPort(3306); + dorisDatasourceParamDTO.setDatabase("default"); + dorisDatasourceParamDTO.setOther(props); + try (MockedStatic mockedPasswordUtils = Mockito.mockStatic(PasswordUtils.class)) { + Mockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); + DorisConnectionParam connectionParams = (DorisConnectionParam) dorisDatasourceProcessor + .createConnectionParams(dorisDatasourceParamDTO); + Assertions.assertEquals("jdbc:mysql:loadbalance://localhost:3306,localhost1:3306", + connectionParams.getAddress()); + Assertions.assertEquals("jdbc:mysql:loadbalance://localhost:3306,localhost1:3306/default", + connectionParams.getJdbcUrl()); + } + } + + @Test + public void testCreateConnectionParams2() { + String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://localhost:3306\"" + + ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:mysql://localhost:3306/default\"}"; + DorisConnectionParam connectionParams = (DorisConnectionParam) dorisDatasourceProcessor + .createConnectionParams(connectionJson); + Assertions.assertNotNull(connectionJson); + Assertions.assertEquals("root", connectionParams.getUser()); + } + + @Test + public void testGetDatasourceDriver() { + Assertions.assertEquals(DataSourceConstants.COM_MYSQL_CJ_JDBC_DRIVER, + dorisDatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + DorisConnectionParam dorisConnectionParam = new DorisConnectionParam(); + dorisConnectionParam.setJdbcUrl( + "jdbc:mysql://localhost:3306/default?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"); + Assertions.assertEquals( + "jdbc:mysql://localhost:3306/default?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false", + dorisDatasourceProcessor.getJdbcUrl(dorisConnectionParam)); + } + + @Test + public void testGetDbType() { + Assertions.assertEquals(DbType.DORIS, dorisDatasourceProcessor.getDbType()); + } + + @Test + public void testGetValidationQuery() { + Assertions.assertEquals(DataSourceConstants.MYSQL_VALIDATION_QUERY, + dorisDatasourceProcessor.getValidationQuery()); + } + + @Test + public void testGetDatasourceUniqueId() { + DorisConnectionParam dorisConnectionParam = new DorisConnectionParam(); + dorisConnectionParam.setJdbcUrl("jdbc:mysql://localhost:3306/default"); + dorisConnectionParam.setUser("root"); + dorisConnectionParam.setPassword("123456"); + try (MockedStatic mockedPasswordUtils = Mockito.mockStatic(PasswordUtils.class)) { + Mockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("123456"); + Assertions.assertEquals("doris@root@123456@jdbc:mysql://localhost:3306/default", + dorisDatasourceProcessor.getDatasourceUniqueId(dorisConnectionParam, DbType.DORIS)); + } + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/provider/JDBCDataSourceProviderTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/provider/JDBCDataSourceProviderTest.java new file mode 100644 index 0000000000..d12420a071 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/provider/JDBCDataSourceProviderTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.doris.provider; + +import org.apache.dolphinscheduler.plugin.datasource.api.provider.JDBCDataSourceProvider; +import org.apache.dolphinscheduler.plugin.doris.param.DorisConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.zaxxer.hikari.HikariDataSource; + +@ExtendWith(MockitoExtension.class) +public class JDBCDataSourceProviderTest { + + @Test + public void testCreateJdbcDataSource() { + try ( + MockedStatic mockedJDBCDataSourceProvider = + Mockito.mockStatic(JDBCDataSourceProvider.class)) { + HikariDataSource dataSource = Mockito.mock(HikariDataSource.class); + mockedJDBCDataSourceProvider + .when(() -> JDBCDataSourceProvider.createJdbcDataSource(Mockito.any(), Mockito.any())) + .thenReturn(dataSource); + Assertions.assertNotNull( + JDBCDataSourceProvider.createJdbcDataSource(new DorisConnectionParam(), DbType.DORIS)); + } + } + + @Test + public void testCreateOneSessionJdbcDataSource() { + try ( + MockedStatic mockedJDBCDataSourceProvider = + Mockito.mockStatic(JDBCDataSourceProvider.class)) { + HikariDataSource dataSource = Mockito.mock(HikariDataSource.class); + mockedJDBCDataSourceProvider + .when(() -> JDBCDataSourceProvider.createOneSessionJdbcDataSource(Mockito.any(), Mockito.any())) + .thenReturn(dataSource); + Assertions.assertNotNull( + JDBCDataSourceProvider.createOneSessionJdbcDataSource(new DorisConnectionParam(), DbType.DORIS)); + } + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/utils/DataSourceUtilsTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/utils/DataSourceUtilsTest.java new file mode 100644 index 0000000000..010ea2971f --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/utils/DataSourceUtilsTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.doris.utils; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.plugin.doris.param.DorisConnectionParam; +import org.apache.dolphinscheduler.plugin.doris.param.DorisDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import java.sql.Connection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class DataSourceUtilsTest { + + @Test + public void testCheckDatasourceParamOne() { + DorisDataSourceParamDTO dorisDatasourceParamDTO = new DorisDataSourceParamDTO(); + dorisDatasourceParamDTO.setHost("localhost,localhost1"); + dorisDatasourceParamDTO.setDatabase("default"); + Map other = new HashMap<>(); + other.put("serverTimezone", "Asia/Shanghai"); + other.put("queryTimeout", "-1"); + other.put("characterEncoding", "utf8"); + dorisDatasourceParamDTO.setOther(other); + DataSourceUtils.checkDatasourceParam(dorisDatasourceParamDTO); + Assertions.assertTrue(true); + Assertions.assertEquals("localhost,localhost1", dorisDatasourceParamDTO.getHost()); + } + + @Test + public void testBuildConnectionParamTwo() { + DorisDataSourceParamDTO dorisDatasourceParamDTO = new DorisDataSourceParamDTO(); + dorisDatasourceParamDTO.setHost("localhost"); + dorisDatasourceParamDTO.setDatabase("default"); + dorisDatasourceParamDTO.setUserName("root"); + dorisDatasourceParamDTO.setPort(3306); + dorisDatasourceParamDTO.setPassword("123456"); + + try ( + MockedStatic mockedStaticPasswordUtils = Mockito.mockStatic(PasswordUtils.class); + MockedStatic mockedStaticCommonUtils = Mockito.mockStatic(CommonUtils.class)) { + mockedStaticPasswordUtils.when(() -> PasswordUtils.encodePassword(Mockito.anyString())) + .thenReturn("123456"); + mockedStaticCommonUtils.when(CommonUtils::getKerberosStartupState).thenReturn(false); + ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(dorisDatasourceParamDTO); + Assertions.assertNotNull(connectionParam); + } + } + + @Test + public void testBuildConnectionParamsThree() { + DorisDataSourceParamDTO dorisDatasourceParamDTO = new DorisDataSourceParamDTO(); + dorisDatasourceParamDTO.setHost("localhost"); + dorisDatasourceParamDTO.setDatabase("default"); + dorisDatasourceParamDTO.setUserName("root"); + dorisDatasourceParamDTO.setPort(3306); + dorisDatasourceParamDTO.setPassword("123456"); + ConnectionParam connectionParam = + DataSourceUtils.buildConnectionParams(DbType.DORIS, JSONUtils.toJsonString(dorisDatasourceParamDTO)); + Assertions.assertNotNull(connectionParam); + } + + @Test + public void testGetConnection() throws ExecutionException { + try ( + MockedStatic mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class); + MockedStatic mockedStaticDataSourceClientProvider = + Mockito.mockStatic(DataSourceClientProvider.class)) { + mockedStaticPropertyUtils.when(() -> PropertyUtils.getLong("kerberos.expire.time", 24L)).thenReturn(24L); + DataSourceClientProvider clientProvider = Mockito.mock(DataSourceClientProvider.class); + mockedStaticDataSourceClientProvider.when(DataSourceClientProvider::getInstance).thenReturn(clientProvider); + + Connection connection = Mockito.mock(Connection.class); + Mockito.when(clientProvider.getConnection(Mockito.any(), Mockito.any())).thenReturn(connection); + + DorisConnectionParam connectionParam = new DorisConnectionParam(); + connectionParam.setUser("root"); + connectionParam.setPassword("123456"); + connection = DataSourceClientProvider.getInstance().getConnection(DbType.DORIS, connectionParam); + + Assertions.assertNotNull(connection); + } + } + + @Test + public void testGetJdbcUrl() { + DorisConnectionParam dorisConnectionParam = new DorisConnectionParam(); + dorisConnectionParam.setJdbcUrl("jdbc:mysql://localhost,localhost2:3308?allowLoadLocalInfile=false"); + String jdbcUrl = DataSourceUtils.getJdbcUrl(DbType.DORIS, dorisConnectionParam); + Assertions.assertEquals( + "jdbc:mysql://localhost,localhost2:3308?allowLoadLocalInfile=false", + jdbcUrl); + } + + @Test + public void testBuildDatasourceParamDTO() { + DorisConnectionParam connectionParam = new DorisConnectionParam(); + connectionParam.setJdbcUrl( + "jdbc:mysql://localhost,localhost2:3308?allowLoadLocalInfile=false"); + connectionParam.setAddress("jdbc:mysql://localhost:3308,localhost2:3308"); + connectionParam.setUser("root"); + connectionParam.setPassword("123456"); + + Assertions.assertNotNull( + DataSourceUtils.buildDatasourceParamDTO(DbType.DORIS, JSONUtils.toJsonString(connectionParam))); + + } + + @Test + public void testGetDatasourceProcessor() { + Assertions.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.DORIS)); + } + + @Test + public void testGetDatasourceProcessorError() { + Assertions.assertThrows(Exception.class, () -> { + DataSourceUtils.getDatasourceProcessor(null); + }); + } +} diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml index 4882382cd7..79261be779 100644 --- a/dolphinscheduler-datasource-plugin/pom.xml +++ b/dolphinscheduler-datasource-plugin/pom.xml @@ -51,6 +51,7 @@ dolphinscheduler-datasource-databend dolphinscheduler-datasource-snowflake dolphinscheduler-datasource-vertica + dolphinscheduler-datasource-doris diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index a937713f6b..169f0ad9bf 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -50,7 +50,8 @@ public enum DbType { DATABEND(19, "databend"), SNOWFLAKE(20, "snowflake"), VERTICA(21, "vertica"), - HANA(22, "hana"); + HANA(22, "hana"), + DORIS(23, "doris"); private static final Map DB_TYPE_MAP = Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); diff --git a/dolphinscheduler-ui/src/service/modules/data-source/types.ts b/dolphinscheduler-ui/src/service/modules/data-source/types.ts index a493f4119a..a01618e4b3 100644 --- a/dolphinscheduler-ui/src/service/modules/data-source/types.ts +++ b/dolphinscheduler-ui/src/service/modules/data-source/types.ts @@ -37,6 +37,7 @@ type IDataBase = | 'DATABEND' | 'SNOWFLAKE' | 'HANA' + | 'DORIS' type IDataBaseLabel = | 'MYSQL' diff --git a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts index 06d7ca3e0e..3f9229e162 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts +++ b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts @@ -401,6 +401,11 @@ export const datasourceType: IDataBaseOptionKeys = { value: 'HANA', label: 'HANA', defaultPort: 30015 + }, + DORIS: { + value: 'DORIS', + label: 'DORIS', + defaultPort: 9030 } } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts index 9d6bfba71d..25640583d7 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts @@ -132,6 +132,11 @@ export function useDatasource( id: 22, code: 'HANA', disabled: false + }, + { + id: 23, + code: 'DORIS', + disabled: false } ]