Browse Source

[Feature] doris add datasource (#14391)

* Added doris data source plug-in

* Update DbType.java

* XML configuration

* doc and ui

* Missing catch of NumberFormatException

* code format

---------

Co-authored-by: xiangzihao <460888207@qq.com>
3.2.1-prepare
Xinxing 1 year ago committed by GitHub
parent
commit
1d0d85b988
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 8
      docs/configs/docsdev.js
  2. 23
      docs/docs/en/guide/datasource/doris.md
  3. 17
      docs/docs/zh/guide/datasource/doris.md
  4. BIN
      docs/img/new_ui/dev/datasource/doris.png
  5. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java
  6. 6
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
  7. 46
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml
  8. 30
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannel.java
  9. 33
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java
  10. 28
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceClient.java
  11. 37
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisConnectionParam.java
  12. 41
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceParamDTO.java
  13. 155
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java
  14. 33
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactoryTest.java
  15. 39
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelTest.java
  16. 125
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessorTest.java
  17. 64
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/provider/JDBCDataSourceProviderTest.java
  18. 151
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/utils/DataSourceUtilsTest.java
  19. 1
      dolphinscheduler-datasource-plugin/pom.xml
  20. 3
      dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
  21. 1
      dolphinscheduler-ui/src/service/modules/data-source/types.ts
  22. 5
      dolphinscheduler-ui/src/views/datasource/list/use-form.ts
  23. 5
      dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts

8
docs/configs/docsdev.js

@ -358,6 +358,10 @@ export default {
{ {
title: 'HANA', title: 'HANA',
link: '/en-us/docs/dev/user_doc/guide/datasource/hana.html', link: '/en-us/docs/dev/user_doc/guide/datasource/hana.html',
},
{
title: 'doris',
link: '/en-us/docs/dev/user_doc/guide/datasource/doris.html',
} }
], ],
}, },
@ -1067,6 +1071,10 @@ export default {
{ {
title: 'HANA', title: 'HANA',
link: '/zh-cn/docs/dev/user_doc/guide/datasource/hana.html', link: '/zh-cn/docs/dev/user_doc/guide/datasource/hana.html',
},
{
title: 'Doris',
link: '/zh-cn/docs/dev/user_doc/guide/datasource/doris.html',
} }
], ],
}, },

23
docs/docs/en/guide/datasource/doris.md

@ -0,0 +1,23 @@
# DORIS
![doris](../../../../img/new_ui/dev/datasource/doris.png)
## Datasource Parameters
| **Datasource** | **Description** |
|----------------------------|---------------------------------------------------------------------------------------|
| Datasource | Select DORIS. |
| Datasource name | Enter the name of the DataSource. |
| Description | Enter a description of the DataSource. |
| IP/Host Name | Enter the DORIS service IP.(If there are multiple IPs, please separate them with `,`) |
| Port | Enter the DORIS service port. |
| Username | Set the username for DORIS connection. |
| Password | Set the password for DORIS connection. |
| Database name | Enter the database name of the DORIS connection. |
| Jdbc connection parameters | Parameter settings for DORIS connection, in JSON format. |
## Native Supported
No, read section example in [datasource-setting](../howto/datasource-setting.md) `DataSource Center` section to activate
this datasource.

17
docs/docs/zh/guide/datasource/doris.md

@ -0,0 +1,17 @@
# DORIS 数据源
![DORIS](../../../../img/new_ui/dev/datasource/doris.png)
- 数据源:选择 DORIS
- 数据源名称:输入数据源的名称
- 描述:输入数据源的描述
- IP 主机名:输入连接 DORIS 的 IP(如有多个IP,请用`,`分隔)
- 端口:输入连接 DORIS 的端口
- 用户名:设置连接 DORIS 的用户名
- 密码:设置连接 DORIS 的密码
- 数据库名:输入连接 DORIS 的数据库名称
- Jdbc 连接参数:用于 DORIS 连接的参数设置,以 JSON 形式填写
## 是否原生支持
否,使用前需请参考 [数据源配置](../howto/datasource-setting.md) 中的 "数据源中心" 章节激活数据源。

BIN
docs/img/new_ui/dev/datasource/doris.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 72 KiB

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java

@ -71,6 +71,7 @@ public class DataSourceConstants {
* jdbc url * jdbc url
*/ */
public static final String JDBC_MYSQL = "jdbc:mysql://"; public static final String JDBC_MYSQL = "jdbc:mysql://";
public static final String JDBC_MYSQL_LOADBALANCE = "jdbc:mysql:loadbalance://";
public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; public static final String JDBC_POSTGRESQL = "jdbc:postgresql://";
public static final String JDBC_HIVE_2 = "jdbc:hive2://"; public static final String JDBC_HIVE_2 = "jdbc:hive2://";
public static final String JDBC_KYUUBI = "jdbc:kyuubi://"; public static final String JDBC_KYUUBI = "jdbc:kyuubi://";

6
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml

@ -132,5 +132,11 @@
<artifactId>dolphinscheduler-datasource-vertica</artifactId> <artifactId>dolphinscheduler-datasource-vertica</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-doris</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies> </dependencies>
</project> </project>

46
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml

@ -0,0 +1,46 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-plugin</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-datasource-doris</artifactId>
<packaging>jar</packaging>
<name>${project.artifactId}</name>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-spi</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-api</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
</dependencies>
</project>

30
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannel.java

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
import org.apache.dolphinscheduler.spi.datasource.DataSourceClient;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class DorisDataSourceChannel implements DataSourceChannel {
@Override
public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
return null;
}
}

33
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactory.java

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
public class DorisDataSourceChannelFactory implements DataSourceChannelFactory {
@Override
public DataSourceChannel create() {
return new DorisDataSourceChannel();
}
@Override
public String getName() {
return "doris";
}
}

28
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceClient.java

@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris;
import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class DorisDataSourceClient extends CommonDataSourceClient {
public DorisDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
super(baseConnectionParam, dbType);
}
}

37
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisConnectionParam.java

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris.param;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
public class DorisConnectionParam extends BaseConnectionParam {
@Override
public String toString() {
return "DorisConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ '}';
}
}

41
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceParamDTO.java

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class DorisDataSourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "DorisDataSourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.DORIS;
}
}

155
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java

@ -0,0 +1,155 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris.param;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.google.auto.service.AutoService;
@AutoService(DataSourceProcessor.class)
public class DorisDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) {
return JSONUtils.parseObject(paramJson, DorisDataSourceParamDTO.class);
}
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) throws NumberFormatException {
DorisConnectionParam connectionParams = (DorisConnectionParam) createConnectionParams(connectionJson);
DorisDataSourceParamDTO dorisDataSourceParamDTO = new DorisDataSourceParamDTO();
dorisDataSourceParamDTO.setUserName(connectionParams.getUser());
dorisDataSourceParamDTO.setDatabase(connectionParams.getDatabase());
dorisDataSourceParamDTO.setOther(connectionParams.getOther());
String address = connectionParams.getAddress();
String[] hostSeperator = address.split(Constants.DOUBLE_SLASH);
String[] hostPortArrays = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA);
dorisDataSourceParamDTO.setPort(Integer.parseInt(hostPortArrays[0].split(Constants.COLON)[1]));
for (int i = 0; i < hostPortArrays.length; i++) {
hostPortArrays[i] = hostPortArrays[i].split(Constants.COLON)[0];
}
dorisDataSourceParamDTO.setHost(String.join(",", hostPortArrays));
return dorisDataSourceParamDTO;
}
@Override
public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
DorisDataSourceParamDTO mysqlDatasourceParam = (DorisDataSourceParamDTO) datasourceParam;
String[] hosts = datasourceParam.getHost().split(Constants.COMMA);
for (int i = 0; i < hosts.length; i++) {
hosts[i] = String.format(Constants.FORMAT_S_S_COLON, hosts[i], mysqlDatasourceParam.getPort());
}
String address = String.format("%s%s", DataSourceConstants.JDBC_MYSQL_LOADBALANCE, String.join(",", hosts));
String jdbcUrl = String.format(Constants.FORMAT_S_S, address, mysqlDatasourceParam.getDatabase());
DorisConnectionParam mysqlConnectionParam = new DorisConnectionParam();
mysqlConnectionParam.setJdbcUrl(jdbcUrl);
mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase());
mysqlConnectionParam.setAddress(address);
mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName());
mysqlConnectionParam.setPassword(PasswordUtils.encodePassword(mysqlDatasourceParam.getPassword()));
mysqlConnectionParam.setDriverClassName(getDatasourceDriver());
mysqlConnectionParam.setValidationQuery(getValidationQuery());
mysqlConnectionParam.setOther(mysqlDatasourceParam.getOther());
return mysqlConnectionParam;
}
@Override
public ConnectionParam createConnectionParams(String connectionJson) {
return JSONUtils.parseObject(connectionJson, DorisConnectionParam.class);
}
@Override
public String getDatasourceDriver() {
return DataSourceConstants.COM_MYSQL_CJ_JDBC_DRIVER;
}
@Override
public String getValidationQuery() {
return DataSourceConstants.MYSQL_VALIDATION_QUERY;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
DorisConnectionParam mysqlConnectionParam = (DorisConnectionParam) connectionParam;
String jdbcUrl = mysqlConnectionParam.getJdbcUrl();
if (MapUtils.isNotEmpty(mysqlConnectionParam.getOther())) {
return String.format("%s?%s", jdbcUrl, transformOther(mysqlConnectionParam.getOther()));
}
return String.format("%s", jdbcUrl);
}
@Override
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
DorisConnectionParam dorisConnectionParam = (DorisConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(connectionParam), dorisConnectionParam.getUser(),
PasswordUtils.decodePassword(dorisConnectionParam.getPassword()));
}
@Override
public DbType getDbType() {
return DbType.DORIS;
}
@Override
public DataSourceProcessor create() {
return new DorisDataSourceProcessor();
}
private String transformOther(Map<String, String> paramMap) {
if (MapUtils.isEmpty(paramMap)) {
return null;
}
Map<String, String> otherMap = new HashMap<>();
paramMap.forEach((k, v) -> otherMap.put(k, v));
if (MapUtils.isEmpty(otherMap)) {
return null;
}
List<String> otherList = new ArrayList<>();
otherMap.forEach((key, value) -> otherList.add(String.format("%s=%s", key, value)));
return String.join("&", otherList);
}
}

33
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelFactoryTest.java

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class DorisDataSourceChannelFactoryTest {
@Test
public void testCreate() {
DorisDataSourceChannelFactory sourceChannelFactory = new DorisDataSourceChannelFactory();
DataSourceChannel dataSourceChannel = sourceChannelFactory.create();
Assertions.assertNotNull(dataSourceChannel);
}
}

39
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/DorisDataSourceChannelTest.java

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris;
import org.apache.dolphinscheduler.plugin.doris.param.DorisConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
public class DorisDataSourceChannelTest {
@Test
public void testCreateDataSourceClient() {
DorisDataSourceChannel sourceChannel = Mockito.mock(DorisDataSourceChannel.class);
DorisDataSourceClient dataSourceClient = Mockito.mock(DorisDataSourceClient.class);
Mockito.when(sourceChannel.createDataSourceClient(Mockito.any(), Mockito.any())).thenReturn(dataSourceClient);
Assertions.assertNotNull(sourceChannel.createDataSourceClient(new DorisConnectionParam(), DbType.DORIS));
}
}

125
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessorTest.java

@ -0,0 +1,125 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris.param;
import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
public class DorisDataSourceProcessorTest {
private DorisDataSourceProcessor dorisDatasourceProcessor = new DorisDataSourceProcessor();
@Test
public void testCreateConnectionParams() {
Map<String, String> props = new HashMap<>();
props.put("serverTimezone", "utc");
DorisDataSourceParamDTO dorisDatasourceParamDTO = new DorisDataSourceParamDTO();
dorisDatasourceParamDTO.setUserName("root");
dorisDatasourceParamDTO.setPassword("123456");
dorisDatasourceParamDTO.setHost("localhost");
dorisDatasourceParamDTO.setPort(3306);
dorisDatasourceParamDTO.setDatabase("default");
dorisDatasourceParamDTO.setOther(props);
try (MockedStatic<PasswordUtils> mockedPasswordUtils = Mockito.mockStatic(PasswordUtils.class)) {
Mockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
DorisConnectionParam connectionParams = (DorisConnectionParam) dorisDatasourceProcessor
.createConnectionParams(dorisDatasourceParamDTO);
Assertions.assertEquals("jdbc:mysql:loadbalance://localhost:3306", connectionParams.getAddress());
Assertions.assertEquals("jdbc:mysql:loadbalance://localhost:3306/default", connectionParams.getJdbcUrl());
}
dorisDatasourceParamDTO.setUserName("root");
dorisDatasourceParamDTO.setPassword("123456");
dorisDatasourceParamDTO.setHost("localhost,localhost1");
dorisDatasourceParamDTO.setPort(3306);
dorisDatasourceParamDTO.setDatabase("default");
dorisDatasourceParamDTO.setOther(props);
try (MockedStatic<PasswordUtils> mockedPasswordUtils = Mockito.mockStatic(PasswordUtils.class)) {
Mockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
DorisConnectionParam connectionParams = (DorisConnectionParam) dorisDatasourceProcessor
.createConnectionParams(dorisDatasourceParamDTO);
Assertions.assertEquals("jdbc:mysql:loadbalance://localhost:3306,localhost1:3306",
connectionParams.getAddress());
Assertions.assertEquals("jdbc:mysql:loadbalance://localhost:3306,localhost1:3306/default",
connectionParams.getJdbcUrl());
}
}
@Test
public void testCreateConnectionParams2() {
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://localhost:3306\""
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:mysql://localhost:3306/default\"}";
DorisConnectionParam connectionParams = (DorisConnectionParam) dorisDatasourceProcessor
.createConnectionParams(connectionJson);
Assertions.assertNotNull(connectionJson);
Assertions.assertEquals("root", connectionParams.getUser());
}
@Test
public void testGetDatasourceDriver() {
Assertions.assertEquals(DataSourceConstants.COM_MYSQL_CJ_JDBC_DRIVER,
dorisDatasourceProcessor.getDatasourceDriver());
}
@Test
public void testGetJdbcUrl() {
DorisConnectionParam dorisConnectionParam = new DorisConnectionParam();
dorisConnectionParam.setJdbcUrl(
"jdbc:mysql://localhost:3306/default?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false");
Assertions.assertEquals(
"jdbc:mysql://localhost:3306/default?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false",
dorisDatasourceProcessor.getJdbcUrl(dorisConnectionParam));
}
@Test
public void testGetDbType() {
Assertions.assertEquals(DbType.DORIS, dorisDatasourceProcessor.getDbType());
}
@Test
public void testGetValidationQuery() {
Assertions.assertEquals(DataSourceConstants.MYSQL_VALIDATION_QUERY,
dorisDatasourceProcessor.getValidationQuery());
}
@Test
public void testGetDatasourceUniqueId() {
DorisConnectionParam dorisConnectionParam = new DorisConnectionParam();
dorisConnectionParam.setJdbcUrl("jdbc:mysql://localhost:3306/default");
dorisConnectionParam.setUser("root");
dorisConnectionParam.setPassword("123456");
try (MockedStatic<PasswordUtils> mockedPasswordUtils = Mockito.mockStatic(PasswordUtils.class)) {
Mockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("123456");
Assertions.assertEquals("doris@root@123456@jdbc:mysql://localhost:3306/default",
dorisDatasourceProcessor.getDatasourceUniqueId(dorisConnectionParam, DbType.DORIS));
}
}
}

64
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/provider/JDBCDataSourceProviderTest.java

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris.provider;
import org.apache.dolphinscheduler.plugin.datasource.api.provider.JDBCDataSourceProvider;
import org.apache.dolphinscheduler.plugin.doris.param.DorisConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import com.zaxxer.hikari.HikariDataSource;
@ExtendWith(MockitoExtension.class)
public class JDBCDataSourceProviderTest {
@Test
public void testCreateJdbcDataSource() {
try (
MockedStatic<JDBCDataSourceProvider> mockedJDBCDataSourceProvider =
Mockito.mockStatic(JDBCDataSourceProvider.class)) {
HikariDataSource dataSource = Mockito.mock(HikariDataSource.class);
mockedJDBCDataSourceProvider
.when(() -> JDBCDataSourceProvider.createJdbcDataSource(Mockito.any(), Mockito.any()))
.thenReturn(dataSource);
Assertions.assertNotNull(
JDBCDataSourceProvider.createJdbcDataSource(new DorisConnectionParam(), DbType.DORIS));
}
}
@Test
public void testCreateOneSessionJdbcDataSource() {
try (
MockedStatic<JDBCDataSourceProvider> mockedJDBCDataSourceProvider =
Mockito.mockStatic(JDBCDataSourceProvider.class)) {
HikariDataSource dataSource = Mockito.mock(HikariDataSource.class);
mockedJDBCDataSourceProvider
.when(() -> JDBCDataSourceProvider.createOneSessionJdbcDataSource(Mockito.any(), Mockito.any()))
.thenReturn(dataSource);
Assertions.assertNotNull(
JDBCDataSourceProvider.createOneSessionJdbcDataSource(new DorisConnectionParam(), DbType.DORIS));
}
}
}

151
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/utils/DataSourceUtilsTest.java

@ -0,0 +1,151 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.doris.utils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.plugin.doris.param.DorisConnectionParam;
import org.apache.dolphinscheduler.plugin.doris.param.DorisDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.sql.Connection;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
public class DataSourceUtilsTest {
@Test
public void testCheckDatasourceParamOne() {
DorisDataSourceParamDTO dorisDatasourceParamDTO = new DorisDataSourceParamDTO();
dorisDatasourceParamDTO.setHost("localhost,localhost1");
dorisDatasourceParamDTO.setDatabase("default");
Map<String, String> other = new HashMap<>();
other.put("serverTimezone", "Asia/Shanghai");
other.put("queryTimeout", "-1");
other.put("characterEncoding", "utf8");
dorisDatasourceParamDTO.setOther(other);
DataSourceUtils.checkDatasourceParam(dorisDatasourceParamDTO);
Assertions.assertTrue(true);
Assertions.assertEquals("localhost,localhost1", dorisDatasourceParamDTO.getHost());
}
@Test
public void testBuildConnectionParamTwo() {
DorisDataSourceParamDTO dorisDatasourceParamDTO = new DorisDataSourceParamDTO();
dorisDatasourceParamDTO.setHost("localhost");
dorisDatasourceParamDTO.setDatabase("default");
dorisDatasourceParamDTO.setUserName("root");
dorisDatasourceParamDTO.setPort(3306);
dorisDatasourceParamDTO.setPassword("123456");
try (
MockedStatic<PasswordUtils> mockedStaticPasswordUtils = Mockito.mockStatic(PasswordUtils.class);
MockedStatic<CommonUtils> mockedStaticCommonUtils = Mockito.mockStatic(CommonUtils.class)) {
mockedStaticPasswordUtils.when(() -> PasswordUtils.encodePassword(Mockito.anyString()))
.thenReturn("123456");
mockedStaticCommonUtils.when(CommonUtils::getKerberosStartupState).thenReturn(false);
ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(dorisDatasourceParamDTO);
Assertions.assertNotNull(connectionParam);
}
}
@Test
public void testBuildConnectionParamsThree() {
DorisDataSourceParamDTO dorisDatasourceParamDTO = new DorisDataSourceParamDTO();
dorisDatasourceParamDTO.setHost("localhost");
dorisDatasourceParamDTO.setDatabase("default");
dorisDatasourceParamDTO.setUserName("root");
dorisDatasourceParamDTO.setPort(3306);
dorisDatasourceParamDTO.setPassword("123456");
ConnectionParam connectionParam =
DataSourceUtils.buildConnectionParams(DbType.DORIS, JSONUtils.toJsonString(dorisDatasourceParamDTO));
Assertions.assertNotNull(connectionParam);
}
@Test
public void testGetConnection() throws ExecutionException {
try (
MockedStatic<PropertyUtils> mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class);
MockedStatic<DataSourceClientProvider> mockedStaticDataSourceClientProvider =
Mockito.mockStatic(DataSourceClientProvider.class)) {
mockedStaticPropertyUtils.when(() -> PropertyUtils.getLong("kerberos.expire.time", 24L)).thenReturn(24L);
DataSourceClientProvider clientProvider = Mockito.mock(DataSourceClientProvider.class);
mockedStaticDataSourceClientProvider.when(DataSourceClientProvider::getInstance).thenReturn(clientProvider);
Connection connection = Mockito.mock(Connection.class);
Mockito.when(clientProvider.getConnection(Mockito.any(), Mockito.any())).thenReturn(connection);
DorisConnectionParam connectionParam = new DorisConnectionParam();
connectionParam.setUser("root");
connectionParam.setPassword("123456");
connection = DataSourceClientProvider.getInstance().getConnection(DbType.DORIS, connectionParam);
Assertions.assertNotNull(connection);
}
}
@Test
public void testGetJdbcUrl() {
DorisConnectionParam dorisConnectionParam = new DorisConnectionParam();
dorisConnectionParam.setJdbcUrl("jdbc:mysql://localhost,localhost2:3308?allowLoadLocalInfile=false");
String jdbcUrl = DataSourceUtils.getJdbcUrl(DbType.DORIS, dorisConnectionParam);
Assertions.assertEquals(
"jdbc:mysql://localhost,localhost2:3308?allowLoadLocalInfile=false",
jdbcUrl);
}
@Test
public void testBuildDatasourceParamDTO() {
DorisConnectionParam connectionParam = new DorisConnectionParam();
connectionParam.setJdbcUrl(
"jdbc:mysql://localhost,localhost2:3308?allowLoadLocalInfile=false");
connectionParam.setAddress("jdbc:mysql://localhost:3308,localhost2:3308");
connectionParam.setUser("root");
connectionParam.setPassword("123456");
Assertions.assertNotNull(
DataSourceUtils.buildDatasourceParamDTO(DbType.DORIS, JSONUtils.toJsonString(connectionParam)));
}
@Test
public void testGetDatasourceProcessor() {
Assertions.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.DORIS));
}
@Test
public void testGetDatasourceProcessorError() {
Assertions.assertThrows(Exception.class, () -> {
DataSourceUtils.getDatasourceProcessor(null);
});
}
}

1
dolphinscheduler-datasource-plugin/pom.xml

@ -51,6 +51,7 @@
<module>dolphinscheduler-datasource-databend</module> <module>dolphinscheduler-datasource-databend</module>
<module>dolphinscheduler-datasource-snowflake</module> <module>dolphinscheduler-datasource-snowflake</module>
<module>dolphinscheduler-datasource-vertica</module> <module>dolphinscheduler-datasource-vertica</module>
<module>dolphinscheduler-datasource-doris</module>
</modules> </modules>
<dependencyManagement> <dependencyManagement>

3
dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java

@ -50,7 +50,8 @@ public enum DbType {
DATABEND(19, "databend"), DATABEND(19, "databend"),
SNOWFLAKE(20, "snowflake"), SNOWFLAKE(20, "snowflake"),
VERTICA(21, "vertica"), VERTICA(21, "vertica"),
HANA(22, "hana"); HANA(22, "hana"),
DORIS(23, "doris");
private static final Map<Integer, DbType> DB_TYPE_MAP = private static final Map<Integer, DbType> DB_TYPE_MAP =
Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity()));

1
dolphinscheduler-ui/src/service/modules/data-source/types.ts

@ -37,6 +37,7 @@ type IDataBase =
| 'DATABEND' | 'DATABEND'
| 'SNOWFLAKE' | 'SNOWFLAKE'
| 'HANA' | 'HANA'
| 'DORIS'
type IDataBaseLabel = type IDataBaseLabel =
| 'MYSQL' | 'MYSQL'

5
dolphinscheduler-ui/src/views/datasource/list/use-form.ts

@ -401,6 +401,11 @@ export const datasourceType: IDataBaseOptionKeys = {
value: 'HANA', value: 'HANA',
label: 'HANA', label: 'HANA',
defaultPort: 30015 defaultPort: 30015
},
DORIS: {
value: 'DORIS',
label: 'DORIS',
defaultPort: 9030
} }
} }

5
dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts

@ -132,6 +132,11 @@ export function useDatasource(
id: 22, id: 22,
code: 'HANA', code: 'HANA',
disabled: false disabled: false
},
{
id: 23,
code: 'DORIS',
disabled: false
} }
] ]

Loading…
Cancel
Save