diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js index 95f042b55c..e005b7a63d 100644 --- a/docs/configs/docsdev.js +++ b/docs/configs/docsdev.js @@ -331,6 +331,10 @@ export default { title: 'SSH', link: '/en-us/docs/dev/user_doc/guide/datasource/ssh.html', }, + { + title: 'Databend', + link: '/en-us/docs/dev/user_doc/guide/datasource/databend.html', + }, ], }, { @@ -1012,6 +1016,10 @@ export default { title: 'SSH', link: '/zh-cn/docs/dev/user_doc/guide/datasource/ssh.html', }, + { + title: 'Databend', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/databend.html', + }, ], }, { diff --git a/docs/docs/en/guide/datasource/databend.md b/docs/docs/en/guide/datasource/databend.md new file mode 100644 index 0000000000..6c5f2ec4ee --- /dev/null +++ b/docs/docs/en/guide/datasource/databend.md @@ -0,0 +1,21 @@ +# Databend + +![Databend Datasource](../../../../img/new_ui/dev/datasource/Databend.png) + +## Datasource Parameters + +| **Datasource** | **Description** | +|-------------------------|-------------------------------------------------------------| +| Datasource | Select DATABEND. | +| Datasource Name | Enter the name of the datasource. | +| Description | Enter a description of the datasource. | +| IP/Host Name | Enter the DATABEND service IP. | +| Port | Enter the DATABEND service port. | +| Username | Set the username for DATABEND connection. | +| Password | Set the password for DATABEND connection. | +| Database Name | Enter the database name of the DATABEND connection. | +| jdbc connect parameters | Parameter settings for DATABEND connection, in JSON format. | + +## Native Supported + +Yes, could use this datasource by default. diff --git a/docs/docs/zh/guide/datasource/databend.md b/docs/docs/zh/guide/datasource/databend.md new file mode 100644 index 0000000000..0283fbab1f --- /dev/null +++ b/docs/docs/zh/guide/datasource/databend.md @@ -0,0 +1,23 @@ +# Databend + +![Databend 数据源](../../../../img/new_ui/dev/datasource/Databend.png) + +## 数据源参数 + +| **数据源** | **描述** | +|-----------|-------------------------------| +| 数据源 | 选择 DATABEND。 | +| 数据源名称 | 输入数据源的名称。 | +| 描述 | 输入数据源的描述。 | +| IP/主机名 | 输入 DATABEND 服务的 IP 地址。 | +| 端口 | 输入 DATABEND 服务的端口。 | +| 用户名 | 设置 DATABEND 连接的用户名。 | +| 密码 | 设置 DATABEND 连接的密码。 | +| 数据库名称 | 输入 DATABEND 连接的数据库名称。 | +| jdbc 连接参数 | DATABEND 连接的参数设置，以 JSON 格式表示。 | + +``jdbc`` 连接参数可参考 [databend-jdbc](https://github.com/databendcloud/databend-jdbc) + +## 是否原生支持 + +是，数据源不需要任务附加操作即可使用。 diff --git a/docs/img/new_ui/dev/datasource/Databend.png b/docs/img/new_ui/dev/datasource/Databend.png new file mode 100644 index 0000000000..dec146f4cb Binary files /dev/null and b/docs/img/new_ui/dev/datasource/Databend.png differ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index fc39e1b72d..e207d16a60 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -678,6 +678,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource schemaPattern = "dbo"; break; case CLICKHOUSE: + case DATABEND: case PRESTO: if (!StringUtils.isEmpty(schema)) { schemaPattern = schema; diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml index 0bff3cb112..8cb407c31c 100644 --- a/dolphinscheduler-bom/pom.xml +++ b/dolphinscheduler-bom/pom.xml @@ -71,6 +71,7 @@ 2.11.0 6.1.1 0.3.2-patch11 + 0.0.7 1.4.0 11.2.1.jre8 0.238.1 @@ -575,6 +576,12 @@ ${clickhouse-jdbc.version} + + com.databend + databend-jdbc + ${databend-jdbc.version} + + com.microsoft.sqlserver mssql-jdbc diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java index 16e66d515a..3812759e3d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java @@ -29,6 +29,7 @@ public class DataSourceConstants { public static final String COM_MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver"; public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"; public static final String COM_CLICKHOUSE_JDBC_DRIVER = "com.clickhouse.jdbc.ClickHouseDriver"; + public static final String COM_DATABEND_JDBC_DRIVER = "com.databend.jdbc.DatabendDriver"; public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.OracleDriver"; public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; @@ -47,6 +48,7 @@ public class DataSourceConstants { public static final String MYSQL_VALIDATION_QUERY = "select 1"; public static final String HIVE_VALIDATION_QUERY = "select 1"; public static final String CLICKHOUSE_VALIDATION_QUERY = "select 1"; + public static final String DATABEND_VALIDATION_QUERY = "select 1"; public static final String ORACLE_VALIDATION_QUERY = "select 1 from dual"; public static final String SQLSERVER_VALIDATION_QUERY = "select 1"; public static final String DB2_VALIDATION_QUERY = "select 1 from sysibm.sysdummy1"; @@ -66,6 +68,7 @@ public class DataSourceConstants { public static final String JDBC_HIVE_2 = "jdbc:hive2://"; public static final String JDBC_KYUUBI = "jdbc:kyuubi://"; public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://"; + public static final String JDBC_DATABEND = "jdbc:databend://"; public static final String JDBC_ORACLE_SID = "jdbc:oracle:thin:@"; public static final String JDBC_ORACLE_SERVICE_NAME = "jdbc:oracle:thin:@//"; public static final String JDBC_SQLSERVER = "jdbc:sqlserver://"; diff --git a/dolphinscheduler-data-quality/pom.xml b/dolphinscheduler-data-quality/pom.xml index 42d781ee01..0643fdac5a 100644 --- a/dolphinscheduler-data-quality/pom.xml +++ b/dolphinscheduler-data-quality/pom.xml @@ -121,6 +121,11 @@ + + com.databend + databend-jdbc + + com.microsoft.sqlserver mssql-jdbc diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml index 5eda89cc12..21eacfe9f4 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml @@ -32,6 +32,11 @@ dolphinscheduler-datasource-clickhouse ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-datasource-databend + ${project.version} + org.apache.dolphinscheduler dolphinscheduler-datasource-db2 diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/pom.xml new file mode 100644 index 0000000000..91478998b0 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/pom.xml @@ -0,0 +1,71 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin + dev-SNAPSHOT + + + dolphinscheduler-datasource-databend + jar + ${project.artifactId} + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + + + + com.databend + databend-jdbc + + + javax.xml.bind + jaxb-api + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + + + + org.lz4 + lz4-java + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannel.java new file mode 100644 index 0000000000..e872f6539f --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannel.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.databend; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class DatabendDataSourceChannel implements DataSourceChannel { + + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + return new DatabendDataSourceClient(baseConnectionParam, dbType); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java new file mode 100644 index 0000000000..0ea40c3b13 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.databend; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.auto.service.AutoService; + +@AutoService(DataSourceChannelFactory.class) +public class DatabendDataSourceChannelFactory implements DataSourceChannelFactory { + + @Override + public String getName() { + return "databend"; + } + + @Override + public DataSourceChannel create() { + return new DatabendDataSourceChannel(); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceClient.java new file mode 100644 index 0000000000..df94966d00 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceClient.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.databend; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class DatabendDataSourceClient extends CommonDataSourceClient { + + public DatabendDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + super(baseConnectionParam, dbType); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendConnectionParam.java new file mode 100644 index 0000000000..ce2da81646 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendConnectionParam.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.databend.param; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +public class DatabendConnectionParam extends BaseConnectionParam { + + @Override + public String toString() { + return "DatabendConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceParamDTO.java new file mode 100644 index 0000000000..82ea41f1bf --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceParamDTO.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.databend.param; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class DatabendDataSourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "DatabendDataSourceParamDTO{" + + "host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.DATABEND; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessor.java new file mode 100644 index 0000000000..e2197b7de7 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessor.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.databend.param; + +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.constants.DataSourceConstants; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.google.auto.service.AutoService; + +@AutoService(DataSourceProcessor.class) +public class DatabendDataSourceProcessor extends AbstractDataSourceProcessor { + + @Override + public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { + return JSONUtils.parseObject(paramJson, DatabendDataSourceParamDTO.class); + } + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + DatabendConnectionParam connectionParams = (DatabendConnectionParam) createConnectionParams(connectionJson); + + DatabendDataSourceParamDTO databendDatasourceParamDTO = new DatabendDataSourceParamDTO(); + databendDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + databendDatasourceParamDTO.setUserName(connectionParams.getUser()); + databendDatasourceParamDTO.setOther(connectionParams.getOther()); + + String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); + databendDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + databendDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + + return databendDatasourceParamDTO; + } + + @Override + public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + DatabendDataSourceParamDTO databendParam = (DatabendDataSourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", DataSourceConstants.JDBC_DATABEND, databendParam.getHost(), + databendParam.getPort()); + String jdbcUrl = address + "/" + databendParam.getDatabase(); + + DatabendConnectionParam databendConnectionParam = new DatabendConnectionParam(); + databendConnectionParam.setDatabase(databendParam.getDatabase()); + databendConnectionParam.setAddress(address); + databendConnectionParam.setJdbcUrl(jdbcUrl); + databendConnectionParam.setUser(databendParam.getUserName()); + databendConnectionParam.setPassword(PasswordUtils.encodePassword(databendParam.getPassword())); + databendConnectionParam.setDriverClassName(getDatasourceDriver()); + databendConnectionParam.setValidationQuery(getValidationQuery()); + databendConnectionParam.setOther(databendParam.getOther()); + return databendConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, DatabendConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return DataSourceConstants.COM_DATABEND_JDBC_DRIVER; + } + + @Override + public String getValidationQuery() { + return DataSourceConstants.DATABEND_VALIDATION_QUERY; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + DatabendConnectionParam databendConnectionParam = (DatabendConnectionParam) connectionParam; + String jdbcUrl = databendConnectionParam.getJdbcUrl(); + if (MapUtils.isNotEmpty(databendConnectionParam.getOther())) { + jdbcUrl = String.format("%s?%s", jdbcUrl, transformOther(databendConnectionParam.getOther())); + } + return jdbcUrl; + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + DatabendConnectionParam databendConnectionParam = (DatabendConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(databendConnectionParam), + databendConnectionParam.getUser(), + PasswordUtils.decodePassword(databendConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.DATABEND; + } + + @Override + public DataSourceProcessor create() { + return new DatabendDataSourceProcessor(); + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + List otherList = new ArrayList<>(); + otherMap.forEach((key, value) -> otherList.add(String.format("%s=%s", key, value))); + return String.join("&", otherList); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactoryTest.java new file mode 100644 index 0000000000..7d7e9353fc --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelFactoryTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.databend; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class DatabendDataSourceChannelFactoryTest { + + @Test + public void testCreate() { + DatabendDataSourceChannelFactory sourceChannelFactory = new DatabendDataSourceChannelFactory(); + DataSourceChannel dataSourceChannel = sourceChannelFactory.create(); + Assertions.assertNotNull(dataSourceChannel); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelTest.java new file mode 100644 index 0000000000..e4dcc4b9a5 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/DatabendDataSourceChannelTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.databend; + +import org.apache.dolphinscheduler.plugin.datasource.databend.param.DatabendConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class DatabendDataSourceChannelTest { + + @Test + public void testCreateDataSourceClient() { + DatabendDataSourceChannel sourceChannel = Mockito.mock(DatabendDataSourceChannel.class); + DatabendDataSourceClient dataSourceClient = Mockito.mock(DatabendDataSourceClient.class); + Mockito.when(sourceChannel.createDataSourceClient(Mockito.any(), Mockito.any())).thenReturn(dataSourceClient); + Assertions.assertNotNull( + sourceChannel.createDataSourceClient(new DatabendConnectionParam(), DbType.DATABEND)); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java new file mode 100644 index 0000000000..f225a2fd3d --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/test/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessorTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.databend.param; + +import org.apache.dolphinscheduler.common.constants.DataSourceConstants; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class DatabendDataSourceProcessorTest { + + private DatabendDataSourceProcessor databendDataSourceProcessor = new DatabendDataSourceProcessor(); + + @Test + public void testCheckDatasourceParam() { + DatabendDataSourceParamDTO databendDataSourceParamDTO = new DatabendDataSourceParamDTO(); + databendDataSourceParamDTO.setHost("localhost"); + databendDataSourceParamDTO.setDatabase("default"); + Map other = new HashMap<>(); + other.put("ssl", "true"); + databendDataSourceParamDTO.setOther(other); + DataSourceUtils.checkDatasourceParam(databendDataSourceParamDTO); + Assertions.assertTrue(true); + } + + @Test + public void testBuildConnectionParams() { + DatabendDataSourceParamDTO databendDataSourceParamDTO = new DatabendDataSourceParamDTO(); + databendDataSourceParamDTO.setHost("localhost"); + databendDataSourceParamDTO.setDatabase("default"); + databendDataSourceParamDTO.setUserName("root"); + databendDataSourceParamDTO.setPort(8000); + databendDataSourceParamDTO.setPassword("123456"); + + try ( + MockedStatic mockedStaticPasswordUtils = Mockito.mockStatic(PasswordUtils.class); + MockedStatic mockedStaticCommonUtils = Mockito.mockStatic(CommonUtils.class)) { + mockedStaticPasswordUtils.when(() -> PasswordUtils.encodePassword(Mockito.anyString())) + .thenReturn("123456"); + mockedStaticCommonUtils.when(CommonUtils::getKerberosStartupState).thenReturn(false); + ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(databendDataSourceParamDTO); + Assertions.assertNotNull(connectionParam); + Assertions.assertEquals(connectionParam.getPassword(), "123456"); + } + } + + @Test + public void testBuildConnectionParams2() { + DatabendDataSourceParamDTO databendDataSourceParamDTO = new DatabendDataSourceParamDTO(); + databendDataSourceParamDTO.setHost("127.0.0.1"); + databendDataSourceParamDTO.setDatabase("databend"); + databendDataSourceParamDTO.setUserName("root"); + databendDataSourceParamDTO.setPort(8000); + databendDataSourceParamDTO.setPassword("databend"); + ConnectionParam connectionParam = + DataSourceUtils.buildConnectionParams(DbType.DATABEND, + JSONUtils.toJsonString(databendDataSourceParamDTO)); + Assertions.assertNotNull(connectionParam); + } + + @Test + public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("ssl", "true"); + DatabendDataSourceParamDTO databendDataSourceParamDTO = new DatabendDataSourceParamDTO(); + databendDataSourceParamDTO.setHost("localhost"); + databendDataSourceParamDTO.setPort(8000); + databendDataSourceParamDTO.setUserName("root"); + databendDataSourceParamDTO.setDatabase("root"); + databendDataSourceParamDTO.setOther(props); + + try ( + MockedStatic mockedStaticPasswordUtils = Mockito.mockStatic(PasswordUtils.class); + MockedStatic mockedStaticCommonUtils = Mockito.mockStatic(CommonUtils.class)) { + mockedStaticPasswordUtils.when(() -> PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); + DatabendConnectionParam connectionParams = (DatabendConnectionParam) databendDataSourceProcessor + .createConnectionParams(databendDataSourceParamDTO); + Assertions.assertNotNull(connectionParams); + Assertions.assertEquals("jdbc:databend://localhost:8000", connectionParams.getAddress()); + } + } + + @Test + public void testCreateConnectionParams2() { + String connectionParam = "{\"user\":\"default\",\"address\":\"jdbc:databend://localhost:8000\"" + + ",\"jdbcUrl\":\"jdbc:databend://localhost:8000/default\"}"; + DatabendConnectionParam connectionParams = (DatabendConnectionParam) databendDataSourceProcessor + .createConnectionParams(connectionParam); + Assertions.assertNotNull(connectionParam); + Assertions.assertEquals("default", connectionParams.getUser()); + Assertions.assertEquals("jdbc:databend://localhost:8000/default", connectionParams.getJdbcUrl()); + } + + @Test + public void testCreateDatasourceParamDTO() { + String connectionParam = "{\"user\":\"root\",\"address\":\"jdbc:databend://localhost:8000\"" + + ",\"jdbcUrl\":\"jdbc:databend://localhost:8000/default\"}"; + DatabendDataSourceParamDTO databendDataSourceParamDTO = (DatabendDataSourceParamDTO) databendDataSourceProcessor + .createDatasourceParamDTO(connectionParam); + Assertions.assertEquals("root", databendDataSourceParamDTO.getUserName()); + Assertions.assertEquals("localhost", databendDataSourceParamDTO.getHost()); + Assertions.assertEquals(8000, databendDataSourceParamDTO.getPort()); + } + + @Test + public void testGetDatasourceDriver() { + Assertions.assertEquals(DataSourceConstants.COM_DATABEND_JDBC_DRIVER, + databendDataSourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + DatabendConnectionParam connectionParam = new DatabendConnectionParam(); + connectionParam.setJdbcUrl("jdbc:databend://localhost:8000/default"); + Map other = new HashMap<>(); + other.put("ssl", "true"); + connectionParam.setOther(other); + Assertions.assertEquals("jdbc:databend://localhost:8000/default?ssl=true", + databendDataSourceProcessor.getJdbcUrl(connectionParam)); + } + + @Test + public void testDbType() { + Assertions.assertEquals(19, DbType.DATABEND.getCode()); + Assertions.assertEquals("databend", DbType.DATABEND.getDescp()); + Assertions.assertEquals(DbType.DATABEND, DbType.of(19)); + } + + @Test + public void testGetDbType() { + Assertions.assertEquals(DbType.DATABEND, databendDataSourceProcessor.getDbType()); + } + + @Test + public void testGetValidationQuery() { + Assertions.assertEquals(DataSourceConstants.DATABEND_VALIDATION_QUERY, + databendDataSourceProcessor.getValidationQuery()); + } + + @Test + public void testBuildString() { + DatabendDataSourceParamDTO databendDataSourceParamDTO = new DatabendDataSourceParamDTO(); + databendDataSourceParamDTO.setHost("localhost"); + databendDataSourceParamDTO.setDatabase("default"); + databendDataSourceParamDTO.setUserName("root"); + databendDataSourceParamDTO.setPort(8000); + databendDataSourceParamDTO.setPassword("root"); + Assertions.assertNotNull(databendDataSourceParamDTO.toString()); + } + +} diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml index 97ece76fa8..4e81fea5c1 100644 --- a/dolphinscheduler-datasource-plugin/pom.xml +++ b/dolphinscheduler-datasource-plugin/pom.xml @@ -48,6 +48,7 @@ dolphinscheduler-datasource-azure-sql dolphinscheduler-datasource-dameng dolphinscheduler-datasource-ssh + dolphinscheduler-datasource-databend diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 49ffdced80..c21d165da9 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -225,6 +225,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. caffeine 2.9.3: https://mvnrepository.com/artifact/com.github.ben-manes.caffeine/caffeine/2.9.3, Apache 2.0 classmate 1.5.1: https://mvnrepository.com/artifact/com.fasterxml/classmate/1.5.1, Apache 2.0 clickhouse-jdbc 0.3.2-patch11: https://mvnrepository.com/artifact/com.clickhouse/clickhouse-jdbc/0.3.2-patch11, Apache 2.0 + databend-jdbc 0.0.7 https://mvnrepository.com/artifact/com.databend/databend-jdbc/0.0.7 Apache 2.0 lz4-java 1.4.0: https://mvnrepository.com/artifact/org.lz4/lz4-java/1.4.0, Apache 2.0 commons-beanutils 1.9.4 https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils/1.9.4, Apache 2.0 commons-cli 1.2: https://mvnrepository.com/artifact/commons-cli/commons-cli/1.2, Apache 2.0 diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-databend-jdbc.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-databend-jdbc.txt new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-databend-jdbc.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 3244f2b9ea..9e8dc067f0 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -46,7 +46,8 @@ public enum DbType { DAMENG(15, "dameng"), OCEANBASE(16, "oceanbase"), SSH(17, "ssh"), - KYUUBI(18, "kyuubi"); + KYUUBI(18, "kyuubi"), + DATABEND(19, "databend"); private static final Map DB_TYPE_MAP = Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java index 41b94b009f..bb7c8fc439 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java @@ -238,7 +238,6 @@ public class TaskConstants { /** * year_week - * */ public static final String YEAR_WEEK = "year_week"; /** @@ -301,6 +300,7 @@ public class TaskConstants { public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; public static final String JDBC_HIVE_2 = "jdbc:hive2://"; public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://"; + public static final String JDBC_DATABEND = "jdbc:databend://"; public static final String JDBC_ORACLE_SID = "jdbc:oracle:thin:@"; public static final String JDBC_ORACLE_SERVICE_NAME = "jdbc:oracle:thin:@//"; public static final String JDBC_SQLSERVER = "jdbc:sqlserver://"; @@ -314,6 +314,7 @@ public class TaskConstants { public static final String COM_MYSQL_CJ_JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"; public static final String COM_CLICKHOUSE_JDBC_DRIVER = "com.clickhouse.jdbc.ClickHouseDriver"; + public static final String COM_DATABEND_JDBC_DRIVER = "com.databend.jdbc.DatabendDriver"; public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.driver.OracleDriver"; public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java index 16fe49fe9a..30b9224d96 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java @@ -51,6 +51,7 @@ public class DataxUtils { public static final String DATAX_WRITER_PLUGIN_SQLSERVER = "sqlserverwriter"; public static final String DATAX_WRITER_PLUGIN_CLICKHOUSE = "clickhousewriter"; + public static final String DATAX_WRITER_PLUGIN_DATABEND = "databendwriter"; public static final String DATAX_WRITER_PLUGIN_RDBMS = "rdbmswriter"; @@ -87,6 +88,8 @@ public class DataxUtils { return DATAX_WRITER_PLUGIN_SQLSERVER; case CLICKHOUSE: return DATAX_WRITER_PLUGIN_CLICKHOUSE; + case DATABEND: + return DATAX_WRITER_PLUGIN_DATABEND; case HIVE: return DATAX_WRITER_PLUGIN_RDBMS; case PRESTO: @@ -151,6 +154,8 @@ public class DataxUtils { return String.format("\"%s\"", column); case CLICKHOUSE: return String.format("`%s`", column); + case DATABEND: + return String.format("`%s`", column); default: return column; } diff --git a/dolphinscheduler-ui/src/service/modules/data-source/types.ts b/dolphinscheduler-ui/src/service/modules/data-source/types.ts index 10a200c9db..3b4246369c 100644 --- a/dolphinscheduler-ui/src/service/modules/data-source/types.ts +++ b/dolphinscheduler-ui/src/service/modules/data-source/types.ts @@ -33,6 +33,7 @@ type IDataBase = | 'DAMENG' | 'OCEANBASE' | 'SSH' + | 'DATABEND' type IDataBaseLabel = | 'MYSQL' diff --git a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts index d1181cb03f..e381511ad8 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts +++ b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts @@ -427,6 +427,11 @@ export const datasourceType: IDataBaseOptionKeys = { value: 'SSH', label: 'SSH', defaultPort: 22 + }, + DATABEND: { + value: 'DATABEND', + label: 'DATABEND', + defaultPort: 8000 } } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts index 4f9f132947..6d51276332 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts @@ -118,6 +118,11 @@ export function useDatasource( code: 'SSH', disabled: true }, + { + id: 16, + code: 'DATABEND', + disabled: false + }, ] const getDatasourceTypes = async () => { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts index 72952bb807..1a8c3041d1 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts @@ -14,288 +14,289 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ref, onMounted, watch } from 'vue' -import { useI18n } from 'vue-i18n' -import { useCustomParams, useDatasource, useResources } from '.' -import type { IJsonItem } from '../types' +import {ref, onMounted, watch} from 'vue' +import {useI18n} from 'vue-i18n' +import {useCustomParams, useDatasource, useResources} from '.' +import type {IJsonItem} from '../types' export function useDataX(model: { [field: string]: any }): IJsonItem[] { - const { t } = useI18n() - const jobSpeedByteOptions: any[] = [ - { - label: `0(${t('project.node.unlimited')})`, - value: 0 - }, - { - label: '1KB', - value: 1024 - }, - { - label: '10KB', - value: 10240 - }, - { - label: '50KB', - value: 51200 - }, - { - label: '100KB', - value: 102400 - }, - { - label: '512KB', - value: 524288 - } - ] - const jobSpeedRecordOptions: any[] = [ - { - label: `0(${t('project.node.unlimited')})`, - value: 0 - }, - { - label: '500', - value: 500 - }, - { - label: '1000', - value: 1000 - }, - { - label: '1500', - value: 1500 - }, - { - label: '2000', - value: 2000 - }, - { - label: '2500', - value: 2500 - }, - { - label: '3000', - value: 3000 - } - ] - const memoryLimitOptions = [ - { - label: '1G', - value: 1 - }, - { - label: '2G', - value: 2 - }, - { - label: '3G', - value: 3 - }, - { - label: '4G', - value: 4 - } - ] + const {t} = useI18n() + const jobSpeedByteOptions: any[] = [ + { + label: `0(${t('project.node.unlimited')})`, + value: 0 + }, + { + label: '1KB', + value: 1024 + }, + { + label: '10KB', + value: 10240 + }, + { + label: '50KB', + value: 51200 + }, + { + label: '100KB', + value: 102400 + }, + { + label: '512KB', + value: 524288 + } + ] + const jobSpeedRecordOptions: any[] = [ + { + label: `0(${t('project.node.unlimited')})`, + value: 0 + }, + { + label: '500', + value: 500 + }, + { + label: '1000', + value: 1000 + }, + { + label: '1500', + value: 1500 + }, + { + label: '2000', + value: 2000 + }, + { + label: '2500', + value: 2500 + }, + { + label: '3000', + value: 3000 + } + ] + const memoryLimitOptions = [ + { + label: '1G', + value: 1 + }, + { + label: '2G', + value: 2 + }, + { + label: '3G', + value: 3 + }, + { + label: '4G', + value: 4 + } + ] - const sqlEditorSpan = ref(24) - const jsonEditorSpan = ref(0) - const datasourceSpan = ref(12) - const destinationDatasourceSpan = ref(8) - const otherStatementSpan = ref(22) - const jobSpeedSpan = ref(12) - const customParameterSpan = ref(0) - const useResourcesSpan = ref(0) + const sqlEditorSpan = ref(24) + const jsonEditorSpan = ref(0) + const datasourceSpan = ref(12) + const destinationDatasourceSpan = ref(8) + const otherStatementSpan = ref(22) + const jobSpeedSpan = ref(12) + const customParameterSpan = ref(0) + const useResourcesSpan = ref(0) - const initConstants = () => { - if (model.customConfig) { - sqlEditorSpan.value = 0 - jsonEditorSpan.value = 24 - datasourceSpan.value = 0 - destinationDatasourceSpan.value = 0 - otherStatementSpan.value = 0 - jobSpeedSpan.value = 0 - customParameterSpan.value = 24 - useResourcesSpan.value = 24 - } else { - sqlEditorSpan.value = 24 - jsonEditorSpan.value = 0 - datasourceSpan.value = 12 - destinationDatasourceSpan.value = 8 - otherStatementSpan.value = 22 - jobSpeedSpan.value = 12 - customParameterSpan.value = 0 - useResourcesSpan.value = 0 - } - } - const supportedDatasourceType = [ - 'MYSQL', - 'POSTGRESQL', - 'ORACLE', - 'SQLSERVER', - 'CLICKHOUSE', - 'HIVE', - 'PRESTO' - ] - onMounted(() => { - initConstants() - }) - watch( - () => model.customConfig, - () => { - initConstants() + const initConstants = () => { + if (model.customConfig) { + sqlEditorSpan.value = 0 + jsonEditorSpan.value = 24 + datasourceSpan.value = 0 + destinationDatasourceSpan.value = 0 + otherStatementSpan.value = 0 + jobSpeedSpan.value = 0 + customParameterSpan.value = 24 + useResourcesSpan.value = 24 + } else { + sqlEditorSpan.value = 24 + jsonEditorSpan.value = 0 + datasourceSpan.value = 12 + destinationDatasourceSpan.value = 8 + otherStatementSpan.value = 22 + jobSpeedSpan.value = 12 + customParameterSpan.value = 0 + useResourcesSpan.value = 0 + } } - ) + const supportedDatasourceType = [ + 'MYSQL', + 'POSTGRESQL', + 'ORACLE', + 'SQLSERVER', + 'CLICKHOUSE', + 'DATABEND', + 'HIVE', + 'PRESTO' + ] + onMounted(() => { + initConstants() + }) + watch( + () => model.customConfig, + () => { + initConstants() + } + ) - return [ - { - type: 'switch', - field: 'customConfig', - name: t('project.node.datax_custom_template') - }, - ...useDatasource(model, { - typeField: 'dsType', - sourceField: 'dataSource', - span: datasourceSpan, - supportedDatasourceType - }), - { - type: 'editor', - field: 'sql', - name: t('project.node.sql_statement'), - span: sqlEditorSpan, - validate: { - trigger: ['input', 'trigger'], - required: true, - message: t('project.node.sql_empty_tips') - } - }, - { - type: 'editor', - field: 'json', - name: t('project.node.datax_json_template'), - span: jsonEditorSpan, - validate: { - trigger: ['input', 'trigger'], - required: true, - message: t('project.node.sql_empty_tips') - } - }, - useResources(useResourcesSpan), - ...useDatasource(model, { - typeField: 'dtType', - sourceField: 'dataTarget', - span: destinationDatasourceSpan, - supportedDatasourceType - }), - { - type: 'input', - field: 'targetTable', - name: t('project.node.datax_target_table'), - span: destinationDatasourceSpan, - props: { - placeholder: t('project.node.datax_target_table_tips') - }, - validate: { - trigger: ['input', 'blur'], - required: true - } - }, - { - type: 'multi-input', - field: 'preStatements', - name: t('project.node.datax_target_database_pre_sql'), - span: otherStatementSpan, - props: { - placeholder: t('project.node.datax_non_query_sql_tips'), - type: 'textarea', - autosize: { minRows: 1 } - } - }, - { - type: 'multi-input', - field: 'postStatements', - name: t('project.node.datax_target_database_post_sql'), - span: otherStatementSpan, - props: { - placeholder: t('project.node.datax_non_query_sql_tips'), - type: 'textarea', - autosize: { minRows: 1 } - } - }, - { - type: 'select', - field: 'jobSpeedByte', - name: t('project.node.datax_job_speed_byte'), - span: jobSpeedSpan, - options: jobSpeedByteOptions, - value: 0 - }, - { - type: 'select', - field: 'jobSpeedRecord', - name: t('project.node.datax_job_speed_record'), - span: jobSpeedSpan, - options: jobSpeedRecordOptions, - value: 1000 - }, - { - type: 'custom-parameters', - field: 'localParams', - name: t('project.node.custom_parameters'), - span: customParameterSpan, - children: [ + return [ + { + type: 'switch', + field: 'customConfig', + name: t('project.node.datax_custom_template') + }, + ...useDatasource(model, { + typeField: 'dsType', + sourceField: 'dataSource', + span: datasourceSpan, + supportedDatasourceType + }), { - type: 'input', - field: 'prop', - span: 10, - props: { - placeholder: t('project.node.prop_tips'), - maxLength: 256 - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.prop_tips')) - } + type: 'editor', + field: 'sql', + name: t('project.node.sql_statement'), + span: sqlEditorSpan, + validate: { + trigger: ['input', 'trigger'], + required: true, + message: t('project.node.sql_empty_tips') + } + }, + { + type: 'editor', + field: 'json', + name: t('project.node.datax_json_template'), + span: jsonEditorSpan, + validate: { + trigger: ['input', 'trigger'], + required: true, + message: t('project.node.sql_empty_tips') + } + }, + useResources(useResourcesSpan), + ...useDatasource(model, { + typeField: 'dtType', + sourceField: 'dataTarget', + span: destinationDatasourceSpan, + supportedDatasourceType + }), + { + type: 'input', + field: 'targetTable', + name: t('project.node.datax_target_table'), + span: destinationDatasourceSpan, + props: { + placeholder: t('project.node.datax_target_table_tips') + }, + validate: { + trigger: ['input', 'blur'], + required: true + } + }, + { + type: 'multi-input', + field: 'preStatements', + name: t('project.node.datax_target_database_pre_sql'), + span: otherStatementSpan, + props: { + placeholder: t('project.node.datax_non_query_sql_tips'), + type: 'textarea', + autosize: {minRows: 1} + } + }, + { + type: 'multi-input', + field: 'postStatements', + name: t('project.node.datax_target_database_post_sql'), + span: otherStatementSpan, + props: { + placeholder: t('project.node.datax_non_query_sql_tips'), + type: 'textarea', + autosize: {minRows: 1} + } + }, + { + type: 'select', + field: 'jobSpeedByte', + name: t('project.node.datax_job_speed_byte'), + span: jobSpeedSpan, + options: jobSpeedByteOptions, + value: 0 + }, + { + type: 'select', + field: 'jobSpeedRecord', + name: t('project.node.datax_job_speed_record'), + span: jobSpeedSpan, + options: jobSpeedRecordOptions, + value: 1000 + }, + { + type: 'custom-parameters', + field: 'localParams', + name: t('project.node.custom_parameters'), + span: customParameterSpan, + children: [ + { + type: 'input', + field: 'prop', + span: 10, + props: { + placeholder: t('project.node.prop_tips'), + maxLength: 256 + }, + validate: { + trigger: ['input', 'blur'], + required: true, + validator(validate: any, value: string) { + if (!value) { + return new Error(t('project.node.prop_tips')) + } - const sameItems = model.localParams.filter( - (item: { prop: string }) => item.prop === value - ) + const sameItems = model.localParams.filter( + (item: { prop: string }) => item.prop === value + ) - if (sameItems.length > 1) { - return new Error(t('project.node.prop_repeat')) - } - } - } + if (sameItems.length > 1) { + return new Error(t('project.node.prop_repeat')) + } + } + } + }, + { + type: 'input', + field: 'value', + span: 10, + props: { + placeholder: t('project.node.value_tips'), + maxLength: 256 + } + } + ] }, { - type: 'input', - field: 'value', - span: 10, - props: { - placeholder: t('project.node.value_tips'), - maxLength: 256 - } - } - ] - }, - { - type: 'select', - field: 'xms', - name: t('project.node.datax_job_runtime_memory_xms'), - span: 12, - options: memoryLimitOptions, - value: 1 - }, - { - type: 'select', - field: 'xmx', - name: t('project.node.datax_job_runtime_memory_xmx'), - span: 12, - options: memoryLimitOptions, - value: 1 - }, - ...useCustomParams({ model, field: 'localParams', isSimple: true }) - ] + type: 'select', + field: 'xms', + name: t('project.node.datax_job_runtime_memory_xms'), + span: 12, + options: memoryLimitOptions, + value: 1 + }, + { + type: 'select', + field: 'xmx', + name: t('project.node.datax_job_runtime_memory_xmx'), + span: 12, + options: memoryLimitOptions, + value: 1 + }, + ...useCustomParams({model, field: 'localParams', isSimple: true}) + ] } diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index ed9e5e3bd2..17459f6f56 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -474,4 +474,6 @@ sshd-sftp-2.8.0.jar sshd-common-2.8.0.jar sshd-core-2.8.0.jar jcl-over-slf4j-1.7.36.jar -kyuubi-hive-jdbc-shaded-1.7.0.jar \ No newline at end of file +databend-jdbc-0.0.7.jar +kyuubi-hive-jdbc-shaded-1.7.0.jar +