diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js index dc76666ae5..f223624bdf 100644 --- a/docs/configs/docsdev.js +++ b/docs/configs/docsdev.js @@ -275,6 +275,10 @@ export default { title: 'Oracle', link: '/en-us/docs/dev/user_doc/guide/datasource/oracle.html', }, + { + title: 'Amazon Athena', + link: '/en-us/docs/dev/user_doc/guide/datasource/athena.html', + }, ], }, { @@ -883,6 +887,10 @@ export default { title: 'Spark', link: '/zh-cn/docs/dev/user_doc/guide/datasource/spark.html', }, + { + title: 'Amazon Athena', + link: '/zh-cn/docs/dev/user_doc/guide/datasource/athena.html', + }, ], }, { diff --git a/docs/docs/en/guide/datasource/athena.md b/docs/docs/en/guide/datasource/athena.md new file mode 100644 index 0000000000..ab92e06238 --- /dev/null +++ b/docs/docs/en/guide/datasource/athena.md @@ -0,0 +1,22 @@ +# AWS Athena + +![AWS Athena](../../../../img/new_ui/dev/datasource/athena.png) + +## Datasource Parameters + +| **Datasource** | **Description** | +| --- | --- | +| Datasource | Select ATHENA. | +| Datasource name | Enter the name of the DataSource. | +| Description | Enter a description of the DataSource. | +| Username | Set the AWS access key. | +| Password | Set the AWS secret access key. | +| AwsRegion | Set the AWS region. | +| Database name | Enter the database name of the ATHENA connection. | +| Jdbc connection parameters | Parameter settings for ATHENA connection, in JSON format. | + +## Native Supported + +- No, read section example in [datasource-setting](../howto/datasource-setting.md) `DataSource Center` section to activate this datasource. +- JDBC driver configuration reference document [athena-connect-with-jdbc](https://docs.amazonaws.cn/athena/latest/ug/connect-with-jdbc.html) +- Driver download link [SimbaAthenaJDBC-2.0.31.1000/AthenaJDBC42.jar](https://s3.cn-north-1.amazonaws.com.cn/athena-downloads-cn/drivers/JDBC/SimbaAthenaJDBC-2.0.31.1000/AthenaJDBC42.jar) diff --git a/docs/docs/zh/guide/datasource/athena.md b/docs/docs/zh/guide/datasource/athena.md new file mode 100644 index 0000000000..77134714b8 --- /dev/null +++ b/docs/docs/zh/guide/datasource/athena.md @@ -0,0 +1,19 @@ +# AWS Athena 数据源 + +![AWS Athena](../../../../img/new_ui/dev/datasource/athena.png) + + +- 数据源：选择 ATHENA +- 数据源名称：输入数据源的名称 +- 描述：输入数据源的描述 +- 用户名：设置连接 AWS 的 access key +- 密码：设置连接 AWS 的 secret access key +- AwsRegion: 设置连接 AWS 的区域 +- 数据库名：输入连接 ATHENA 的数据库名称 +- Jdbc 连接参数：用于 ATHENA 连接的参数设置，以 JSON 形式填写 + +## 是否原生支持 + +- 否，使用前需请参考 [数据源配置](../howto/datasource-setting.md) 中的 "数据源中心" 章节激活数据源。 +- JDBC驱动配置参考文档 [athena-connect-with-jdbc](https://docs.amazonaws.cn/athena/latest/ug/connect-with-jdbc.html) +- 驱动下载链接 [SimbaAthenaJDBC-2.0.31.1000/AthenaJDBC42.jar](https://s3.cn-north-1.amazonaws.com.cn/athena-downloads-cn/drivers/JDBC/SimbaAthenaJDBC-2.0.31.1000/AthenaJDBC42.jar) diff --git a/docs/img/new_ui/dev/datasource/athena.png b/docs/img/new_ui/dev/datasource/athena.png new file mode 100644 index 0000000000..4f6a393626 Binary files /dev/null and b/docs/img/new_ui/dev/datasource/athena.png differ diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml index 9e1a5b104c..cd564e2a2d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml @@ -68,6 +68,11 @@ dolphinscheduler-datasource-redshift ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-datasource-athena + ${project.version} + org.apache.dolphinscheduler dolphinscheduler-datasource-spark diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java index 04c619d955..28217c51d6 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java @@ -41,7 +41,7 @@ public abstract class AbstractDataSourceProcessor implements DataSourceProcessor @Override public void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { checkHost(baseDataSourceParamDTO.getHost()); - checkDatasourcePatter(baseDataSourceParamDTO.getDatabase()); + checkDatabasePatter(baseDataSourceParamDTO.getDatabase()); checkOther(baseDataSourceParamDTO.getOther()); } @@ -61,9 +61,9 @@ public abstract class AbstractDataSourceProcessor implements DataSourceProcessor * * @param database database name */ - protected void checkDatasourcePatter(String database) { + protected void checkDatabasePatter(String database) { if (!DATABASE_PATTER.matcher(database).matches()) { - throw new IllegalArgumentException("datasource name illegal"); + throw new IllegalArgumentException("database name illegal"); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml new file mode 100644 index 0000000000..0ca15dc16d --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml @@ -0,0 +1,44 @@ + + + + + + dolphinscheduler-datasource-plugin + org.apache.dolphinscheduler + dev-SNAPSHOT + + 4.0.0 + + dolphinscheduler-datasource-athena + jar + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + + + + \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java new file mode 100644 index 0000000000..216c0112ac --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.athena; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class AthenaDataSourceChannel implements DataSourceChannel { + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + return new AthenaDataSourceClient(baseConnectionParam,dbType); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java new file mode 100644 index 0000000000..d1439570c3 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.athena; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.auto.service.AutoService; + +@AutoService(DataSourceChannelFactory.class) +public class AthenaDataSourceChannelFactory implements DataSourceChannelFactory { + @Override + public DataSourceChannel create() { + return new AthenaDataSourceChannel(); + } + + @Override + public String getName() { + return "athena"; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java new file mode 100644 index 0000000000..58bde0cce8 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.athena; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class AthenaDataSourceClient extends CommonDataSourceClient { + public AthenaDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + super(baseConnectionParam, dbType); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaConnectionParam.java new file mode 100644 index 0000000000..ce56f5d2bf --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaConnectionParam.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.athena.param; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +public class AthenaConnectionParam extends BaseConnectionParam { + + protected String awsRegion; + + public String getAwsRegion() { + return awsRegion; + } + + public void setAwsRegion(String awsRegion) { + this.awsRegion = awsRegion; + } + + @Override + public String toString() { + return "AthenaConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + + ", other='" + other + '\'' + + ", awsRegion='" + awsRegion + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java new file mode 100644 index 0000000000..ea89de7a3c --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.athena.param; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class AthenaDataSourceParamDTO extends BaseDataSourceParamDTO { + protected String awsRegion; + + @Override + public DbType getType() { + return DbType.ATHENA; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java new file mode 100644 index 0000000000..4187b37a50 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.athena.param; + +import com.google.auto.service.AutoService; +import org.apache.commons.collections4.MapUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@AutoService(DataSourceProcessor.class) +public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor { + + @Override + public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { + return JSONUtils.parseObject(paramJson, AthenaDataSourceParamDTO.class); + } + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + AthenaConnectionParam + connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson); + + AthenaDataSourceParamDTO + athenaDatasourceParamDTO = new AthenaDataSourceParamDTO(); + athenaDatasourceParamDTO.setAwsRegion(connectionParams.getAwsRegion()); + athenaDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + athenaDatasourceParamDTO.setUserName(connectionParams.getUser()); + athenaDatasourceParamDTO.setOther(this.parseOther(connectionParams.getOther())); + + return athenaDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + AthenaDataSourceParamDTO athenaParam = (AthenaDataSourceParamDTO) datasourceParam; + String address = String.format("%s%s=%s;", Constants.JDBC_ATHENA, "AwsRegion", athenaParam.getAwsRegion()); + + AthenaConnectionParam + athenaConnectionParam = new AthenaConnectionParam(); + athenaConnectionParam.setUser(athenaParam.getUserName()); + athenaConnectionParam.setPassword(PasswordUtils.encodePassword(athenaParam.getPassword())); + athenaConnectionParam.setAwsRegion(athenaParam.getAwsRegion()); + athenaConnectionParam.setOther(this.transformOther(athenaParam.getOther())); + athenaConnectionParam.setAddress(address); + athenaConnectionParam.setJdbcUrl(address); + athenaConnectionParam.setDatabase(athenaParam.getDatabase()); + athenaConnectionParam.setDriverClassName(this.getDatasourceDriver()); + athenaConnectionParam.setValidationQuery(this.getValidationQuery()); + athenaConnectionParam.setProps(athenaParam.getOther()); + + return athenaConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, AthenaConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.COM_ATHENA_JDBC_DRIVER; + } + + @Override + public String getValidationQuery() { + return Constants.ATHENA_VALIDATION_QUERY; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + AthenaConnectionParam + athenaConnectionParam = (AthenaConnectionParam) connectionParam; + if (!StringUtils.isEmpty(athenaConnectionParam.getOther())) { + return String.format("%s%s", athenaConnectionParam.getJdbcUrl(), athenaConnectionParam.getOther()); + } + return athenaConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + AthenaConnectionParam athenaConnectionParam = (AthenaConnectionParam) connectionParam; + Class.forName(this.getDatasourceDriver()); + return DriverManager.getConnection(this.getJdbcUrl(connectionParam), + athenaConnectionParam.getUser(), PasswordUtils.decodePassword(athenaConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.ATHENA; + } + + @Override + public DataSourceProcessor create() { + return new AthenaDataSourceProcessor(); + } + + private String transformOther(Map otherMap) { + if (MapUtils.isNotEmpty(otherMap)) { + List list = new ArrayList<>(otherMap.size()); + otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); + return String.join(Constants.SEMICOLON, list); + } + return null; + } + + private Map parseOther(String other) { + Map otherMap = new LinkedHashMap<>(); + if (StringUtils.isEmpty(other)) { + return otherMap; + } + String[] configs = other.split(Constants.SEMICOLON); + for (String config : configs) { + otherMap.put(config.split(Constants.EQUAL_SIGN)[0], config.split(Constants.EQUAL_SIGN)[1]); + } + return otherMap; + } + + @Override + protected void checkHost(String host) { + // Do not need to set the host, nothing to do + } + + @Override + protected void checkDatabasePatter(String database) { + // Do not need to set the database, nothing to do + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/test/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/test/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessorTest.java new file mode 100644 index 0000000000..15a671f064 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/test/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessorTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.athena.param; + +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; + +import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class, DataSourceUtils.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) +public class AthenaDataSourceProcessorTest { + + private final AthenaDataSourceProcessor athenaDataSourceProcessor = new AthenaDataSourceProcessor(); + + @Test + public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("LogLevel", "6"); + AthenaDataSourceParamDTO athenaDataSourceParamDTO = new AthenaDataSourceParamDTO(); + athenaDataSourceParamDTO.setDatabase(""); + athenaDataSourceParamDTO.setUserName("awsuser"); + athenaDataSourceParamDTO.setPassword("123456"); + athenaDataSourceParamDTO.setAwsRegion("cn-north-1"); + athenaDataSourceParamDTO.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); + AthenaConnectionParam connectionParams = (AthenaConnectionParam) this.athenaDataSourceProcessor + .createConnectionParams(athenaDataSourceParamDTO); + Assert.assertEquals("jdbc:awsathena://AwsRegion=cn-north-1;", connectionParams.getAddress()); + Assert.assertEquals("jdbc:awsathena://AwsRegion=cn-north-1;", connectionParams.getJdbcUrl()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionJson = "{\"user\":\"awsuser\",\"password\":\"123456\",\"address\":\"jdbc:awsathena://AwsRegion=cn-north-1;\"" + + ",\"database\":\"\",\"jdbcUrl\":\"jdbc:awsathena://AwsRegion=cn-north-1;\", \"awsRegion\":\"cn-north-1\"}"; + AthenaConnectionParam connectionParams = (AthenaConnectionParam) this.athenaDataSourceProcessor + .createConnectionParams(connectionJson); + Assert.assertNotNull(connectionParams); + Assert.assertEquals("awsuser", connectionParams.getUser()); + Assert.assertEquals("cn-north-1", connectionParams.getAwsRegion()); + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertEquals(Constants.COM_ATHENA_JDBC_DRIVER, this.athenaDataSourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + AthenaConnectionParam athenaConnectionParam = new AthenaConnectionParam(); + athenaConnectionParam.setJdbcUrl("jdbc:awsathena://AwsRegion=cn-north-1;"); + athenaConnectionParam.setOther("LogLevel=6;LogPath=/tmp;"); + Assert.assertEquals("jdbc:awsathena://AwsRegion=cn-north-1;LogLevel=6;LogPath=/tmp;", + this.athenaDataSourceProcessor.getJdbcUrl(athenaConnectionParam)); + + } + + @Test + public void testGetJdbcUrlNoOther() { + AthenaConnectionParam athenaConnectionParam = new AthenaConnectionParam(); + athenaConnectionParam.setJdbcUrl("jdbc:awsathena://AwsRegion=cn-north-1;"); + athenaConnectionParam.setOther(""); + Assert.assertEquals("jdbc:awsathena://AwsRegion=cn-north-1;", + this.athenaDataSourceProcessor.getJdbcUrl(athenaConnectionParam)); + + } + + @Test + public void testGetDbType() { + Assert.assertEquals(DbType.ATHENA, this.athenaDataSourceProcessor.getDbType()); + } + + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.ATHENA_VALIDATION_QUERY, this.athenaDataSourceProcessor.getValidationQuery()); + + } + + @Test + public void testCreateDatasourceParamDTO() { + String connectionJson = "{\"user\":\"awsuser\",\"password\":\"123456\",\"address\":\"jdbc:awsathena://AwsRegion=cn-north-1;\"" + + ",\"database\":\"\",\"jdbcUrl\":\"jdbc:awsathena://AwsRegion=cn-north-1;\", \"awsRegion\":\"cn-north-1\"}"; + AthenaDataSourceParamDTO athenaDataSourceParamDTO = (AthenaDataSourceParamDTO) this.athenaDataSourceProcessor + .createDatasourceParamDTO(connectionJson); + Assert.assertEquals("awsuser", athenaDataSourceParamDTO.getUserName()); + Assert.assertEquals("cn-north-1", athenaDataSourceParamDTO.getAwsRegion()); + Assert.assertEquals("", athenaDataSourceParamDTO.getDatabase()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml index 6517ceba2e..80857c2ba2 100644 --- a/dolphinscheduler-datasource-plugin/pom.xml +++ b/dolphinscheduler-datasource-plugin/pom.xml @@ -41,6 +41,7 @@ dolphinscheduler-datasource-api dolphinscheduler-datasource-all dolphinscheduler-datasource-redshift + dolphinscheduler-datasource-athena diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 869111621a..c1e0144448 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -17,14 +17,14 @@ package org.apache.dolphinscheduler.spi.enums; -import com.baomidou.mybatisplus.annotation.EnumValue; -import com.google.common.base.Functions; +import static java.util.stream.Collectors.toMap; import java.util.Arrays; import java.util.Map; import java.util.NoSuchElementException; -import static java.util.stream.Collectors.toMap; +import com.baomidou.mybatisplus.annotation.EnumValue; +import com.google.common.base.Functions; public enum DbType { MYSQL(0, "mysql"), @@ -38,6 +38,7 @@ public enum DbType { PRESTO(8, "presto"), H2(9, "h2"), REDSHIFT(10,"redshift"), + ATHENA(11,"athena"), ; @EnumValue diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java index cd05241cde..259bb45322 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java @@ -162,6 +162,7 @@ public class Constants { public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"; public static final String COM_REDSHIFT_JDBC_DRIVER = "com.amazon.redshift.jdbc42.Driver"; + public static final String COM_ATHENA_JDBC_DRIVER = "com.simba.athena.jdbc.Driver"; /** @@ -176,6 +177,7 @@ public class Constants { public static final String DB2_VALIDATION_QUERY = "select 1 from sysibm.sysdummy1"; public static final String PRESTO_VALIDATION_QUERY = "select 1"; public static final String REDHIFT_VALIDATION_QUERY = "select 1"; + public static final String ATHENA_VALIDATION_QUERY = "select 1"; /** * jdbc url @@ -190,6 +192,7 @@ public class Constants { public static final String JDBC_DB2 = "jdbc:db2://"; public static final String JDBC_PRESTO = "jdbc:presto://"; public static final String JDBC_REDSHIFT = "jdbc:redshift://"; + public static final String JDBC_ATHENA = "jdbc:awsathena://"; public static final String ADDRESS = "address"; public static final String DATABASE = "database"; diff --git a/dolphinscheduler-ui/src/locales/en_US/datasource.ts b/dolphinscheduler-ui/src/locales/en_US/datasource.ts index 681bce904e..c6ae1e1b2f 100644 --- a/dolphinscheduler-ui/src/locales/en_US/datasource.ts +++ b/dolphinscheduler-ui/src/locales/en_US/datasource.ts @@ -62,5 +62,7 @@ export default { user_name_tips: 'Please enter your username', user_password: 'Password', user_password_tips: 'Please enter your password', + aws_region: 'Aws Region', + aws_region_tips: 'Please enter AwsRegion', jdbc_format_tips: 'jdbc connection parameters is not a correct JSON format' } diff --git a/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts b/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts index 8eaba984a8..a2f7e7cb4e 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts @@ -59,5 +59,7 @@ export default { user_name_tips: '请输入用户名', user_password: '密码', user_password_tips: '请输入密码', + aws_region: 'AwsRegion', + aws_region_tips: '请输入AwsRegion', jdbc_format_tips: 'jdbc连接参数不是一个正确的JSON格式' } diff --git a/dolphinscheduler-ui/src/service/modules/data-source/types.ts b/dolphinscheduler-ui/src/service/modules/data-source/types.ts index ebfec18ab1..06e700c6a1 100644 --- a/dolphinscheduler-ui/src/service/modules/data-source/types.ts +++ b/dolphinscheduler-ui/src/service/modules/data-source/types.ts @@ -26,6 +26,7 @@ type IDataBase = | 'DB2' | 'PRESTO' | 'REDSHIFT' + | 'ATHENA' interface IDataSource { id?: number @@ -40,6 +41,7 @@ interface IDataSource { loginUserKeytabPath?: string userName?: string password?: string + awsRegion?: string database?: string connectType?: string other?: object diff --git a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx index 951ec34e46..5abb05287e 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx +++ b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx @@ -125,6 +125,9 @@ const DetailModal = defineComponent({ detailForm, rules, requiredDataBase, + showHost, + showPort, + showAwsRegion, showConnectType, showPrincipal, loading, @@ -194,6 +197,7 @@ const DetailModal = defineComponent({ /> + + +