diff --git a/docs/docs/en/guide/datasource/redshift.md b/docs/docs/en/guide/datasource/redshift.md index 60dd982492..712815a3e0 100644 --- a/docs/docs/en/guide/datasource/redshift.md +++ b/docs/docs/en/guide/datasource/redshift.md @@ -1,7 +1,5 @@ # Amazon Redshift -![Redshift datasource](../../../../img/new_ui/dev/datasource/redshift.png) - ## Datasource Parameters | **Datasource** | **Description** | @@ -11,11 +9,29 @@ | Description | Enter a description of the datasource. | | IP/Host Name | Enter the Redshift service IP. | | Port | Enter the Redshift service port. | +| Validation | Enter the Redshift authentication mode. | | Username | Set the username for Redshift connection. | | Password | Set the password for Redshift connection. | | Database Name | Enter the database name of the Redshift connection. | | jdbc connect parameters | Parameter settings for Redshift connection, in JSON format. | +| AccessKeyID | Mode IAM-accessKey access key ID. | +| SecretAccessKey | Mode IAM-accessKey secret access key. | + +### Validation: Password + +![password](../../../../img/new_ui/dev/datasource/redshift-password.png) + +Use AWS redshift database username and password to login. + +### Validation: IAM-accessKey + +![IAM1](../../../../img/new_ui/dev/datasource/redshift-iam1.png) +![IAM2](../../../../img/new_ui/dev/datasource/redshift-iam2.png) + +Use cluster ID, AWS Region, port(optional) and IAM to login. ## Native Supported Yes, could use this datasource by default. + +Read more about Redshift IAM JDBC driver configuration reference document [redshift-connect-IAM-jdbc](https://docs.aws.amazon.com/redshift/latest/mgmt/generating-iam-credentials-configure-jdbc-odbc.html) diff --git a/docs/docs/zh/guide/datasource/redshift.md b/docs/docs/zh/guide/datasource/redshift.md new file mode 100644 index 0000000000..f08753c971 --- /dev/null +++ b/docs/docs/zh/guide/datasource/redshift.md @@ -0,0 +1,36 @@ +# Amazon Redshift + +## 数据源参数 + +使用数据库服务器的用户名和密码验证。 +- 数据源：选择 AZURE Redshift +- 数据源名称：输入数据源的名称 +- 描述：输入数据源的描述 +- IP 主机名：输入连接 Redshift 的 HOST 或 IP ，例如：cluster-name.xxx.region.redshift.amazonaws.com.cn +- 端口：输入连接 Redshift 的端口，默认5439 +- 验证模式：输入 Redshift 的连接模式，目前支持：Password,IAM-accessKey +- 用户名：设置连接 Redshift 的用户名 +- 密码：设置连接 Redshift 的密码 +- 数据库名：输入连接 Redshift 的数据库名称 +- Jdbc 连接参数：用于 Redshift 连接的参数设置，以 JSON 形式填写 +- AccessKeyID：IAM-accessKey模式下的access key ID +- SecretAccessKey：IAM-accessKey模式下的secret access key + +### 验证: Password + +![password](../../../../img/new_ui/dev/datasource/redshift-password.png) + +使用Redshift数据库的用户名和密码验证。 + +### 验证: IAM-accessKey + +![IAM1](../../../../img/new_ui/dev/datasource/redshift-iam1.png) +![IAM2](../../../../img/new_ui/dev/datasource/redshift-iam2.png) + +使用 cluster ID, AWS Region, port(可选) and IAM信息来登录。 + +## 是否原生支持 + +是，数据源不需要任务附加操作即可使用。 + +参考更多关于Redshift相关的JDBC文档[校验模式](https://docs.aws.amazon.com/redshift/latest/mgmt/generating-iam-credentials-configure-jdbc-odbc.html) diff --git a/docs/img/new_ui/dev/datasource/redshift-iam1.png b/docs/img/new_ui/dev/datasource/redshift-iam1.png new file mode 100644 index 0000000000..f2355c63ca Binary files /dev/null and b/docs/img/new_ui/dev/datasource/redshift-iam1.png differ diff --git a/docs/img/new_ui/dev/datasource/redshift-iam2.png b/docs/img/new_ui/dev/datasource/redshift-iam2.png new file mode 100644 index 0000000000..7f5a021dc9 Binary files /dev/null and b/docs/img/new_ui/dev/datasource/redshift-iam2.png differ diff --git a/docs/img/new_ui/dev/datasource/redshift-password.png b/docs/img/new_ui/dev/datasource/redshift-password.png new file mode 100644 index 0000000000..f56c8fbfd4 Binary files /dev/null and b/docs/img/new_ui/dev/datasource/redshift-password.png differ diff --git a/docs/img/new_ui/dev/datasource/redshift.png b/docs/img/new_ui/dev/datasource/redshift.png deleted file mode 100644 index 333fe0214a..0000000000 Binary files a/docs/img/new_ui/dev/datasource/redshift.png and /dev/null differ diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml index 6d2c156b16..a691d3f2cc 100644 --- a/dolphinscheduler-bom/pom.xml +++ b/dolphinscheduler-bom/pom.xml @@ -101,6 +101,8 @@ 1.15 402 1.7.1 + 2.1.0.9 + 1.12.300 @@ -796,6 +798,18 @@ azure-identity ${azure-identity.version} + + com.amazon.redshift + redshift-jdbc42 + ${redshift-jdbc42.version} + test + + + com.amazonaws + aws-java-sdk-redshift + ${aws-java-sdk-redshift.version} + test + diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java index 7400636a9d..58602ba3b2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java @@ -67,6 +67,7 @@ public class DataSourceConstants { public static final String JDBC_DB2 = "jdbc:db2://"; public static final String JDBC_PRESTO = "jdbc:presto://"; public static final String JDBC_REDSHIFT = "jdbc:redshift://"; + public static final String JDBC_REDSHIFT_IAM = "jdbc:redshift:iam://"; public static final String JDBC_ATHENA = "jdbc:awsathena://"; public static final String JDBC_TRINO = "jdbc:trino://"; public static final String JDBC_DAMENG = "jdbc:dm://"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java index ae0bad6b59..a5d7502762 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java @@ -49,7 +49,10 @@ public abstract class AbstractDataSourceProcessor implements DataSourceProcessor @Override public void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { - checkHost(baseDataSourceParamDTO.getHost()); + if (!baseDataSourceParamDTO.getType().equals(DbType.REDSHIFT)) { + // due to redshift use not regular hosts + checkHost(baseDataSourceParamDTO.getHost()); + } checkDatabasePatter(baseDataSourceParamDTO.getDatabase()); checkOther(baseDataSourceParamDTO.getOther()); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml index 041dbea12e..850aef34d1 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml @@ -38,6 +38,14 @@ org.apache.dolphinscheduler dolphinscheduler-datasource-api + + com.amazon.redshift + redshift-jdbc42 + + + com.amazonaws + aws-java-sdk-redshift + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java index c9a31080d5..944d507d6d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java @@ -18,12 +18,65 @@ package org.apache.dolphinscheduler.plugin.datasource.redshift; import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.plugin.datasource.redshift.param.RedshiftAuthMode; +import org.apache.dolphinscheduler.plugin.datasource.redshift.param.RedshiftConnectionParam; +import org.apache.dolphinscheduler.plugin.datasource.redshift.param.RedshiftDataSourceProcessor; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Stopwatch; + public class RedshiftDataSourceClient extends CommonDataSourceClient { + private static final Logger logger = LoggerFactory.getLogger(RedshiftDataSourceClient.class); + public RedshiftDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { super(baseConnectionParam, dbType); } + + @Override + public Connection getConnection() { + RedshiftConnectionParam connectionParam = (RedshiftConnectionParam) this.baseConnectionParam; + if (connectionParam.getMode().equals(RedshiftAuthMode.PASSWORD)) { + return super.getConnection(); + } + return RedshiftDataSourceProcessor.getConnectionByIAM(connectionParam); + } + + @Override + public void checkClient() { + RedshiftConnectionParam connectionParam = (RedshiftConnectionParam) this.baseConnectionParam; + Stopwatch stopwatch = Stopwatch.createStarted(); + String validationQuery = this.baseConnectionParam.getValidationQuery(); + if (connectionParam.getMode().equals(RedshiftAuthMode.PASSWORD)) { + // Checking data source client + try { + this.jdbcTemplate.execute(validationQuery); + } catch (Exception e) { + throw new RuntimeException("JDBC connect failed", e); + } finally { + logger.info("Time to execute check jdbc client with sql {} for {} ms ", + this.baseConnectionParam.getValidationQuery(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); + } + } else { + try (Statement statement = getConnection().createStatement()) { + if (!statement.execute(validationQuery)) { + throw new SQLException("execute check redshift access key failed : " + validationQuery); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } finally { + logger.info("Time to execute check redshift access key with sql {} for {} ms ", + this.baseConnectionParam.getValidationQuery(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); + } + } + } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftAuthMode.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftAuthMode.java new file mode 100644 index 0000000000..2fa62ae2eb --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftAuthMode.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.redshift.param; + +import static java.util.stream.Collectors.toMap; + +import java.util.Arrays; +import java.util.Map; +import java.util.NoSuchElementException; + +import com.fasterxml.jackson.annotation.JsonValue; +import com.google.common.base.Functions; + +public enum RedshiftAuthMode { + + PASSWORD(0, "password"), + IAM_ACCESS_KEY(1, "IAM-accessKey"), + ; + + private static final Map AUTH_TYPE_MAP = + Arrays.stream(RedshiftAuthMode.values()).collect(toMap(RedshiftAuthMode::getCode, Functions.identity())); + private final int code; + @JsonValue + private final String descp; + + RedshiftAuthMode(int code, String descp) { + this.code = code; + this.descp = descp; + } + + public static RedshiftAuthMode of(int type) { + if (AUTH_TYPE_MAP.containsKey(type)) { + return AUTH_TYPE_MAP.get(type); + } + return null; + } + + public static RedshiftAuthMode ofName(String name) { + return Arrays.stream(RedshiftAuthMode.values()).filter(e -> e.name().equals(name)).findFirst() + .orElseThrow(() -> new NoSuchElementException("no such auth type")); + } + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java index dd3c2d1b6e..f671e026d0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java @@ -19,8 +19,16 @@ package org.apache.dolphinscheduler.plugin.datasource.redshift.param; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter public class RedshiftConnectionParam extends BaseConnectionParam { + protected RedshiftAuthMode mode; + protected String dbUser; + @Override public String toString() { return "RedshiftConnectionParam{" @@ -33,6 +41,8 @@ public class RedshiftConnectionParam extends BaseConnectionParam { + ", driverClassName='" + driverClassName + '\'' + ", validationQuery='" + validationQuery + '\'' + ", other='" + other + '\'' + + ", dbUser='" + dbUser + '\'' + + ", mode='" + mode + '\'' + '}'; } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceParamDTO.java index e7f5a49246..6fbb4b23e3 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceParamDTO.java @@ -20,8 +20,21 @@ package org.apache.dolphinscheduler.plugin.datasource.redshift.param; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbType; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter public class RedshiftDataSourceParamDTO extends BaseDataSourceParamDTO { + protected RedshiftAuthMode mode; + protected String dbUser; + + @Override + public DbType getType() { + return DbType.REDSHIFT; + } + @Override public String toString() { return "RedshiftDataSourceParamDTO{" @@ -33,11 +46,8 @@ public class RedshiftDataSourceParamDTO extends BaseDataSourceParamDTO { + ", userName='" + userName + '\'' + ", password='" + password + '\'' + ", other='" + other + '\'' + + ", dbUser='" + dbUser + '\'' + + ", mode='" + mode + '\'' + '}'; } - - @Override - public DbType getType() { - return DbType.REDSHIFT; - } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java index df951f8345..c9163fb5df 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; import java.sql.Connection; import java.sql.DriverManager; @@ -55,8 +56,25 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor { String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); RedshiftDataSourceParamDTO redshiftDatasourceParamDTO = new RedshiftDataSourceParamDTO(); - redshiftDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); - redshiftDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + redshiftDatasourceParamDTO.setMode(connectionParams.getMode()); + redshiftDatasourceParamDTO.setDbUser(connectionParams.getDbUser()); + if (connectionParams.getMode().equals(RedshiftAuthMode.PASSWORD)) { + redshiftDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + redshiftDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + } else { + if (hostPortArray[0].contains(Constants.COLON)) { + String portString = hostPortArray[0].split(Constants.COLON)[1]; + if (StringUtils.isNumeric(portString)) { + redshiftDatasourceParamDTO.setPort(Integer.parseInt(portString)); + redshiftDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + } else { + redshiftDatasourceParamDTO.setHost(hostPortArray[0]); + } + } else { + redshiftDatasourceParamDTO.setHost(hostPortArray[0]); + } + + } redshiftDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); redshiftDatasourceParamDTO.setUserName(connectionParams.getUser()); redshiftDatasourceParamDTO.setOther(connectionParams.getOther()); @@ -67,9 +85,7 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { RedshiftDataSourceParamDTO redshiftParam = (RedshiftDataSourceParamDTO) datasourceParam; - String address = - String.format("%s%s:%s", DataSourceConstants.JDBC_REDSHIFT, redshiftParam.getHost(), - redshiftParam.getPort()); + String address = getAddress(redshiftParam); String jdbcUrl = address + Constants.SLASH + redshiftParam.getDatabase(); RedshiftConnectionParam redshiftConnectionParam = new RedshiftConnectionParam(); @@ -81,6 +97,8 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor { redshiftConnectionParam.setDatabase(redshiftParam.getDatabase()); redshiftConnectionParam.setDriverClassName(getDatasourceDriver()); redshiftConnectionParam.setValidationQuery(getValidationQuery()); + redshiftConnectionParam.setMode(redshiftParam.getMode()); + redshiftConnectionParam.setDbUser(redshiftParam.getDbUser()); return redshiftConnectionParam; } @@ -114,8 +132,14 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor { public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { RedshiftConnectionParam redshiftConnectionParam = (RedshiftConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), - redshiftConnectionParam.getUser(), PasswordUtils.decodePassword(redshiftConnectionParam.getPassword())); + if (redshiftConnectionParam.getMode().equals(RedshiftAuthMode.PASSWORD)) { + return DriverManager.getConnection(getJdbcUrl(connectionParam), + redshiftConnectionParam.getUser(), + PasswordUtils.decodePassword(redshiftConnectionParam.getPassword())); + } else if (redshiftConnectionParam.getMode().equals(RedshiftAuthMode.IAM_ACCESS_KEY)) { + return getConnectionByIAM(redshiftConnectionParam); + } + return null; } @Override @@ -128,7 +152,34 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor { return new RedshiftDataSourceProcessor(); } - private String transformOther(Map otherMap) { + /** + * 2 auth mode + * PASSWORD: address example: jdbc:redshift://examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com:5439 + * IAM_ACCESS_KEY: + * address example1: jdbc:redshift:iam://examplecluster:us-west-2 + * address example2: jdbc:redshift:iam://examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com:5439 + * + * @param redshiftParam + * @return + */ + private String getAddress(RedshiftDataSourceParamDTO redshiftParam) { + if (redshiftParam.getMode().equals(RedshiftAuthMode.PASSWORD)) { + return String.format("%s%s:%s", DataSourceConstants.JDBC_REDSHIFT, redshiftParam.getHost(), + redshiftParam.getPort()); + } else if (redshiftParam.getMode().equals(RedshiftAuthMode.IAM_ACCESS_KEY)) { + if (redshiftParam.getPort() == null) { + // construct IAM_ACCESS_KEY example 1 format + return String.format("%s%s", DataSourceConstants.JDBC_REDSHIFT_IAM, redshiftParam.getHost()); + } else { + // construct IAM_ACCESS_KEY example 2 format + return String.format("%s%s:%s", DataSourceConstants.JDBC_REDSHIFT_IAM, redshiftParam.getHost(), + redshiftParam.getPort()); + } + } + return null; + } + + private static String transformOther(Map otherMap) { if (MapUtils.isNotEmpty(otherMap)) { List list = new ArrayList<>(otherMap.size()); otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); @@ -137,4 +188,42 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor { return null; } + /** + * 2 auth mode + * PASSWORD: address example: jdbc:redshift://examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com:5439/dev + * IAM_ACCESS_KEY: + * address example1: jdbc:redshift:iam://examplecluster:us-west-2/dev?AccessKeyID=xxx&SecretAccessKey=xxx&DbUser=kristen + * address example2: jdbc:redshift:iam://examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com:5439/dev?AccessKeyID=xxx&SecretAccessKey=xxx&DbUser=kristen + * + * @param redshiftConnectionParam + * @return + */ + public static Connection getConnectionByIAM(RedshiftConnectionParam redshiftConnectionParam) { + String basic; + String authParams = String.format("AccessKeyID=%s&SecretAccessKey=%s&DbUser=%s", + redshiftConnectionParam.getUser(), PasswordUtils.decodePassword(redshiftConnectionParam.getPassword()), + redshiftConnectionParam.getDbUser()); + String connectionUrl; + if (MapUtils.isNotEmpty(redshiftConnectionParam.getOther())) { + basic = String.format("%s?%s", redshiftConnectionParam.getJdbcUrl(), + transformOther(redshiftConnectionParam.getOther())); + // if have other params map, basic will be + // 'jdbc:redshift:iam://examplecluster:us-west-2/dev?param1=xx¶m2=xx' + // append AccessKeyID &SecretAccessKey &DbUser + connectionUrl = String.format("%s&%s", basic, authParams); + } else { + basic = redshiftConnectionParam.getJdbcUrl(); + // if none other params map, basic will be 'jdbc:redshift:iam://examplecluster:us-west-2/dev' + // append AccessKeyID &SecretAccessKey &DbUser + connectionUrl = String.format("%s?%s", basic, authParams); + } + try { + Class.forName(DataSourceConstants.COM_REDSHIFT_JDBC_DRIVER); + return DriverManager.getConnection(connectionUrl); + } catch (SQLException e) { + throw new RuntimeException(e); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/test/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/test/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessorTest.java index 1c752667b4..2e61f71536 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/test/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/test/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessorTest.java @@ -49,7 +49,7 @@ public class RedshiftDataSourceProcessorTest { redshiftDatasourceParamDTO.setUserName("awsuser"); redshiftDatasourceParamDTO.setPassword("123456"); redshiftDatasourceParamDTO.setOther(props); - + redshiftDatasourceParamDTO.setMode(RedshiftAuthMode.PASSWORD); try (MockedStatic mockedStaticPasswordUtils = Mockito.mockStatic(PasswordUtils.class)) { mockedStaticPasswordUtils.when(() -> PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); RedshiftConnectionParam connectionParams = (RedshiftConnectionParam) redshiftDatasourceProcessor @@ -63,7 +63,7 @@ public class RedshiftDataSourceProcessorTest { public void testCreateConnectionParams2() { String connectionJson = "{\"user\":\"awsuser\",\"password\":\"123456\",\"address\":\"jdbc:redshift://localhost:5439\"" - + ",\"database\":\"dev\",\"jdbcUrl\":\"jdbc:redshift://localhost:5439/dev\"}"; + + ",\"database\":\"dev\",\"jdbcUrl\":\"jdbc:redshift://localhost:5439/dev\",\"mode\":\"password\"}"; RedshiftConnectionParam connectionParams = (RedshiftConnectionParam) redshiftDatasourceProcessor .createConnectionParams(connectionJson); Assertions.assertNotNull(connectionParams); diff --git a/dolphinscheduler-ui/src/locales/en_US/datasource.ts b/dolphinscheduler-ui/src/locales/en_US/datasource.ts index 6c1e48eee8..dfa8e6d01d 100644 --- a/dolphinscheduler-ui/src/locales/en_US/datasource.ts +++ b/dolphinscheduler-ui/src/locales/en_US/datasource.ts @@ -86,4 +86,10 @@ export default { clientSecret: 'ClientSecret', OAuth_token_endpoint: 'OAuth 2.0 token endpoint', endpoint_tips: 'Please enter OAuth Token', + AccessKeyID:'AccessKeyID', + AccessKeyID_tips:'Please input AccessKeyID', + SecretAccessKey:'SecretAccessKey', + SecretAccessKey_tips:'Please input SecretAccessKey', + dbUser:'DbUser', + dbUser_tips:'Please input DbUser', } diff --git a/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts b/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts index 48c29992e3..749647270d 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts @@ -82,5 +82,11 @@ export default { clientId: 'ClientId', clientSecret: 'ClientSecret', OAuth_token_endpoint: 'OAuth 2.0 token endpoint', - endpoint_tips: '请输入OAuth' + endpoint_tips: '请输入OAuth', + AccessKeyID:'AccessKeyID', + AccessKeyID_tips:'请输入AccessKeyID', + SecretAccessKey:'SecretAccessKey', + SecretAccessKey_tips:'请输入SecretAccessKey', + dbUser:'DbUser', + dbUser_tips:'请输入DbUser', } diff --git a/dolphinscheduler-ui/src/service/modules/data-source/types.ts b/dolphinscheduler-ui/src/service/modules/data-source/types.ts index 0d0b3e8867..aa43909bc4 100644 --- a/dolphinscheduler-ui/src/service/modules/data-source/types.ts +++ b/dolphinscheduler-ui/src/service/modules/data-source/types.ts @@ -72,6 +72,7 @@ interface IDataSource { bindTestId?: number endpoint?: string MSIClientId?: string + dbUser?: string } interface ListReq { diff --git a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx index bb46683265..0d9dd6235a 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx +++ b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx @@ -162,6 +162,7 @@ const DetailModal = defineComponent({ showPrincipal, showMode, modeOptions, + redShitModeOptions, loading, saving, testing, @@ -245,7 +246,9 @@ const DetailModal = defineComponent({ v-show={showPort} label={t('datasource.port')} path='port' - show-require-mark + show-require-mark={ + !(showMode && detailForm.mode === 'IAM-accessKey') + } > {/* SqlPassword */} @@ -427,10 +430,47 @@ const DetailModal = defineComponent({ placeholder={t('datasource.OAuth_token_endpoint')} /> - - - - + + + + + + + + + diff --git a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts index dcf15e766c..0159905a34 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts +++ b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts @@ -54,7 +54,8 @@ export function useForm(id?: number) { testFlag: -1, bindTestId: undefined, endpoint: '', - MSIClientId: '' + MSIClientId: '', + dbUser: '' } as IDataSourceDetail const state = reactive({ @@ -88,6 +89,9 @@ export function useForm(id?: number) { port: { trigger: ['input'], validator() { + if (state.showMode && state.detailForm.mode === 'IAM-accessKey') { + return + } if (!state.detailForm.port && state.showPort) { return new Error(t('datasource.port_tips')) } @@ -180,6 +184,18 @@ export function useForm(id?: number) { } } }, + dbUser: { + trigger: ['input'], + validator() { + if ( + !state.detailForm.dbUser && + state.showMode && + state.detailForm.mode === 'IAM-accessKey' + ) { + return new Error(t('datasource.IAM-accessKey')) + } + } + } // databaseUserName: { // trigger: ['input'], // validator() { @@ -210,6 +226,16 @@ export function useForm(id?: number) { label: "accessToken", value: 'accessToken', }, + ], + redShitModeOptions: [ + { + label: 'password', + value: 'password' + }, + { + label: 'IAM-accessKey', + value: 'IAM-accessKey' + } ] }) @@ -222,7 +248,7 @@ export function useForm(id?: number) { state.showHost = type !== 'ATHENA' state.showPort = type !== 'ATHENA' state.showAwsRegion = type === 'ATHENA' - state.showMode = type === 'AZURESQL' + state.showMode = ['AZURESQL', 'REDSHIFT'].includes(type) if (type === 'ORACLE' && !id) { state.detailForm.connectType = 'ORACLE_SERVICE_NAME'