diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml index 8da9b912b5..ff206141ce 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml @@ -56,5 +56,9 @@ org.apache.dolphinscheduler dolphinscheduler-datasource-sqlserver + + org.apache.dolphinscheduler + dolphinscheduler-datasource-redshift + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java index e825285b1a..7d9cbb2cf6 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLD import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSQLDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift.RedshiftDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SQLServerDataSourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbType; @@ -46,6 +47,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; * see {@link SQLServerDataSourceParamDTO} * see {@link Db2DataSourceParamDTO} * see {@link PrestoDataSourceParamDTO} + * see {@link RedshiftDataSourceParamDTO} */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { @@ -58,6 +60,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = SQLServerDataSourceParamDTO.class, name = "SQLSERVER"), @JsonSubTypes.Type(value = Db2DataSourceParamDTO.class, name = "DB2"), @JsonSubTypes.Type(value = PrestoDataSourceParamDTO.class, name = "PRESTO"), + @JsonSubTypes.Type(value = RedshiftDataSourceParamDTO.class, name = "REDSHIFT"), }) public abstract class BaseDataSourceParamDTO implements Serializable { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftConnectionParam.java new file mode 100644 index 0000000000..a266a2eb0a --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftConnectionParam.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +public class RedshiftConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "RedshiftConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceParamDTO.java new file mode 100644 index 0000000000..16d3281f40 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceParamDTO.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class RedshiftDataSourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "RedshiftDataSourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.REDSHIFT; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessor.java new file mode 100644 index 0000000000..5a9c521fce --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessor.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; + +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang.StringUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + RedshiftConnectionParam + connectionParams = (RedshiftConnectionParam) createConnectionParams(connectionJson); + + String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); + + RedshiftDataSourceParamDTO + redshiftDatasourceParamDTO = new RedshiftDataSourceParamDTO(); + redshiftDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + redshiftDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + redshiftDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + redshiftDatasourceParamDTO.setUserName(connectionParams.getUser()); + redshiftDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + return redshiftDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + RedshiftDataSourceParamDTO redshiftParam = (RedshiftDataSourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", Constants.JDBC_REDSHIFT, redshiftParam.getHost(), redshiftParam.getPort()); + String jdbcUrl = address + Constants.SLASH + redshiftParam.getDatabase(); + + RedshiftConnectionParam + redshiftConnectionParam = new RedshiftConnectionParam(); + redshiftConnectionParam.setUser(redshiftParam.getUserName()); + redshiftConnectionParam.setPassword(PasswordUtils.encodePassword(redshiftParam.getPassword())); + redshiftConnectionParam.setOther(transformOther(redshiftParam.getOther())); + redshiftConnectionParam.setAddress(address); + redshiftConnectionParam.setJdbcUrl(jdbcUrl); + redshiftConnectionParam.setDatabase(redshiftParam.getDatabase()); + redshiftConnectionParam.setDriverClassName(getDatasourceDriver()); + redshiftConnectionParam.setValidationQuery(getValidationQuery()); + redshiftConnectionParam.setProps(redshiftParam.getOther()); + + return redshiftConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, RedshiftConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.COM_REDSHIFT_JDBC_DRIVER; + } + + @Override + public String getValidationQuery() { + return Constants.REDHIFT_VALIDATION_QUERY; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + RedshiftConnectionParam + redshiftConnectionParam = (RedshiftConnectionParam) connectionParam; + if (!StringUtils.isEmpty(redshiftConnectionParam.getOther())) { + return String.format("%s?%s", redshiftConnectionParam.getJdbcUrl(), redshiftConnectionParam.getOther()); + } + return redshiftConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + RedshiftConnectionParam redshiftConnectionParam = (RedshiftConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(connectionParam), + redshiftConnectionParam.getUser(), PasswordUtils.decodePassword(redshiftConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.REDSHIFT; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isNotEmpty(otherMap)) { + List list = new ArrayList<>(otherMap.size()); + otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); + return String.join(Constants.SEMICOLON, list); + } + return null; + } + + private Map parseOther(String other) { + Map otherMap = new LinkedHashMap<>(); + if (StringUtils.isEmpty(other)) { + return otherMap; + } + String[] configs = other.split(Constants.SEMICOLON); + for (String config : configs) { + otherMap.put(config.split(Constants.EQUAL_SIGN)[0], config.split(Constants.EQUAL_SIGN)[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java index cf41bc2965..edcd1a332d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLD import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSQLDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift.RedshiftDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SQLServerDataSourceProcessor; import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; @@ -52,6 +53,7 @@ public class DataSourceUtils { private static final DataSourceProcessor sqlServerProcessor = new SQLServerDataSourceProcessor(); private static final DataSourceProcessor db2PROCESSOR = new Db2DataSourceProcessor(); private static final DataSourceProcessor prestoPROCESSOR = new PrestoDataSourceProcessor(); + private static final DataSourceProcessor redshiftProcessor = new RedshiftDataSourceProcessor(); /** * check datasource param @@ -120,6 +122,8 @@ public class DataSourceUtils { return db2PROCESSOR; case PRESTO: return prestoPROCESSOR; + case REDSHIFT: + return redshiftProcessor; default: throw new IllegalArgumentException("datasource type illegal:" + dbType); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessorTest.java new file mode 100644 index 0000000000..a1756c3551 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessorTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift; + +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; + +import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class, DataSourceUtils.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) +public class RedshiftDataSourceProcessorTest { + + private RedshiftDataSourceProcessor redshiftDatasourceProcessor = new RedshiftDataSourceProcessor(); + + @Test + public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); + RedshiftDataSourceParamDTO redshiftDatasourceParamDTO = new RedshiftDataSourceParamDTO(); + redshiftDatasourceParamDTO.setHost("localhost"); + redshiftDatasourceParamDTO.setPort(5439); + redshiftDatasourceParamDTO.setDatabase("dev"); + redshiftDatasourceParamDTO.setUserName("awsuser"); + redshiftDatasourceParamDTO.setPassword("123456"); + redshiftDatasourceParamDTO.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); + RedshiftConnectionParam connectionParams = (RedshiftConnectionParam) redshiftDatasourceProcessor + .createConnectionParams(redshiftDatasourceParamDTO); + Assert.assertEquals("jdbc:redshift://localhost:5439", connectionParams.getAddress()); + Assert.assertEquals("jdbc:redshift://localhost:5439/dev", connectionParams.getJdbcUrl()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionJson = "{\"user\":\"awsuser\",\"password\":\"123456\",\"address\":\"jdbc:redshift://localhost:5439\"" + + ",\"database\":\"dev\",\"jdbcUrl\":\"jdbc:redshift://localhost:5439/dev\"}"; + RedshiftConnectionParam connectionParams = (RedshiftConnectionParam) redshiftDatasourceProcessor + .createConnectionParams(connectionJson); + Assert.assertNotNull(connectionParams); + Assert.assertEquals("awsuser", connectionParams.getUser()); + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertEquals(Constants.COM_REDSHIFT_JDBC_DRIVER, redshiftDatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + RedshiftConnectionParam redshiftConnectionParam = new RedshiftConnectionParam(); + redshiftConnectionParam.setJdbcUrl("jdbc:redshift://localhost:5439/default"); + redshiftConnectionParam.setOther("DSILogLevel=6;defaultRowFetchSize=100"); + Assert.assertEquals("jdbc:redshift://localhost:5439/default?DSILogLevel=6;defaultRowFetchSize=100", + redshiftDatasourceProcessor.getJdbcUrl(redshiftConnectionParam)); + + } + + @Test + public void testGetDbType() { + Assert.assertEquals(DbType.REDSHIFT, redshiftDatasourceProcessor.getDbType()); + } + + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.REDHIFT_VALIDATION_QUERY, redshiftDatasourceProcessor.getValidationQuery()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml new file mode 100644 index 0000000000..d837b0d83c --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml @@ -0,0 +1,44 @@ + + + + + dolphinscheduler-datasource-plugin + org.apache.dolphinscheduler + 2.0.4-SNAPSHOT + + 4.0.0 + + dolphinscheduler-datasource-redshift + jar + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + + + + \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java new file mode 100644 index 0000000000..13e8172478 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.redshift; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class RedshiftDataSourceChannel implements DataSourceChannel { + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + return new RedshiftDataSourceClient(baseConnectionParam,dbType); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java new file mode 100644 index 0000000000..576a519a2e --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.redshift; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.auto.service.AutoService; + +@AutoService(DataSourceChannelFactory.class) +public class RedshiftDataSourceChannelFactory implements DataSourceChannelFactory { + @Override + public DataSourceChannel create() { + return new RedshiftDataSourceChannel(); + } + + @Override + public String getName() { + return "redshift"; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java new file mode 100644 index 0000000000..f10f7e1f80 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.redshift; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class RedshiftDataSourceClient extends CommonDataSourceClient { + public RedshiftDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + super(baseConnectionParam, dbType); + } +} diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml index c167daba0c..a8c018ead7 100644 --- a/dolphinscheduler-datasource-plugin/pom.xml +++ b/dolphinscheduler-datasource-plugin/pom.xml @@ -38,5 +38,6 @@ dolphinscheduler-datasource-postgresql dolphinscheduler-datasource-api dolphinscheduler-datasource-all + dolphinscheduler-datasource-redshift diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 2055249d8a..099ec7f767 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -35,7 +35,9 @@ public enum DbType { SQLSERVER(6, "sqlserver"), DB2(7, "db2"), PRESTO(8, "presto"), - H2(9, "h2"); + H2(9, "h2"), + REDSHIFT(10,"redshift"), + ; @EnumValue private final int code; diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java index 8effe9a256..cd05241cde 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java @@ -161,6 +161,7 @@ public class Constants { public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"; + public static final String COM_REDSHIFT_JDBC_DRIVER = "com.amazon.redshift.jdbc42.Driver"; /** @@ -174,6 +175,7 @@ public class Constants { public static final String SQLSERVER_VALIDATION_QUERY = "select 1"; public static final String DB2_VALIDATION_QUERY = "select 1 from sysibm.sysdummy1"; public static final String PRESTO_VALIDATION_QUERY = "select 1"; + public static final String REDHIFT_VALIDATION_QUERY = "select 1"; /** * jdbc url @@ -187,7 +189,7 @@ public class Constants { public static final String JDBC_SQLSERVER = "jdbc:sqlserver://"; public static final String JDBC_DB2 = "jdbc:db2://"; public static final String JDBC_PRESTO = "jdbc:presto://"; - + public static final String JDBC_REDSHIFT = "jdbc:redshift://"; public static final String ADDRESS = "address"; public static final String DATABASE = "database"; @@ -204,6 +206,11 @@ public class Constants { */ public static final String DOUBLE_SLASH = "//"; + /** + * SLASH / + */ + public static final String SLASH = "/"; + /** * comma , */ @@ -214,11 +221,23 @@ public class Constants { */ public static final String COLON = ":"; + /** - * AT SIGN + * AT SIGN @ */ public static final String AT_SIGN = "@"; + /** + * SEMICOLON ; + */ + public static final String SEMICOLON = ";"; + + + /** + * EQUAL_SIGN = + */ + public static final String EQUAL_SIGN = "="; + /** * datasource encryption salt */ diff --git a/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts b/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts index e17c6a241c..ebfec18ab1 100644 --- a/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts +++ b/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts @@ -25,6 +25,7 @@ type IDataBase = | 'SQLSERVER' | 'DB2' | 'PRESTO' + | 'REDSHIFT' interface IDataSource { id?: number diff --git a/dolphinscheduler-ui-next/src/views/datasource/list/use-form.ts b/dolphinscheduler-ui-next/src/views/datasource/list/use-form.ts index 1f091bed70..9ea39ad978 100644 --- a/dolphinscheduler-ui-next/src/views/datasource/list/use-form.ts +++ b/dolphinscheduler-ui-next/src/views/datasource/list/use-form.ts @@ -202,6 +202,11 @@ const datasourceType: IDataBaseOptionKeys = { value: 'PRESTO', label: 'PRESTO', defaultPort: 8080 + }, + REDSHIFT: { + value: 'REDSHIFT', + label: 'REDSHIFT', + defaultPort: 5439 } } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts index c0827845d6..13368029d8 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts @@ -75,6 +75,11 @@ export function useDatasourceType( id: 8, code: 'PRESTO', disabled: false + }, + { + id: 9, + code: 'REDSHIFT', + disabled: false } ] diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue index 08f95fac45..d26c175d2e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue @@ -279,6 +279,10 @@ { value: 'PRESTO', label: 'PRESTO' + }, + { + value: 'REDSHIFT', + label: 'REDSHIFT' } ] } @@ -490,6 +494,9 @@ case 'PRESTO': defaultPort = '8080' break + case 'REDSHIFT': + defaultPort = '5439' + break default: break } diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js index 6d8011f3d8..416dccb6d9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js @@ -113,6 +113,11 @@ export default { id: 8, code: 'PRESTO', disabled: false + }, + { + id: 9, + code: 'REDSHIFT', + disabled: false } ], // Alarm interface diff --git a/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js index 754eb66dd8..a8b4529c73 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js @@ -20,7 +20,7 @@ import io from '@/module/io' export default { /** * Data source creation - * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO + * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO, REDSHIFT * @param "name": string, * @param "desc": string, * @param "parameter":string //{"address":"jdbc:hive2://192.168.220.189:10000","autoReconnect":"true","characterEncoding":"utf8","database":"default","initialTimeout":3000,"jdbcUrl":"jdbc:hive2://192.168.220.189:10000/default","maxReconnect":10,"password":"","useUnicode":true,"user":"hive"} @@ -53,7 +53,7 @@ export default { }, /** * Query data source list - no paging - * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO + * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO, REDSHIFT */ getDatasourcesList ({ state }, payload) { return new Promise((resolve, reject) => { diff --git a/pom.xml b/pom.xml index 673d8d3d8c..0647e3f0c3 100644 --- a/pom.xml +++ b/pom.xml @@ -435,6 +435,11 @@ dolphinscheduler-datasource-sqlserver ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-datasource-redshift + ${project.version} + org.apache.dolphinscheduler